feat: data lifecycle rewrite, download clients, wanted list, and central catalog index
Ships the fresh-start schema cleanup: rebuilt explore catalog index pipeline (dump import, artifact fetch/build, incremental listen-count refresh), a new download subsystem (Lidarr/Prowlarr/qBittorrent/SABnzbd/ slskd/yt-dlp providers, staging, reconciliation, wanted list), and the supporting schema/query/store changes across backend and frontend. Also includes two smaller follow-ups: bump the central index's rebuild-after cadence from 90 to 180 days, and remove the Explore "library only" online/offline toggle entirely (frontend-only, no backend counterpart) rather than carry unused UI/state. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y2Agd9af5hE7qzti2ackiS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,167 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Every remote provider here talks to a self-hosted service over the
|
||||
// same shape of API: a base URL, one header carrying an API key, JSON
|
||||
// in and out, and a handful of status codes that mean the same thing
|
||||
// everywhere. This is that, once.
|
||||
//
|
||||
// Each adapter supplies its own sentinel errors so callers can still
|
||||
// distinguish "slskd is down" from "Prowlarr is down" with errors.Is.
|
||||
|
||||
// apiClient is a small JSON-over-HTTP client for a self-hosted service.
|
||||
type apiClient struct {
|
||||
http *http.Client
|
||||
|
||||
baseURL string
|
||||
authKey string
|
||||
authValue string
|
||||
|
||||
// errUnreachable and errAuth are the adapter's sentinels, returned
|
||||
// for transport failures and rejected credentials respectively.
|
||||
errUnreachable error
|
||||
errAuth error
|
||||
}
|
||||
|
||||
// newAPIClient builds a client for one service.
|
||||
func newAPIClient(
|
||||
baseURL, authHeader, authValue string,
|
||||
timeout time.Duration,
|
||||
errUnreachable, errAuth error,
|
||||
) *apiClient {
|
||||
return &apiClient{
|
||||
http: &http.Client{Timeout: timeout},
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
authKey: authHeader,
|
||||
authValue: authValue,
|
||||
errUnreachable: errUnreachable,
|
||||
errAuth: errAuth,
|
||||
}
|
||||
}
|
||||
|
||||
// get performs a GET, decoding the response into out when non-nil.
|
||||
func (c *apiClient) get(ctx context.Context, endpoint string, out any) error {
|
||||
return c.do(ctx, http.MethodGet, endpoint, nil, out)
|
||||
}
|
||||
|
||||
// post performs a POST with a JSON body.
|
||||
func (c *apiClient) post(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
body, out any,
|
||||
) error {
|
||||
return c.do(ctx, http.MethodPost, endpoint, body, out)
|
||||
}
|
||||
|
||||
// put performs a PUT with a JSON body.
|
||||
func (c *apiClient) put(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
body, out any,
|
||||
) error {
|
||||
return c.do(ctx, http.MethodPut, endpoint, body, out)
|
||||
}
|
||||
|
||||
// delete performs a DELETE.
|
||||
func (c *apiClient) delete(ctx context.Context, endpoint string) error {
|
||||
return c.do(ctx, http.MethodDelete, endpoint, nil, nil)
|
||||
}
|
||||
|
||||
// do performs one request.
|
||||
func (c *apiClient) do(
|
||||
ctx context.Context,
|
||||
method, endpoint string,
|
||||
body, out any,
|
||||
) error {
|
||||
var reader io.Reader
|
||||
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode request body: %w", err)
|
||||
}
|
||||
|
||||
reader = bytes.NewReader(encoded)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, method, c.baseURL+endpoint, reader,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
|
||||
if c.authKey != "" {
|
||||
req.Header.Set(c.authKey, c.authValue)
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", c.errUnreachable, err)
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if err := c.checkStatus(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkStatus maps HTTP status onto the adapter's sentinel errors,
|
||||
// including a snippet of the body — self-hosted services put the useful
|
||||
// part of a failure there, not in the status line.
|
||||
func (c *apiClient) checkStatus(resp *http.Response) error {
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusUnauthorized,
|
||||
resp.StatusCode == http.StatusForbidden:
|
||||
return c.errAuth
|
||||
case resp.StatusCode >= 400:
|
||||
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
|
||||
return fmt.Errorf(
|
||||
"%w: HTTP %d: %s",
|
||||
c.errUnreachable,
|
||||
resp.StatusCode,
|
||||
strings.TrimSpace(string(snippet)),
|
||||
)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// decodeJSON decodes a JSON string into out. Providers whose auth or
|
||||
// response handling does not fit apiClient still parse bodies the same
|
||||
// way, so the helper lives here rather than being repeated.
|
||||
func decodeJSON(body string, out any) error {
|
||||
if err := json.Unmarshal([]byte(body), out); err != nil {
|
||||
return fmt.Errorf("decode json: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg Config
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "slskd defaults to one",
|
||||
cfg: Config{Kind: KindSlskd},
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "usenet defaults higher",
|
||||
cfg: Config{Kind: KindSABnzbd},
|
||||
want: 4,
|
||||
},
|
||||
{
|
||||
name: "explicit override wins",
|
||||
cfg: Config{
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "3"},
|
||||
},
|
||||
want: 3,
|
||||
},
|
||||
{
|
||||
name: "nonsense override falls back",
|
||||
cfg: Config{
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "not a number"},
|
||||
},
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "zero override falls back",
|
||||
cfg: Config{
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "0"},
|
||||
},
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "unknown kind falls back to the global default",
|
||||
cfg: Config{Kind: Kind("something-new")},
|
||||
want: defaultConcurrency,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := concurrencyFor(tt.cfg); got != tt.want {
|
||||
t.Errorf("%s: got %d, want %d", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The reason the per-provider cap exists: a Soulseek daemon capped at
|
||||
// one transfer must serialize, even when the global cap would allow
|
||||
// more and the user has queued several albums at once.
|
||||
func TestPerProviderCapSerializesTransfers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetMaxConcurrent(4)
|
||||
|
||||
slow := fakeWithAlbum(1, "slskd-like", ".flac")
|
||||
slow.GrabGate = make(chan struct{})
|
||||
|
||||
f.manager.installProvider(Config{
|
||||
ID: 1,
|
||||
Kind: KindSlskd,
|
||||
Priority: 50,
|
||||
}, slow)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Three requests against the same one-at-a-time provider.
|
||||
for i := range 3 {
|
||||
req := fourTrackRequest()
|
||||
req.ID = "req-" + string(rune('a'+i))
|
||||
|
||||
if err := f.store.CreateRequest(ctx, req); err != nil {
|
||||
t.Fatalf("CreateRequest: %v", err)
|
||||
}
|
||||
|
||||
candidate := slow.Candidates[0]
|
||||
candidate.ProviderID = 1
|
||||
|
||||
go f.manager.grab(ctx, req, candidate, nil)
|
||||
}
|
||||
|
||||
// Give all three a chance to reach the transport, then check how
|
||||
// many actually got through the gate.
|
||||
waitFor(t, func() bool { return slow.GrabCallCount() >= 1 }, "no grab started")
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
if got := slow.MaxParallelGrabs(); got != 1 {
|
||||
t.Errorf("%d simultaneous transfers, want 1", got)
|
||||
}
|
||||
|
||||
close(slow.GrabGate)
|
||||
|
||||
waitFor(
|
||||
t,
|
||||
func() bool { return slow.GrabCallCount() == 3 },
|
||||
"not every queued transfer ran once the first finished",
|
||||
)
|
||||
|
||||
if got := slow.MaxParallelGrabs(); got != 1 {
|
||||
t.Errorf("%d simultaneous transfers overall, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider that tolerates parallelism is not held to Soulseek's
|
||||
// limit, and the global cap is what bounds it.
|
||||
func TestPerProviderCapAllowsParallelWhereSafe(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetMaxConcurrent(4)
|
||||
|
||||
fast := fakeWithAlbum(1, "usenet-like", ".flac")
|
||||
fast.GrabGate = make(chan struct{})
|
||||
|
||||
f.manager.installProvider(Config{
|
||||
ID: 1,
|
||||
Kind: KindSABnzbd,
|
||||
Priority: 50,
|
||||
}, fast)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := range 3 {
|
||||
req := fourTrackRequest()
|
||||
req.ID = "req-" + string(rune('a'+i))
|
||||
|
||||
if err := f.store.CreateRequest(ctx, req); err != nil {
|
||||
t.Fatalf("CreateRequest: %v", err)
|
||||
}
|
||||
|
||||
candidate := fast.Candidates[0]
|
||||
candidate.ProviderID = 1
|
||||
|
||||
go f.manager.grab(ctx, req, candidate, nil)
|
||||
}
|
||||
|
||||
waitFor(
|
||||
t,
|
||||
func() bool { return fast.MaxParallelGrabs() >= 3 },
|
||||
"transfers were serialized against a provider that allows parallelism",
|
||||
)
|
||||
|
||||
close(fast.GrabGate)
|
||||
}
|
||||
|
||||
// Reload must not strand a running transfer's slot when a provider's
|
||||
// limit changes underneath it.
|
||||
func TestSyncSemaphoresReplacesChangedLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd}, nil)
|
||||
|
||||
first := f.manager.semaphoreFor(1)
|
||||
if cap(first) != 1 {
|
||||
t.Fatalf("slskd semaphore cap = %d, want 1", cap(first))
|
||||
}
|
||||
|
||||
// Same limit: the semaphore is kept, so in-flight accounting is not
|
||||
// reset by an unrelated settings save.
|
||||
f.manager.syncSemaphores(map[int64]Config{1: {ID: 1, Kind: KindSlskd}})
|
||||
|
||||
if again := f.manager.semaphoreFor(1); again != first {
|
||||
t.Error("semaphore was replaced despite an unchanged limit")
|
||||
}
|
||||
|
||||
// Changed limit: a new semaphore, with the new capacity.
|
||||
f.manager.syncSemaphores(map[int64]Config{1: {
|
||||
ID: 1,
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "5"},
|
||||
}})
|
||||
|
||||
f.manager.provMu.Lock()
|
||||
f.manager.configs[1] = Config{
|
||||
ID: 1,
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "5"},
|
||||
}
|
||||
f.manager.provMu.Unlock()
|
||||
|
||||
if changed := f.manager.semaphoreFor(1); cap(changed) != 5 {
|
||||
t.Errorf("semaphore cap = %d after raising the limit, want 5", cap(changed))
|
||||
}
|
||||
|
||||
// A provider that is gone leaves no semaphore behind.
|
||||
f.manager.syncSemaphores(map[int64]Config{})
|
||||
|
||||
f.manager.semMu.Lock()
|
||||
_, still := f.manager.provSem[1]
|
||||
f.manager.semMu.Unlock()
|
||||
|
||||
if still {
|
||||
t.Error("semaphore survived the provider being removed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package download
|
||||
|
||||
import "time"
|
||||
|
||||
// UserConfig is the download subsystem's slice of the TOML config file.
|
||||
// Provider connections are not here — they live in the database, keyed
|
||||
// by row, because there can be many of them and they change through the
|
||||
// settings UI rather than by hand-editing.
|
||||
type UserConfig struct {
|
||||
// PathTemplate lays out imported files under the library root.
|
||||
// Tokens: {albumartist} {artist} {album} {year} {track} {disc}
|
||||
// {title}. Empty falls back to DefaultPathTemplate.
|
||||
PathTemplate string `toml:"PathTemplate"`
|
||||
|
||||
// AutoPick lets a single high-confidence, high-quality candidate
|
||||
// download without asking. Off by default: an unattended download
|
||||
// that picks wrong puts the wrong files in the library, and the
|
||||
// ranking has to earn that trust on a given user's sources first.
|
||||
AutoPick bool `toml:"AutoPick"`
|
||||
|
||||
// MaxConcurrent bounds simultaneous transfers across all providers.
|
||||
// Per-provider limits sit underneath it and are set on the provider
|
||||
// itself, since the right number depends on what is on the other
|
||||
// end: one Soulseek peer, or a usenet server built for parallelism.
|
||||
MaxConcurrent int `toml:"MaxConcurrent"`
|
||||
|
||||
// WantedIntervalMinutes is how often the wanted list is reconciled:
|
||||
// artist subscriptions expanded, owned items retired, due wants
|
||||
// searched for. Zero uses the default.
|
||||
WantedIntervalMinutes int `toml:"WantedIntervalMinutes"`
|
||||
|
||||
// WantedBatch bounds how many wants one reconcile pass searches
|
||||
// for. A large list should be worked through steadily rather than
|
||||
// in one burst that every provider sees as a flood.
|
||||
WantedBatch int `toml:"WantedBatch"`
|
||||
}
|
||||
|
||||
// ApplyDefaults fills unset fields.
|
||||
func (c *UserConfig) ApplyDefaults() {
|
||||
if c.PathTemplate == "" {
|
||||
c.PathTemplate = DefaultPathTemplate
|
||||
}
|
||||
|
||||
if c.MaxConcurrent <= 0 {
|
||||
c.MaxConcurrent = defaultConcurrency
|
||||
}
|
||||
|
||||
if c.WantedIntervalMinutes <= 0 {
|
||||
c.WantedIntervalMinutes = int(defaultReconcileInterval / time.Minute)
|
||||
}
|
||||
|
||||
if c.WantedBatch <= 0 {
|
||||
c.WantedBatch = defaultDueBatch
|
||||
}
|
||||
}
|
||||
|
||||
// WantedInterval is the reconcile interval as a duration.
|
||||
func (c *UserConfig) WantedInterval() time.Duration {
|
||||
return time.Duration(c.WantedIntervalMinutes) * time.Minute
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// The fake provider exists so the pipeline can be tested end to end
|
||||
// without a network, a daemon, or a binary on PATH. It is registered
|
||||
// like any real adapter and excluded from Descriptors(), so it can
|
||||
// never be offered in the UI.
|
||||
|
||||
// FakeProvider is an in-memory Provider used by tests. It fills all
|
||||
// three roles; which ones are active is controlled by its Caps.
|
||||
type FakeProvider struct {
|
||||
info ProviderInfo
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
// Candidates is what Search returns.
|
||||
Candidates []Candidate
|
||||
|
||||
// SearchErr, GrabErr and CheckErr are returned when set.
|
||||
SearchErr error
|
||||
GrabErr error
|
||||
CheckErr error
|
||||
|
||||
// Written maps a relative file name to the bytes Grab creates in
|
||||
// the staging directory, simulating a completed transfer.
|
||||
Written map[string][]byte
|
||||
|
||||
// DelegateStatuses is returned by Poll in order, the last repeating.
|
||||
DelegateStatuses []DelegateStatus
|
||||
|
||||
// GrabGate, when set, blocks Grab until it is closed or the
|
||||
// context ends. It lets a test hold transfers open long enough to
|
||||
// observe how many run at once.
|
||||
GrabGate chan struct{}
|
||||
|
||||
// concurrent tracks how many Grabs are in flight, and maxParallel
|
||||
// the high-water mark, which is what a concurrency cap is asserted
|
||||
// against.
|
||||
concurrent int
|
||||
maxParallel int
|
||||
|
||||
// Calls records what happened, for assertions.
|
||||
SearchCalls int
|
||||
GrabCalls int
|
||||
DelegateCalls int
|
||||
pollIndex int
|
||||
}
|
||||
|
||||
// NewFakeProvider returns a fake with the given capabilities.
|
||||
func NewFakeProvider(id int64, name string, caps Caps) *FakeProvider {
|
||||
return &FakeProvider{
|
||||
info: ProviderInfo{
|
||||
ID: id,
|
||||
Kind: KindFake,
|
||||
Name: name,
|
||||
Enabled: true,
|
||||
Priority: 50,
|
||||
Caps: caps,
|
||||
},
|
||||
Written: map[string][]byte{},
|
||||
}
|
||||
}
|
||||
|
||||
// Info returns the fake's identity.
|
||||
func (f *FakeProvider) Info() ProviderInfo {
|
||||
return f.info
|
||||
}
|
||||
|
||||
// SetPriority adjusts the fake's priority.
|
||||
func (f *FakeProvider) SetPriority(p int) {
|
||||
f.info.Priority = p
|
||||
}
|
||||
|
||||
// Check reports configured health.
|
||||
func (f *FakeProvider) Check(_ context.Context) error {
|
||||
return f.CheckErr
|
||||
}
|
||||
|
||||
// Close is a no-op.
|
||||
func (f *FakeProvider) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search returns the configured candidates.
|
||||
func (f *FakeProvider) Search(
|
||||
ctx context.Context,
|
||||
_ Request,
|
||||
) ([]Candidate, error) {
|
||||
f.mu.Lock()
|
||||
f.SearchCalls++
|
||||
f.mu.Unlock()
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err //nolint:wrapcheck // context error, already meaningful
|
||||
}
|
||||
|
||||
if f.SearchErr != nil {
|
||||
return nil, f.SearchErr
|
||||
}
|
||||
|
||||
out := make([]Candidate, len(f.Candidates))
|
||||
copy(out, f.Candidates)
|
||||
|
||||
for i := range out {
|
||||
out[i].ProviderID = f.info.ID
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Grab writes the configured files into dst.
|
||||
func (f *FakeProvider) Grab(
|
||||
ctx context.Context,
|
||||
_ Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error) {
|
||||
f.mu.Lock()
|
||||
f.GrabCalls++
|
||||
f.concurrent++
|
||||
|
||||
if f.concurrent > f.maxParallel {
|
||||
f.maxParallel = f.concurrent
|
||||
}
|
||||
|
||||
gate := f.GrabGate
|
||||
f.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
f.mu.Lock()
|
||||
f.concurrent--
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
|
||||
if gate != nil {
|
||||
select {
|
||||
case <-gate:
|
||||
case <-ctx.Done():
|
||||
return Result{}, ctx.Err() //nolint:wrapcheck // context error
|
||||
}
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err //nolint:wrapcheck // context error
|
||||
}
|
||||
|
||||
if f.GrabErr != nil {
|
||||
return Result{}, f.GrabErr
|
||||
}
|
||||
|
||||
files := make([]string, 0, len(f.Written))
|
||||
|
||||
var total int64
|
||||
|
||||
for name, data := range f.Written {
|
||||
path := filepath.Join(dst, name)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
return Result{}, err //nolint:wrapcheck // test helper
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, data, 0o640); err != nil {
|
||||
return Result{}, err //nolint:wrapcheck // test helper
|
||||
}
|
||||
|
||||
files = append(files, path)
|
||||
total += int64(len(data))
|
||||
|
||||
if onProgress != nil {
|
||||
onProgress(Progress{Current: total, Total: total})
|
||||
}
|
||||
}
|
||||
|
||||
return Result{Dir: dst, Files: files, BytesTransferred: total}, nil
|
||||
}
|
||||
|
||||
// GrabCallCount reports how many transfers have been started, safe to
|
||||
// read while transfers are in flight.
|
||||
func (f *FakeProvider) GrabCallCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
return f.GrabCalls
|
||||
}
|
||||
|
||||
// MaxParallelGrabs reports the most simultaneous transfers this
|
||||
// provider ever saw, which is what a per-provider cap is asserted
|
||||
// against.
|
||||
func (f *FakeProvider) MaxParallelGrabs() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
return f.maxParallel
|
||||
}
|
||||
|
||||
// errNoDelegateStatus is returned when a fake delegator runs out of
|
||||
// scripted statuses.
|
||||
var errNoDelegateStatus = errors.New("fake: no delegate status configured")
|
||||
|
||||
// Delegate records the call and returns a fixed external ID.
|
||||
func (f *FakeProvider) Delegate(
|
||||
_ context.Context,
|
||||
_ Request,
|
||||
) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.DelegateCalls++
|
||||
|
||||
return "fake-external-1", nil
|
||||
}
|
||||
|
||||
// Poll returns the next scripted status, repeating the last.
|
||||
func (f *FakeProvider) Poll(
|
||||
_ context.Context,
|
||||
_ string,
|
||||
) (DelegateStatus, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if len(f.DelegateStatuses) == 0 {
|
||||
return DelegateStatus{}, errNoDelegateStatus
|
||||
}
|
||||
|
||||
i := f.pollIndex
|
||||
if i >= len(f.DelegateStatuses) {
|
||||
i = len(f.DelegateStatuses) - 1
|
||||
} else {
|
||||
f.pollIndex++
|
||||
}
|
||||
|
||||
return f.DelegateStatuses[i], nil
|
||||
}
|
||||
|
||||
// Withdraw is a no-op.
|
||||
func (f *FakeProvider) Withdraw(_ context.Context, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeRegistry lets tests install providers directly, bypassing the
|
||||
// database and the constructor registry.
|
||||
func (m *Manager) installProvider(cfg Config, p Provider) {
|
||||
m.provMu.Lock()
|
||||
defer m.provMu.Unlock()
|
||||
|
||||
m.providers[cfg.ID] = p
|
||||
m.configs[cfg.ID] = cfg
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindFake,
|
||||
Name: "Fake (testing)",
|
||||
Summary: "In-memory provider used by the test suite.",
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
},
|
||||
},
|
||||
func(cfg Config, _ SecretLookup, _ *slog.Logger) (Provider, error) {
|
||||
return NewFakeProvider(cfg.ID, cfg.Name, Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
}), nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// slogDiscard returns a logger that writes nowhere, for tests.
|
||||
func slogDiscard() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelError + 1,
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"yellowjacket/backend/tagwriter"
|
||||
)
|
||||
|
||||
// The import step is the only writer into library paths. Everything
|
||||
// before it happens in staging, where a bad download is a directory to
|
||||
// delete rather than a row to un-ingest.
|
||||
//
|
||||
// Order matters: tags are written while the files are still staged, so
|
||||
// the scanner's first sight of a file is already correct. Tagging
|
||||
// after the move would mean a window where the library holds a track
|
||||
// titled "01 - Track01.flac", and the user would watch it fix itself.
|
||||
|
||||
// Import errors.
|
||||
var (
|
||||
// ErrNoAudio means the grab produced no playable audio files.
|
||||
ErrNoAudio = errors.New("download contained no audio files")
|
||||
|
||||
// ErrTooIncomplete means too few of the expected tracks arrived to
|
||||
// call the download successful.
|
||||
ErrTooIncomplete = errors.New("download is missing too many tracks")
|
||||
|
||||
// ErrDestinationExists means the computed library path is already
|
||||
// occupied by a different file.
|
||||
ErrDestinationExists = errors.New("destination file already exists")
|
||||
)
|
||||
|
||||
// minCompleteness is the fraction of the expected tracklist that must
|
||||
// arrive for an anchored import to proceed. Below this the download is
|
||||
// a different thing than what was asked for — a single, a sampler, a
|
||||
// partial transfer — and quietly importing it would corrupt the
|
||||
// library's idea of the album.
|
||||
const minCompleteness = 0.8
|
||||
|
||||
// TagWriterPort is the tag-writing capability the importer needs.
|
||||
// Narrow interface rather than *tagwriter.TagWriter so importer tests
|
||||
// do not need a database.
|
||||
type TagWriterPort interface {
|
||||
WriteUntrackedFileTags(filePath string, changes tagwriter.TagChanges) error
|
||||
}
|
||||
|
||||
// LibraryPort is the library-side capability the importer needs.
|
||||
type LibraryPort interface {
|
||||
// ScanLibrary triggers a rescan so imported files are ingested.
|
||||
ScanLibrary(id int64) error
|
||||
}
|
||||
|
||||
// ImportOptions configures how imported files are laid out.
|
||||
type ImportOptions struct {
|
||||
// LibraryRoot is the directory imported files are placed under.
|
||||
LibraryRoot string
|
||||
|
||||
// PathTemplate lays out the destination path. Supported tokens:
|
||||
// {albumartist} {artist} {album} {year} {track} {disc} {title}.
|
||||
// Empty means flat: everything into LibraryRoot/{albumartist}/{album}.
|
||||
PathTemplate string
|
||||
|
||||
// WriteTags controls whether the importer tags files before moving
|
||||
// them. Off for delegate providers, which have already imported
|
||||
// and tagged the files themselves.
|
||||
WriteTags bool
|
||||
}
|
||||
|
||||
// DefaultPathTemplate is the layout used when none is configured.
|
||||
const DefaultPathTemplate = "{albumartist}/{album}/{track} {title}"
|
||||
|
||||
// Importer moves verified downloads into the library.
|
||||
type Importer struct {
|
||||
logger *slog.Logger
|
||||
staging *Staging
|
||||
tags TagWriterPort
|
||||
library LibraryPort
|
||||
}
|
||||
|
||||
// NewImporter builds an importer.
|
||||
func NewImporter(
|
||||
logger *slog.Logger,
|
||||
staging *Staging,
|
||||
tags TagWriterPort,
|
||||
library LibraryPort,
|
||||
) *Importer {
|
||||
return &Importer{
|
||||
logger: logger,
|
||||
staging: staging,
|
||||
tags: tags,
|
||||
library: library,
|
||||
}
|
||||
}
|
||||
|
||||
// ImportResult reports what an import placed where.
|
||||
type ImportResult struct {
|
||||
// Paths are the library paths files ended up at.
|
||||
Paths []string
|
||||
|
||||
// Tagged counts files whose tags were rewritten.
|
||||
Tagged int
|
||||
|
||||
// Skipped counts non-audio files left in staging (logs, cue sheets,
|
||||
// scene .nfo files) — deliberately not imported.
|
||||
Skipped int
|
||||
}
|
||||
|
||||
// Import verifies, tags and moves a completed grab into the library.
|
||||
//
|
||||
// On any failure the staging directory is left intact so the user can
|
||||
// retry or inspect it; only a fully successful import releases staging.
|
||||
func (i *Importer) Import(
|
||||
ctx context.Context,
|
||||
req Request,
|
||||
result Result,
|
||||
opts ImportOptions,
|
||||
) (ImportResult, error) {
|
||||
files, err := i.staging.Verify(result.Dir, result.Files)
|
||||
if err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
|
||||
audio, skipped := splitAudio(files)
|
||||
if len(audio) == 0 {
|
||||
return ImportResult{}, ErrNoAudio
|
||||
}
|
||||
|
||||
if err := checkCompleteness(len(audio), req); err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
|
||||
// Align staged files to the expected tracklist so tags and
|
||||
// filenames reflect the release, not the uploader's naming.
|
||||
plan := i.planFiles(audio, req)
|
||||
|
||||
out := ImportResult{
|
||||
Paths: make([]string, 0, len(plan)),
|
||||
Skipped: skipped,
|
||||
}
|
||||
|
||||
for _, p := range plan {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return out, fmt.Errorf("import cancelled: %w", err)
|
||||
}
|
||||
|
||||
if opts.WriteTags {
|
||||
if err := i.tagFile(p, req); err != nil {
|
||||
// A file that cannot be tagged is still worth importing
|
||||
// — the scanner will read whatever tags it has, and the
|
||||
// autotag queue can pick it up later. Losing the whole
|
||||
// album over one unwritable file would be worse.
|
||||
i.logger.Warn(
|
||||
"could not tag downloaded file before import",
|
||||
"path", p.Source,
|
||||
"error", err,
|
||||
)
|
||||
} else {
|
||||
out.Tagged++
|
||||
}
|
||||
}
|
||||
|
||||
dest, err := i.destinationFor(p, req, opts)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
if err := movePath(p.Source, dest); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
out.Paths = append(out.Paths, dest)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// plannedFile pairs a staged file with the expected track it matched.
|
||||
type plannedFile struct {
|
||||
Source string
|
||||
|
||||
// Track is the matched expected track, or the zero value when the
|
||||
// file could not be aligned (free-text requests, bonus tracks).
|
||||
Track ExpectedTrack
|
||||
Matched bool
|
||||
}
|
||||
|
||||
// planFiles aligns staged files to the expected tracklist.
|
||||
func (i *Importer) planFiles(audio []string, req Request) []plannedFile {
|
||||
files := make([]CandidateFile, 0, len(audio))
|
||||
|
||||
for _, a := range audio {
|
||||
format, isAudio := FormatForPath(a)
|
||||
files = append(files, CandidateFile{
|
||||
Path: a,
|
||||
Format: format,
|
||||
IsAudio: isAudio,
|
||||
})
|
||||
}
|
||||
|
||||
matched, _ := matchFiles(files, req.Expected)
|
||||
|
||||
byPosition := make(map[int]ExpectedTrack, len(req.Expected))
|
||||
for _, e := range req.Expected {
|
||||
byPosition[e.Position] = e
|
||||
}
|
||||
|
||||
out := make([]plannedFile, 0, len(matched))
|
||||
|
||||
for _, m := range matched {
|
||||
p := plannedFile{Source: m.Path}
|
||||
|
||||
if t, ok := byPosition[m.MatchedTo]; ok && m.MatchedTo != 0 {
|
||||
p.Track = t
|
||||
p.Matched = true
|
||||
}
|
||||
|
||||
out = append(out, p)
|
||||
}
|
||||
|
||||
// Stable order: matched tracks by position, then unmatched by path,
|
||||
// so a partial import is reproducible.
|
||||
sort.SliceStable(out, func(a, b int) bool {
|
||||
if out[a].Matched != out[b].Matched {
|
||||
return out[a].Matched
|
||||
}
|
||||
|
||||
if out[a].Matched {
|
||||
if out[a].Track.DiscNumber != out[b].Track.DiscNumber {
|
||||
return out[a].Track.DiscNumber < out[b].Track.DiscNumber
|
||||
}
|
||||
|
||||
return out[a].Track.Position < out[b].Track.Position
|
||||
}
|
||||
|
||||
return out[a].Source < out[b].Source
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// tagFile writes the release's metadata onto a staged file.
|
||||
func (i *Importer) tagFile(p plannedFile, req Request) error {
|
||||
if i.tags == nil || !p.Matched {
|
||||
return nil
|
||||
}
|
||||
|
||||
changes := tagwriter.TagChanges{
|
||||
tagwriter.FieldAlbum: req.Album,
|
||||
tagwriter.FieldAlbumArtist: req.Artist,
|
||||
tagwriter.FieldTitle: p.Track.Title,
|
||||
tagwriter.FieldTrackNumber: p.Track.Position,
|
||||
}
|
||||
|
||||
if p.Track.Artist != "" {
|
||||
changes[tagwriter.FieldArtist] = p.Track.Artist
|
||||
} else {
|
||||
changes[tagwriter.FieldArtist] = req.Artist
|
||||
}
|
||||
|
||||
if p.Track.DiscNumber > 0 {
|
||||
changes[tagwriter.FieldDiscNumber] = p.Track.DiscNumber
|
||||
}
|
||||
|
||||
if err := i.tags.WriteUntrackedFileTags(p.Source, changes); err != nil {
|
||||
return fmt.Errorf("write tags: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// destinationFor computes a file's library path from the template.
|
||||
func (i *Importer) destinationFor(
|
||||
p plannedFile,
|
||||
req Request,
|
||||
opts ImportOptions,
|
||||
) (string, error) {
|
||||
if opts.LibraryRoot == "" {
|
||||
return "", fmt.Errorf(
|
||||
"%w: no library root configured", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
tmpl := opts.PathTemplate
|
||||
if tmpl == "" {
|
||||
tmpl = DefaultPathTemplate
|
||||
}
|
||||
|
||||
ext := filepath.Ext(p.Source)
|
||||
|
||||
title := p.Track.Title
|
||||
if title == "" {
|
||||
// Unmatched file: keep the uploader's name rather than
|
||||
// inventing one, so nothing is silently renamed to a track it
|
||||
// may not be.
|
||||
title = strings.TrimSuffix(filepath.Base(p.Source), ext)
|
||||
}
|
||||
|
||||
artist := p.Track.Artist
|
||||
if artist == "" {
|
||||
artist = req.Artist
|
||||
}
|
||||
|
||||
repl := strings.NewReplacer(
|
||||
"{albumartist}", sanitizePathPart(fallback(req.Artist, "Unknown Artist")),
|
||||
"{artist}", sanitizePathPart(fallback(artist, "Unknown Artist")),
|
||||
"{album}", sanitizePathPart(fallback(req.Album, "Unknown Album")),
|
||||
"{title}", sanitizePathPart(title),
|
||||
"{track}", trackToken(p.Track.Position),
|
||||
"{disc}", strconv.Itoa(p.Track.DiscNumber),
|
||||
"{year}", "",
|
||||
)
|
||||
|
||||
rel := repl.Replace(tmpl)
|
||||
|
||||
// Clean up any empty segments left by unset tokens.
|
||||
parts := make([]string, 0, 4)
|
||||
|
||||
for _, seg := range strings.Split(rel, "/") {
|
||||
seg = strings.TrimSpace(seg)
|
||||
if seg != "" {
|
||||
parts = append(parts, seg)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
"%w: path template produced an empty path", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
dest := filepath.Join(opts.LibraryRoot, filepath.Join(parts...)) + ext
|
||||
|
||||
return uniqueDestination(dest)
|
||||
}
|
||||
|
||||
// uniqueDestination returns dest, or a numbered variant when dest is
|
||||
// taken. Overwriting is never right here: the existing file may be a
|
||||
// better copy the user already owns, and the download is not
|
||||
// authoritative just because it arrived later.
|
||||
func uniqueDestination(dest string) (string, error) {
|
||||
const maxAttempts = 50
|
||||
|
||||
ext := filepath.Ext(dest)
|
||||
base := strings.TrimSuffix(dest, ext)
|
||||
|
||||
for n := range maxAttempts {
|
||||
candidate := dest
|
||||
if n > 0 {
|
||||
candidate = base + " (" + strconv.Itoa(n+1) + ")" + ext
|
||||
}
|
||||
|
||||
_, err := os.Stat(candidate)
|
||||
if os.IsNotExist(err) {
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stat destination: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("%w: %s", ErrDestinationExists, dest)
|
||||
}
|
||||
|
||||
// movePath moves a file, falling back to copy+remove when the staging
|
||||
// area and the library are on different filesystems — which is the
|
||||
// normal case, since staging lives in the user data directory.
|
||||
func movePath(src, dest string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil {
|
||||
return fmt.Errorf("create destination directory: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(src, dest); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := copyFile(src, dest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Remove(src); err != nil {
|
||||
// The copy succeeded, so the import is good; a leftover staged
|
||||
// file is swept later.
|
||||
return nil //nolint:nilerr // staging sweep handles the leftover
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies src to dest, writing to a temporary file first so an
|
||||
// interrupted copy never leaves a partial file at a library path where
|
||||
// the scanner would find it.
|
||||
func copyFile(src, dest string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open downloaded file: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = in.Close() }()
|
||||
|
||||
tmp := dest + ".part"
|
||||
|
||||
out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create library file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
_ = os.Remove(tmp)
|
||||
|
||||
return fmt.Errorf("copy into library: %w", err)
|
||||
}
|
||||
|
||||
if err := out.Close(); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
|
||||
return fmt.Errorf("close library file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, dest); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
|
||||
return fmt.Errorf("finalize library file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkCompleteness rejects an anchored download that is missing too
|
||||
// much of its tracklist.
|
||||
func checkCompleteness(got int, req Request) error {
|
||||
if len(req.Expected) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ratio := float64(got) / float64(len(req.Expected))
|
||||
if ratio < minCompleteness {
|
||||
return fmt.Errorf(
|
||||
"%w: got %d of %d tracks",
|
||||
ErrTooIncomplete, got, len(req.Expected),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitAudio partitions verified files into audio and a count of the
|
||||
// rest.
|
||||
func splitAudio(files []string) (audio []string, skipped int) {
|
||||
audio = make([]string, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
if _, ok := FormatForPath(f); ok {
|
||||
audio = append(audio, f)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
skipped++
|
||||
}
|
||||
|
||||
return audio, skipped
|
||||
}
|
||||
|
||||
// trackToken formats a track number as a zero-padded two-digit string,
|
||||
// or empty when unknown.
|
||||
func trackToken(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
if n < 10 {
|
||||
return "0" + strconv.Itoa(n)
|
||||
}
|
||||
|
||||
return strconv.Itoa(n)
|
||||
}
|
||||
|
||||
// fallback returns s, or alt when s is blank.
|
||||
func fallback(s, alt string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return alt
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// sanitizePathPart makes a string safe as a single path segment on
|
||||
// every supported platform: Windows reserves characters that are legal
|
||||
// on Linux, and a library synced between the two must not produce
|
||||
// unopenable files.
|
||||
func sanitizePathPart(s string) string {
|
||||
const maxSegment = 120
|
||||
|
||||
var b strings.Builder
|
||||
|
||||
b.Grow(len(s))
|
||||
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '/', '\\', ':', '*', '?', '"', '<', '>', '|':
|
||||
b.WriteByte('_')
|
||||
default:
|
||||
if r < 0x20 {
|
||||
continue
|
||||
}
|
||||
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
out := strings.TrimSpace(b.String())
|
||||
|
||||
// Trailing dots and spaces are silently stripped by Windows, which
|
||||
// turns "Vol. 2 " into a name that no longer round-trips.
|
||||
out = strings.TrimRight(out, ". ")
|
||||
|
||||
if len(out) > maxSegment {
|
||||
out = strings.TrimSpace(out[:maxSegment])
|
||||
}
|
||||
|
||||
if out == "" {
|
||||
return "Unknown"
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/tagwriter"
|
||||
)
|
||||
|
||||
// recordingTagWriter captures tag writes instead of touching files, so
|
||||
// importer tests do not need real audio.
|
||||
type recordingTagWriter struct {
|
||||
mu sync.Mutex
|
||||
writes map[string]tagwriter.TagChanges
|
||||
failFor string
|
||||
}
|
||||
|
||||
func newRecordingTagWriter() *recordingTagWriter {
|
||||
return &recordingTagWriter{writes: map[string]tagwriter.TagChanges{}}
|
||||
}
|
||||
|
||||
var errTagWriteFailed = errors.New("tag write failed")
|
||||
|
||||
func (r *recordingTagWriter) WriteUntrackedFileTags(
|
||||
path string,
|
||||
changes tagwriter.TagChanges,
|
||||
) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if r.failFor != "" && strings.Contains(path, r.failFor) {
|
||||
return errTagWriteFailed
|
||||
}
|
||||
|
||||
r.writes[filepath.Base(path)] = changes
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// stubLibrary records scan requests.
|
||||
type stubLibrary struct {
|
||||
mu sync.Mutex
|
||||
scanned []int64
|
||||
}
|
||||
|
||||
func (s *stubLibrary) ScanLibrary(id int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.scanned = append(s.scanned, id)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// importFixture stages a set of files and returns the pieces an import
|
||||
// needs.
|
||||
type importFixture struct {
|
||||
staging *Staging
|
||||
importer *Importer
|
||||
tags *recordingTagWriter
|
||||
lib *stubLibrary
|
||||
dir string
|
||||
root string
|
||||
files []string
|
||||
}
|
||||
|
||||
func newImportFixture(t *testing.T, names ...string) importFixture {
|
||||
t.Helper()
|
||||
|
||||
staging := newTestStaging(t)
|
||||
tags := newRecordingTagWriter()
|
||||
lib := &stubLibrary{}
|
||||
imp := NewImporter(slogDiscard(), staging, tags, lib)
|
||||
|
||||
dir, err := staging.Reserve("item-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
files := make([]string, 0, len(names))
|
||||
|
||||
for _, n := range names {
|
||||
p := filepath.Join(dir, n)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(p, []byte("audio-data"), 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
files = append(files, p)
|
||||
}
|
||||
|
||||
return importFixture{
|
||||
staging: staging,
|
||||
importer: imp,
|
||||
tags: tags,
|
||||
lib: lib,
|
||||
dir: dir,
|
||||
root: t.TempDir(),
|
||||
files: files,
|
||||
}
|
||||
}
|
||||
|
||||
func fourTrackRequest() Request {
|
||||
return Request{
|
||||
ID: "req-1",
|
||||
LibraryID: 1,
|
||||
ReleaseMBID: "mbid-1",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
Expected: []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
{Position: 3, Title: "Subterranean Homesick Alien"},
|
||||
{Position: 4, Title: "Exit Music (For a Film)"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportPlacesAndTagsFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t,
|
||||
"01 - Airbag.flac",
|
||||
"02 - Paranoid Android.flac",
|
||||
"03 - Subterranean Homesick Alien.flac",
|
||||
"04 - Exit Music (For a Film).flac",
|
||||
)
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(),
|
||||
fourTrackRequest(),
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Paths) != 4 {
|
||||
t.Fatalf("imported %d files, want 4", len(got.Paths))
|
||||
}
|
||||
|
||||
if got.Tagged != 4 {
|
||||
t.Errorf("tagged %d files, want 4", got.Tagged)
|
||||
}
|
||||
|
||||
want := filepath.Join(
|
||||
f.root, "Radiohead", "OK Computer", "01 Airbag.flac",
|
||||
)
|
||||
|
||||
if got.Paths[0] != want {
|
||||
t.Errorf("first path = %s, want %s", got.Paths[0], want)
|
||||
}
|
||||
|
||||
for _, p := range got.Paths {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Errorf("imported file missing: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tags must be written while the file is still staged: if the library
|
||||
// ever sees an untagged file, the scanner ingests it and the user
|
||||
// watches it correct itself.
|
||||
func TestImportTagsBeforeMoving(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "01 - Airbag.flac")
|
||||
|
||||
req := fourTrackRequest()
|
||||
req.Expected = req.Expected[:1]
|
||||
|
||||
if _, err := f.importer.Import(
|
||||
context.Background(),
|
||||
req,
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
); err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
changes, ok := f.tags.writes["01 - Airbag.flac"]
|
||||
if !ok {
|
||||
t.Fatalf(
|
||||
"tags were not written to the staged filename; got writes for %v",
|
||||
keysOf(f.tags.writes),
|
||||
)
|
||||
}
|
||||
|
||||
if changes[tagwriter.FieldTitle] != "Airbag" {
|
||||
t.Errorf("title = %v, want Airbag", changes[tagwriter.FieldTitle])
|
||||
}
|
||||
|
||||
if changes[tagwriter.FieldAlbum] != "OK Computer" {
|
||||
t.Errorf("album = %v, want OK Computer", changes[tagwriter.FieldAlbum])
|
||||
}
|
||||
}
|
||||
|
||||
// One unwritable file should not cost the whole album.
|
||||
func TestImportContinuesWhenTaggingFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t,
|
||||
"01 - Airbag.flac",
|
||||
"02 - Paranoid Android.flac",
|
||||
"03 - Subterranean Homesick Alien.flac",
|
||||
"04 - Exit Music (For a Film).flac",
|
||||
)
|
||||
f.tags.failFor = "Paranoid"
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(),
|
||||
fourTrackRequest(),
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Paths) != 4 {
|
||||
t.Errorf("imported %d files, want all 4", len(got.Paths))
|
||||
}
|
||||
|
||||
if got.Tagged != 3 {
|
||||
t.Errorf("tagged %d, want 3 (one failure)", got.Tagged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsTooIncomplete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "01 - Airbag.flac")
|
||||
|
||||
_, err := f.importer.Import(
|
||||
context.Background(),
|
||||
fourTrackRequest(),
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrTooIncomplete) {
|
||||
t.Fatalf("error = %v, want ErrTooIncomplete", err)
|
||||
}
|
||||
|
||||
entries, _ := os.ReadDir(f.root)
|
||||
if len(entries) != 0 {
|
||||
t.Error("failed import wrote into the library root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRejectsNoAudio(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "rip.log", "cover.jpg")
|
||||
|
||||
_, err := f.importer.Import(
|
||||
context.Background(),
|
||||
fourTrackRequest(),
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrNoAudio) {
|
||||
t.Fatalf("error = %v, want ErrNoAudio", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportSkipsNonAudioFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t,
|
||||
"01 - Airbag.flac",
|
||||
"02 - Paranoid Android.flac",
|
||||
"03 - Subterranean Homesick Alien.flac",
|
||||
"04 - Exit Music (For a Film).flac",
|
||||
"rip.log",
|
||||
"cover.jpg",
|
||||
)
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(),
|
||||
fourTrackRequest(),
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
if got.Skipped != 2 {
|
||||
t.Errorf("skipped = %d, want 2", got.Skipped)
|
||||
}
|
||||
|
||||
if len(got.Paths) != 4 {
|
||||
t.Errorf("imported %d, want 4 audio files only", len(got.Paths))
|
||||
}
|
||||
}
|
||||
|
||||
// An existing file is never overwritten: it may be a better copy the
|
||||
// user already owns, and arriving later does not make a download
|
||||
// authoritative.
|
||||
func TestImportNeverOverwrites(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "01 - Airbag.flac")
|
||||
|
||||
req := fourTrackRequest()
|
||||
req.Expected = req.Expected[:1]
|
||||
|
||||
existing := filepath.Join(
|
||||
f.root, "Radiohead", "OK Computer", "01 Airbag.flac",
|
||||
)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(existing), 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(existing, []byte("original"), 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(),
|
||||
req,
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(existing)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != "original" {
|
||||
t.Error("import overwrote an existing library file")
|
||||
}
|
||||
|
||||
if got.Paths[0] == existing {
|
||||
t.Errorf("imported to the occupied path %s", existing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCustomPathTemplate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "01 - Airbag.flac")
|
||||
|
||||
req := fourTrackRequest()
|
||||
req.Expected = req.Expected[:1]
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(),
|
||||
req,
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{
|
||||
LibraryRoot: f.root,
|
||||
PathTemplate: "{albumartist} - {album}/{track}. {title}",
|
||||
WriteTags: true,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
want := filepath.Join(
|
||||
f.root, "Radiohead - OK Computer", "01. Airbag.flac",
|
||||
)
|
||||
|
||||
if got.Paths[0] != want {
|
||||
t.Errorf("path = %s, want %s", got.Paths[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizePathPart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"AC/DC", "AC_DC"},
|
||||
{"Where Are We Now?", "Where Are We Now_"},
|
||||
{`Bad: Title*`, "Bad_ Title_"},
|
||||
{"Vol. 2 ", "Vol. 2"},
|
||||
{"trailing dots...", "trailing dots"},
|
||||
{"", "Unknown"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := sanitizePathPart(tt.in); got != tt.want {
|
||||
t.Errorf("sanitizePathPart(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRequiresLibraryRoot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t, "01 - Airbag.flac")
|
||||
|
||||
req := fourTrackRequest()
|
||||
req.Expected = req.Expected[:1]
|
||||
|
||||
_, err := f.importer.Import(
|
||||
context.Background(),
|
||||
req,
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{WriteTags: true},
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrNotConfigured) {
|
||||
t.Fatalf("error = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(m map[string]tagwriter.TagChanges) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,507 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// managerFixture wires a manager over a real (in-memory) database, a
|
||||
// temp staging area and a temp library root, with no network anywhere.
|
||||
type managerFixture struct {
|
||||
manager *Manager
|
||||
store *Store
|
||||
staging *Staging
|
||||
lib *stubLibrary
|
||||
tags *recordingTagWriter
|
||||
root string
|
||||
}
|
||||
|
||||
func newManagerFixture(t *testing.T) managerFixture {
|
||||
t.Helper()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
seedLibrary(t, db)
|
||||
|
||||
store := NewStore(db)
|
||||
staging := newTestStaging(t)
|
||||
tags := newRecordingTagWriter()
|
||||
lib := &stubLibrary{}
|
||||
root := t.TempDir()
|
||||
|
||||
imp := NewImporter(slogDiscard(), staging, tags, lib)
|
||||
|
||||
m := NewManager(
|
||||
slogDiscard(), store, NewMemSecretStore(), staging, imp, lib,
|
||||
)
|
||||
m.SetImportOptions(ImportOptions{LibraryRoot: root})
|
||||
|
||||
return managerFixture{
|
||||
manager: m,
|
||||
store: store,
|
||||
staging: staging,
|
||||
lib: lib,
|
||||
tags: tags,
|
||||
root: root,
|
||||
}
|
||||
}
|
||||
|
||||
// seedLibrary inserts the library row download_requests references.
|
||||
func seedLibrary(t *testing.T, db *database.DB) {
|
||||
t.Helper()
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO libraries (id, name, path) VALUES (1, 'Test', '/music')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed library: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeWithAlbum returns a fake provider that finds and can deliver the
|
||||
// four-track reference album.
|
||||
func fakeWithAlbum(id int64, name string, ext string) *FakeProvider {
|
||||
f := NewFakeProvider(id, name, Caps{CanSearch: true, CanTransport: true})
|
||||
|
||||
titles := allTitles()
|
||||
c := candidateFor(name+"-cand", titles, ext, 30_000_000)
|
||||
c.ProviderID = id
|
||||
|
||||
f.Candidates = []Candidate{c}
|
||||
|
||||
for i, tt := range titles {
|
||||
f.Written[trackToken(i+1)+" - "+tt+ext] = []byte("audio-data")
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func TestManagerSearchRanksAcrossProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
mp3 := fakeWithAlbum(1, "mp3-source", ".mp3")
|
||||
flac := fakeWithAlbum(2, "flac-source", ".flac")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, mp3)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 50}, flac)
|
||||
|
||||
ranked, err := f.manager.Search(context.Background(), fourTrackRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(ranked) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2", len(ranked))
|
||||
}
|
||||
|
||||
if ranked[0].ProviderID != 2 {
|
||||
t.Errorf(
|
||||
"winner from provider %d, want 2 (FLAC)", ranked[0].ProviderID,
|
||||
)
|
||||
}
|
||||
|
||||
if mp3.SearchCalls != 1 || flac.SearchCalls != 1 {
|
||||
t.Errorf(
|
||||
"search calls: mp3=%d flac=%d, want 1 each",
|
||||
mp3.SearchCalls, flac.SearchCalls,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// One broken provider must not take the others down with it.
|
||||
func TestManagerSearchToleratesProviderFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
broken := NewFakeProvider(1, "broken", Caps{CanSearch: true})
|
||||
broken.SearchErr = errors.New("connection refused") //nolint:err113 // test
|
||||
|
||||
working := fakeWithAlbum(2, "working", ".flac")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, broken)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 50}, working)
|
||||
|
||||
ranked, err := f.manager.Search(context.Background(), fourTrackRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(ranked) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1 from the working provider", len(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerSearchNoProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
_, err := f.manager.Search(context.Background(), fourTrackRequest())
|
||||
if !errors.Is(err, ErrNoProviders) {
|
||||
t.Fatalf("error = %v, want ErrNoProviders", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerSearchNoCandidates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
empty := NewFakeProvider(1, "empty", Caps{CanSearch: true})
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, empty)
|
||||
|
||||
_, err := f.manager.Search(context.Background(), fourTrackRequest())
|
||||
if !errors.Is(err, ErrNoCandidates) {
|
||||
t.Fatalf("error = %v, want ErrNoCandidates", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole pipeline: request → search → auto-pick → grab → verify →
|
||||
// tag → import → library scan.
|
||||
func TestManagerEndToEndAutoPick(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
provider := fakeWithAlbum(1, "flac-source", ".flac")
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
ranked, err := f.manager.Start(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
if !AutoPickable(req, ranked) {
|
||||
t.Fatalf(
|
||||
"expected a clear winner to auto-pick; best match %f quality %f",
|
||||
ranked[0].Match.Overall, ranked[0].Quality.Overall,
|
||||
)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateComplete)
|
||||
|
||||
if provider.GrabCalls != 1 {
|
||||
t.Errorf("grab calls = %d, want 1", provider.GrabCalls)
|
||||
}
|
||||
|
||||
// Files landed in the library, laid out by the template.
|
||||
want := filepath.Join(f.root, "Radiohead", "OK Computer", "01 Airbag.flac")
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Errorf("expected imported file at %s: %v", want, err)
|
||||
}
|
||||
|
||||
// Staging was released only after a successful import.
|
||||
entries, err := os.ReadDir(f.staging.Root())
|
||||
if err != nil {
|
||||
t.Fatalf("read staging root: %v", err)
|
||||
}
|
||||
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("staging not released: %d dirs remain", len(entries))
|
||||
}
|
||||
|
||||
// The library was told to rescan.
|
||||
f.lib.mu.Lock()
|
||||
scanned := len(f.lib.scanned)
|
||||
f.lib.mu.Unlock()
|
||||
|
||||
if scanned != 1 {
|
||||
t.Errorf("library scans = %d, want 1", scanned)
|
||||
}
|
||||
}
|
||||
|
||||
// An ambiguous result set must park for the user rather than guess.
|
||||
func TestManagerWaitsWhenAmbiguous(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
a := fakeWithAlbum(1, "source-a", ".flac")
|
||||
b := fakeWithAlbum(2, "source-b", ".flac")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, a)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 50}, b)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
ranked, err := f.manager.Start(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
if AutoPickable(req, ranked) {
|
||||
t.Fatal("two equivalent candidates must not auto-pick")
|
||||
}
|
||||
|
||||
// Nothing was grabbed while waiting for the user.
|
||||
if a.GrabCalls != 0 || b.GrabCalls != 0 {
|
||||
t.Errorf(
|
||||
"grabs happened without a pick: a=%d b=%d",
|
||||
a.GrabCalls, b.GrabCalls,
|
||||
)
|
||||
}
|
||||
|
||||
stored, err := f.store.GetRequest(context.Background(), req.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetRequest: %v", err)
|
||||
}
|
||||
|
||||
if stored.ID != req.ID {
|
||||
t.Errorf("stored request id = %s, want %s", stored.ID, req.ID)
|
||||
}
|
||||
|
||||
// The user picks the second one explicitly.
|
||||
if err := f.manager.Pick(
|
||||
context.Background(), req.ID, ranked[1].ID,
|
||||
); err != nil {
|
||||
t.Fatalf("Pick: %v", err)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateComplete)
|
||||
}
|
||||
|
||||
func TestManagerPickUnknownCandidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
provider := fakeWithAlbum(1, "source", ".flac")
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
req := fourTrackRequest()
|
||||
req.Expected = nil // free text: never auto-picks
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), req); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
err := f.manager.Pick(context.Background(), req.ID, "no-such-candidate")
|
||||
if !errors.Is(err, ErrCandidateGone) {
|
||||
t.Fatalf("error = %v, want ErrCandidateGone", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed grab must leave staging intact for retry and must not put
|
||||
// anything in the library.
|
||||
func TestManagerFailedGrabLeavesLibraryClean(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
provider := fakeWithAlbum(1, "source", ".flac")
|
||||
provider.GrabErr = errors.New("peer went offline") //nolint:err113 // test
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), req); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateFailed)
|
||||
|
||||
entries, err := os.ReadDir(f.root)
|
||||
if err != nil {
|
||||
t.Fatalf("read library root: %v", err)
|
||||
}
|
||||
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("failed grab wrote %d entries into the library", len(entries))
|
||||
}
|
||||
|
||||
staged, err := os.ReadDir(f.staging.Root())
|
||||
if err != nil {
|
||||
t.Fatalf("read staging root: %v", err)
|
||||
}
|
||||
|
||||
if len(staged) == 0 {
|
||||
t.Error("staging released after a failure; nothing left to retry")
|
||||
}
|
||||
}
|
||||
|
||||
// A search-only provider's candidate is fetched by whichever enabled
|
||||
// transport handles its protocol.
|
||||
func TestManagerPairsSearcherWithTransport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
searcher := NewFakeProvider(1, "indexer", Caps{CanSearch: true})
|
||||
|
||||
titles := allTitles()
|
||||
c := candidateFor("torrent-cand", titles, ".flac", 30_000_000)
|
||||
c.Protocol = ProtocolTorrent
|
||||
searcher.Candidates = []Candidate{c}
|
||||
|
||||
transport := NewFakeProvider(2, "torrent-client", Caps{
|
||||
CanTransport: true,
|
||||
Transports: []Protocol{ProtocolTorrent},
|
||||
})
|
||||
|
||||
for i, tt := range titles {
|
||||
transport.Written[trackToken(i+1)+" - "+tt+".flac"] = []byte("audio-data")
|
||||
}
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, searcher)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 50}, transport)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), req); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateComplete)
|
||||
|
||||
if transport.GrabCalls != 1 {
|
||||
t.Errorf("transport grabs = %d, want 1", transport.GrabCalls)
|
||||
}
|
||||
|
||||
if searcher.GrabCalls != 0 {
|
||||
t.Errorf("searcher should not have grabbed, got %d", searcher.GrabCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// A candidate whose protocol nothing handles must fail loudly rather
|
||||
// than being silently dropped.
|
||||
func TestManagerNoTransportForProtocol(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
searcher := NewFakeProvider(1, "indexer", Caps{CanSearch: true})
|
||||
|
||||
c := candidateFor("usenet-cand", allTitles(), ".flac", 30_000_000)
|
||||
c.Protocol = ProtocolUsenet
|
||||
searcher.Candidates = []Candidate{c}
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, searcher)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), req); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateFailed)
|
||||
}
|
||||
|
||||
// waitForRequestState polls until a request reaches the wanted state.
|
||||
// The pipeline runs on its own goroutine, so tests observe it through
|
||||
// the store rather than by reaching into the manager.
|
||||
func waitForRequestState(
|
||||
t *testing.T,
|
||||
store *Store,
|
||||
requestID string,
|
||||
want State,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
|
||||
var last State
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
state, _, err := store.GetRequestState(context.Background(), requestID)
|
||||
if err == nil {
|
||||
last = state
|
||||
if last == want {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
t.Fatalf("request state = %q after 5s, want %q", last, want)
|
||||
}
|
||||
|
||||
// A delegate's files are already in the external manager's library,
|
||||
// tagged by it and at paths it chose. The pipeline must record them in
|
||||
// place rather than moving them out from under a system that is still
|
||||
// managing them.
|
||||
func TestManagerDelegateReconcilesInPlace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.delegatePoll = time.Millisecond
|
||||
|
||||
delegate := NewFakeProvider(1, "lidarr", Caps{
|
||||
CanSearch: true,
|
||||
CanDelegate: true,
|
||||
})
|
||||
|
||||
c := candidateFor("delegate-cand", allTitles(), ".flac", 30_000_000)
|
||||
c.ProviderID = 1
|
||||
delegate.Candidates = []Candidate{c}
|
||||
|
||||
external := []string{
|
||||
"/external/library/Radiohead/OK Computer/01 Airbag.flac",
|
||||
"/external/library/Radiohead/OK Computer/02 Paranoid Android.flac",
|
||||
}
|
||||
|
||||
delegate.DelegateStatuses = []DelegateStatus{
|
||||
{State: StateGrabbing, Progress: 0.5},
|
||||
{State: StateComplete, Progress: 1, ImportedPaths: external},
|
||||
}
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, delegate)
|
||||
|
||||
req := fourTrackRequest()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), req); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForRequestState(t, f.store, req.ID, StateComplete)
|
||||
|
||||
if delegate.DelegateCalls != 1 {
|
||||
t.Errorf("delegate calls = %d, want 1", delegate.DelegateCalls)
|
||||
}
|
||||
|
||||
// Nothing was tagged or written into our library root — the files
|
||||
// belong to the external manager.
|
||||
entries, err := os.ReadDir(f.root)
|
||||
if err != nil {
|
||||
t.Fatalf("read library root: %v", err)
|
||||
}
|
||||
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("delegate import wrote %d entries into our library", len(entries))
|
||||
}
|
||||
|
||||
f.tags.mu.Lock()
|
||||
writes := len(f.tags.writes)
|
||||
f.tags.mu.Unlock()
|
||||
|
||||
if writes != 0 {
|
||||
t.Errorf("tagged %d files, want 0 for a delegated import", writes)
|
||||
}
|
||||
|
||||
// The external paths were recorded against the item.
|
||||
items, err := f.store.ListItemsForRequest(context.Background(), req.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListItemsForRequest: %v", err)
|
||||
}
|
||||
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("got %d items, want 1", len(items))
|
||||
}
|
||||
|
||||
if len(items[0].Imported) != len(external) {
|
||||
t.Errorf(
|
||||
"recorded %v, want the external manager's paths %v",
|
||||
items[0].Imported, external,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"yellowjacket/backend/autotag"
|
||||
)
|
||||
|
||||
// Candidate files arrive as paths, not tags — a Soulseek result is
|
||||
// `@@abc\Music\Pink Floyd - The Wall (1979) [FLAC]\1-01 In The Flesh.flac`
|
||||
// and nothing more. Everything the ranker knows about whether a
|
||||
// candidate is the right album comes from parsing that string, so the
|
||||
// heuristics here carry real weight.
|
||||
|
||||
// audioExtensions maps a lowercase file extension to its format.
|
||||
var audioExtensions = map[string]Format{
|
||||
".flac": FormatFLAC,
|
||||
".mp3": FormatMP3,
|
||||
".ogg": FormatOGG,
|
||||
".oga": FormatOGG,
|
||||
".opus": FormatOpus,
|
||||
".wav": FormatWAV,
|
||||
".m4a": FormatAAC,
|
||||
".aac": FormatAAC,
|
||||
".alac": FormatALAC,
|
||||
".wma": FormatWMA,
|
||||
".ape": FormatUnknown,
|
||||
".wv": FormatUnknown,
|
||||
}
|
||||
|
||||
var (
|
||||
// trackNumPattern matches a leading track number in the common
|
||||
// shapes: "01 - Title", "1. Title", "1-01 Title" (disc-track),
|
||||
// "[01] Title". The disc group is optional.
|
||||
trackNumPattern = regexp.MustCompile(
|
||||
`^\s*\[?(?:(\d{1,2})\s*[-_.]\s*)?(\d{1,3})\]?\s*[-_.)\]]?\s+`,
|
||||
)
|
||||
|
||||
// bareTrackNumPattern matches a number with no separator at all
|
||||
// ("01Title" is rare, but "01 Title" with a single space is not).
|
||||
bareTrackNumPattern = regexp.MustCompile(`^\s*(\d{1,3})\s+`)
|
||||
|
||||
// bitratePattern finds a bitrate hint in a folder or file name:
|
||||
// "[320]", "V0", "320kbps", "(V2)".
|
||||
bitratePattern = regexp.MustCompile(
|
||||
`(?i)\b(\d{2,4})\s*k(?:bps|b/s)?\b|\[(\d{2,4})\]`,
|
||||
)
|
||||
|
||||
// vbrPattern finds LAME VBR preset names, which imply a bitrate
|
||||
// band rather than a number.
|
||||
vbrPattern = regexp.MustCompile(`(?i)\b(V[0-2])\b`)
|
||||
|
||||
// yearPattern finds a 4-digit year in parentheses or brackets.
|
||||
yearPattern = regexp.MustCompile(`[(\[](19|20)\d{2}[)\]]`)
|
||||
|
||||
// junkSuffixPattern strips scene/rip tags from a folder name before
|
||||
// comparing it to an album title.
|
||||
junkSuffixPattern = regexp.MustCompile(
|
||||
`(?i)[\[(]\s*(flac|mp3|web|cd|vinyl|24bit|16bit|lossless|` +
|
||||
`v0|v2|320|256|192|128|kbps|reissue|remaster(ed)?|` +
|
||||
`\d{2,3}\s*k(bps)?)\s*[^\])]*[\])]`,
|
||||
)
|
||||
|
||||
// separatorPattern splits "Artist - Album" style folder names.
|
||||
separatorPattern = regexp.MustCompile(`\s+[-–—]\s+`)
|
||||
)
|
||||
|
||||
// FormatForPath returns the audio format implied by a path's extension,
|
||||
// and whether the path is audio at all. Cue sheets, logs, playlists
|
||||
// and cover images are not.
|
||||
func FormatForPath(p string) (Format, bool) {
|
||||
ext := strings.ToLower(path.Ext(strings.ReplaceAll(p, `\`, "/")))
|
||||
|
||||
f, ok := audioExtensions[ext]
|
||||
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// TrackHint is what a single candidate file's path reveals about the
|
||||
// track it holds. Every field is best-effort and may be zero.
|
||||
type TrackHint struct {
|
||||
Disc int
|
||||
Track int
|
||||
|
||||
// Title is the filename with the extension, track number and any
|
||||
// leading artist credit removed.
|
||||
Title string
|
||||
|
||||
// Folder is the immediate parent directory name, cleaned of scene
|
||||
// tags — the best available proxy for the album title.
|
||||
Folder string
|
||||
}
|
||||
|
||||
// ParsePath extracts what it can from one candidate file path.
|
||||
func ParsePath(p string) TrackHint {
|
||||
// Soulseek paths are Windows-style; normalize before splitting.
|
||||
norm := strings.ReplaceAll(p, `\`, "/")
|
||||
base := path.Base(norm)
|
||||
folder := path.Base(path.Dir(norm))
|
||||
|
||||
name := strings.TrimSuffix(base, path.Ext(base))
|
||||
|
||||
hint := TrackHint{Folder: cleanAlbumName(folder)}
|
||||
|
||||
if m := trackNumPattern.FindStringSubmatch(name); m != nil {
|
||||
if m[1] != "" {
|
||||
hint.Disc, _ = strconv.Atoi(m[1])
|
||||
}
|
||||
|
||||
hint.Track, _ = strconv.Atoi(m[2])
|
||||
name = name[len(m[0]):]
|
||||
} else if m := bareTrackNumPattern.FindStringSubmatch(name); m != nil {
|
||||
hint.Track, _ = strconv.Atoi(m[1])
|
||||
name = name[len(m[0]):]
|
||||
}
|
||||
|
||||
// "Artist - Title" inside the filename: drop the leading credit
|
||||
// when what follows is substantial. Guessing wrong here costs a
|
||||
// little title similarity; not doing it costs a lot, because most
|
||||
// Soulseek folders name the artist in every file.
|
||||
if parts := separatorPattern.Split(name, 2); len(parts) == 2 {
|
||||
if len(strings.TrimSpace(parts[1])) >= 3 {
|
||||
name = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
hint.Title = strings.TrimSpace(name)
|
||||
|
||||
return hint
|
||||
}
|
||||
|
||||
// cleanAlbumName strips year markers and scene tags from a folder name
|
||||
// so it can be compared against a release title.
|
||||
func cleanAlbumName(folder string) string {
|
||||
s := junkSuffixPattern.ReplaceAllString(folder, " ")
|
||||
s = yearPattern.ReplaceAllString(s, " ")
|
||||
|
||||
// A folder is often "Artist - Album"; keep the right-hand side when
|
||||
// there is one, since the album is what we compare against.
|
||||
if parts := separatorPattern.Split(s, 2); len(parts) == 2 {
|
||||
if len(strings.TrimSpace(parts[1])) >= 2 {
|
||||
s = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
|
||||
// BitrateForPath infers a bitrate in kbps from path text. Returns 0
|
||||
// when nothing is stated. VBR presets map to their nominal average.
|
||||
func BitrateForPath(p string) int {
|
||||
if m := vbrPattern.FindStringSubmatch(p); m != nil {
|
||||
switch strings.ToUpper(m[1]) {
|
||||
case "V0":
|
||||
return 245
|
||||
case "V1":
|
||||
return 225
|
||||
case "V2":
|
||||
return 190
|
||||
}
|
||||
}
|
||||
|
||||
if m := bitratePattern.FindStringSubmatch(p); m != nil {
|
||||
raw := m[1]
|
||||
if raw == "" {
|
||||
raw = m[2]
|
||||
}
|
||||
|
||||
if n, err := strconv.Atoi(raw); err == nil && n >= 32 && n <= 3000 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// AnnotateFiles fills in Format, IsAudio and Bitrate for a candidate's
|
||||
// files. Providers call this so each adapter does not re-derive the
|
||||
// same things from the same paths.
|
||||
func AnnotateFiles(files []CandidateFile) []CandidateFile {
|
||||
out := make([]CandidateFile, len(files))
|
||||
|
||||
for i, f := range files {
|
||||
format, isAudio := FormatForPath(f.Path)
|
||||
|
||||
f.IsAudio = isAudio
|
||||
if f.Format == FormatUnknown {
|
||||
f.Format = format
|
||||
}
|
||||
|
||||
if f.Bitrate == 0 {
|
||||
f.Bitrate = BitrateForPath(f.Path)
|
||||
}
|
||||
|
||||
out[i] = f
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// matchFiles aligns a candidate's audio files to the expected tracklist
|
||||
// and returns the per-file assignment plus the mean title similarity of
|
||||
// the aligned pairs.
|
||||
//
|
||||
// Alignment is greedy by score rather than optimal: candidate folders
|
||||
// are small (a few dozen files at most) and the common cases — correct
|
||||
// track numbers, or clean "NN Title" names — are unambiguous, so the
|
||||
// extra machinery of Hungarian assignment buys nothing here.
|
||||
func matchFiles(
|
||||
files []CandidateFile,
|
||||
expected []ExpectedTrack,
|
||||
) ([]CandidateFile, float64) {
|
||||
annotated := make([]CandidateFile, len(files))
|
||||
copy(annotated, files)
|
||||
|
||||
if len(expected) == 0 {
|
||||
return annotated, 0
|
||||
}
|
||||
|
||||
hints := make([]TrackHint, len(annotated))
|
||||
for i, f := range annotated {
|
||||
hints[i] = ParsePath(f.Path)
|
||||
}
|
||||
|
||||
takenExpected := make(map[int]bool, len(expected))
|
||||
|
||||
var (
|
||||
total float64
|
||||
matched int
|
||||
)
|
||||
|
||||
// Pass 1: trust explicit track numbers when they are unique and in
|
||||
// range. A folder that numbers its files correctly is the strong
|
||||
// case, and title comparison only adds noise there.
|
||||
for i := range annotated {
|
||||
if !annotated[i].IsAudio || hints[i].Track == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
idx := indexForPosition(expected, hints[i].Disc, hints[i].Track)
|
||||
if idx < 0 || takenExpected[idx] {
|
||||
continue
|
||||
}
|
||||
|
||||
takenExpected[idx] = true
|
||||
annotated[i].MatchedTo = expected[idx].Position
|
||||
|
||||
total += autotag.TitleSimilarity(hints[i].Title, expected[idx].Title)
|
||||
matched++
|
||||
}
|
||||
|
||||
// Pass 2: title similarity for whatever is left.
|
||||
for i := range annotated {
|
||||
if !annotated[i].IsAudio || annotated[i].MatchedTo != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
bestIdx, bestSim := -1, 0.0
|
||||
|
||||
for j := range expected {
|
||||
if takenExpected[j] {
|
||||
continue
|
||||
}
|
||||
|
||||
sim := autotag.TitleSimilarity(hints[i].Title, expected[j].Title)
|
||||
if sim > bestSim {
|
||||
bestIdx, bestSim = j, sim
|
||||
}
|
||||
}
|
||||
|
||||
// Below this the "match" is two unrelated strings sharing a few
|
||||
// characters, and counting it drags the mean toward noise.
|
||||
const minTitleSim = 0.55
|
||||
|
||||
if bestIdx < 0 || bestSim < minTitleSim {
|
||||
continue
|
||||
}
|
||||
|
||||
takenExpected[bestIdx] = true
|
||||
annotated[i].MatchedTo = expected[bestIdx].Position
|
||||
|
||||
total += bestSim
|
||||
matched++
|
||||
}
|
||||
|
||||
if matched == 0 {
|
||||
return annotated, 0
|
||||
}
|
||||
|
||||
return annotated, total / float64(matched)
|
||||
}
|
||||
|
||||
// indexForPosition finds the expected track at a disc/track position.
|
||||
// A zero disc hint matches on track number alone, which is right for
|
||||
// single-disc releases and the best guess for multi-disc folders that
|
||||
// do not encode the disc.
|
||||
func indexForPosition(expected []ExpectedTrack, disc, track int) int {
|
||||
for i, e := range expected {
|
||||
if e.Position != track {
|
||||
continue
|
||||
}
|
||||
|
||||
if disc != 0 && e.DiscNumber != 0 && e.DiscNumber != disc {
|
||||
continue
|
||||
}
|
||||
|
||||
return i
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package download
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParsePath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantDisc int
|
||||
wantTrack int
|
||||
wantTitle string
|
||||
}{
|
||||
{
|
||||
name: "soulseek windows path with disc and track",
|
||||
path: `@@abc\Music\Pink Floyd - The Wall (1979) [FLAC]\1-05 Another Brick In The Wall.flac`,
|
||||
wantDisc: 1,
|
||||
wantTrack: 5,
|
||||
wantTitle: "Another Brick In The Wall",
|
||||
},
|
||||
{
|
||||
name: "dash separated track number",
|
||||
path: "Radiohead - OK Computer/03 - Subterranean Homesick Alien.mp3",
|
||||
wantTrack: 3,
|
||||
wantTitle: "Subterranean Homesick Alien",
|
||||
},
|
||||
{
|
||||
name: "dotted track number",
|
||||
path: "Album/7. Karma Police.flac",
|
||||
wantTrack: 7,
|
||||
wantTitle: "Karma Police",
|
||||
},
|
||||
{
|
||||
name: "bracketed track number",
|
||||
path: "Album/[02] Paranoid Android.mp3",
|
||||
wantTrack: 2,
|
||||
wantTitle: "Paranoid Android",
|
||||
},
|
||||
{
|
||||
name: "bare number and space",
|
||||
path: "Album/11 Lucky.ogg",
|
||||
wantTrack: 11,
|
||||
wantTitle: "Lucky",
|
||||
},
|
||||
{
|
||||
name: "artist credit inside filename is dropped",
|
||||
path: "VA - Comp/04 - Aphex Twin - Xtal.flac",
|
||||
wantTrack: 4,
|
||||
wantTitle: "Xtal",
|
||||
},
|
||||
{
|
||||
name: "no track number",
|
||||
path: "Album/Introduction.flac",
|
||||
wantTrack: 0,
|
||||
wantTitle: "Introduction",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := ParsePath(tt.path)
|
||||
|
||||
if got.Disc != tt.wantDisc {
|
||||
t.Errorf("Disc = %d, want %d", got.Disc, tt.wantDisc)
|
||||
}
|
||||
|
||||
if got.Track != tt.wantTrack {
|
||||
t.Errorf("Track = %d, want %d", got.Track, tt.wantTrack)
|
||||
}
|
||||
|
||||
if got.Title != tt.wantTitle {
|
||||
t.Errorf("Title = %q, want %q", got.Title, tt.wantTitle)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanAlbumName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"Pink Floyd - The Wall (1979) [FLAC]", "The Wall"},
|
||||
{"Radiohead - OK Computer [V0]", "OK Computer"},
|
||||
{"In Rainbows", "In Rainbows"},
|
||||
{"Artist - Album [320kbps]", "Album"},
|
||||
{"Kid A (2000)", "Kid A"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := cleanAlbumName(tt.in); got != tt.want {
|
||||
t.Errorf("cleanAlbumName(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitrateForPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want int
|
||||
}{
|
||||
{"Album [320]/01 Track.mp3", 320},
|
||||
{"Album [V0]/01 Track.mp3", 245},
|
||||
{"Album (V2)/01 Track.mp3", 190},
|
||||
{"Album 192kbps/01 Track.mp3", 192},
|
||||
{"Album/01 Track.flac", 0},
|
||||
{"Album [9999]/01 Track.mp3", 0}, // out of plausible range
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := BitrateForPath(tt.in); got != tt.want {
|
||||
t.Errorf("BitrateForPath(%q) = %d, want %d", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatForPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want Format
|
||||
wantAudio bool
|
||||
}{
|
||||
{"a/b.flac", FormatFLAC, true},
|
||||
{"a/b.MP3", FormatMP3, true},
|
||||
{`a\b.ogg`, FormatOGG, true},
|
||||
{"a/cover.jpg", FormatUnknown, false},
|
||||
{"a/rip.log", FormatUnknown, false},
|
||||
{"a/playlist.m3u", FormatUnknown, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, isAudio := FormatForPath(tt.path)
|
||||
|
||||
if isAudio != tt.wantAudio {
|
||||
t.Errorf("isAudio = %v, want %v", isAudio, tt.wantAudio)
|
||||
}
|
||||
|
||||
if isAudio && got != tt.want {
|
||||
t.Errorf("format = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchFilesByTrackNumber(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expected := []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
{Position: 3, Title: "Subterranean Homesick Alien"},
|
||||
}
|
||||
|
||||
files := []CandidateFile{
|
||||
{Path: "OK Computer/01 - Airbag.flac", IsAudio: true},
|
||||
{Path: "OK Computer/02 - Paranoid Android.flac", IsAudio: true},
|
||||
{Path: "OK Computer/03 - Subterranean Homesick Alien.flac", IsAudio: true},
|
||||
}
|
||||
|
||||
matched, sim := matchFiles(files, expected)
|
||||
|
||||
for i, m := range matched {
|
||||
if m.MatchedTo != i+1 {
|
||||
t.Errorf("file %d matched to %d, want %d", i, m.MatchedTo, i+1)
|
||||
}
|
||||
}
|
||||
|
||||
if sim < 0.99 {
|
||||
t.Errorf("similarity = %f, want ~1.0", sim)
|
||||
}
|
||||
}
|
||||
|
||||
// Track numbers that lie are the common Soulseek failure: a folder
|
||||
// numbered 1..N whose contents are a different album entirely. Title
|
||||
// matching has to be what catches it.
|
||||
func TestMatchFilesFallsBackToTitles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expected := []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
}
|
||||
|
||||
files := []CandidateFile{
|
||||
{Path: "Album/Paranoid Android.flac", IsAudio: true},
|
||||
{Path: "Album/Airbag.flac", IsAudio: true},
|
||||
}
|
||||
|
||||
matched, sim := matchFiles(files, expected)
|
||||
|
||||
if matched[0].MatchedTo != 2 {
|
||||
t.Errorf("first file matched to %d, want 2", matched[0].MatchedTo)
|
||||
}
|
||||
|
||||
if matched[1].MatchedTo != 1 {
|
||||
t.Errorf("second file matched to %d, want 1", matched[1].MatchedTo)
|
||||
}
|
||||
|
||||
if sim < 0.9 {
|
||||
t.Errorf("similarity = %f, want high", sim)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchFilesUnrelatedScoresLow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expected := []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
}
|
||||
|
||||
files := []CandidateFile{
|
||||
{Path: "Other/Enter Sandman.flac", IsAudio: true},
|
||||
{Path: "Other/Master Of Puppets.flac", IsAudio: true},
|
||||
}
|
||||
|
||||
_, sim := matchFiles(files, expected)
|
||||
|
||||
if sim > 0.5 {
|
||||
t.Errorf("similarity = %f, want low for unrelated tracks", sim)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Provider errors.
|
||||
var (
|
||||
// ErrUnknownKind is returned when a stored provider row names a kind
|
||||
// no constructor is registered for — an old row after a provider
|
||||
// was removed, or a config file from a newer build.
|
||||
ErrUnknownKind = errors.New("unknown provider kind")
|
||||
|
||||
// ErrNotConfigured means the provider exists but is missing
|
||||
// required settings (host, API key) and cannot be used yet.
|
||||
ErrNotConfigured = errors.New("provider is not configured")
|
||||
|
||||
// ErrUnsupported is returned when a caller asks a provider for a
|
||||
// role it does not fill.
|
||||
ErrUnsupported = errors.New("provider does not support this operation")
|
||||
|
||||
// ErrNoTransport means a search-only provider produced a candidate
|
||||
// whose protocol no enabled transport can fetch.
|
||||
ErrNoTransport = errors.New("no enabled transport handles this protocol")
|
||||
)
|
||||
|
||||
// Provider is the common surface every adapter implements. The three
|
||||
// role interfaces below are optional and discovered by type assertion,
|
||||
// gated on what Caps declares.
|
||||
type Provider interface {
|
||||
// Info returns the provider's identity and declared capabilities.
|
||||
Info() ProviderInfo
|
||||
|
||||
// Check verifies the provider is reachable and configured
|
||||
// correctly. It backs the "test connection" button, and the
|
||||
// pipeline calls it before first use in a session.
|
||||
Check(ctx context.Context) error
|
||||
|
||||
// Close releases any long-lived resources (sessions, cookies).
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Searcher turns a request into candidates. Implementations must
|
||||
// respect ctx deadlines: the pipeline searches providers concurrently
|
||||
// with a per-provider timeout and takes whatever came back in time.
|
||||
type Searcher interface {
|
||||
Search(ctx context.Context, req Request) ([]Candidate, error)
|
||||
}
|
||||
|
||||
// Transporter moves a candidate's bytes into dst, which the pipeline
|
||||
// has already created and which the transport owns for the duration.
|
||||
//
|
||||
// Implementations report progress through onProgress (best-effort, may
|
||||
// be nil) and must return promptly when ctx is cancelled, leaving
|
||||
// partial files in place — the pipeline sweeps them.
|
||||
type Transporter interface {
|
||||
Grab(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error)
|
||||
}
|
||||
|
||||
// Delegator hands the whole request to an external manager. Unlike a
|
||||
// Transporter we do not own the transfer, so the pipeline polls until
|
||||
// the manager reports terminal state.
|
||||
type Delegator interface {
|
||||
// Delegate submits the request and returns the manager's own ID.
|
||||
Delegate(ctx context.Context, req Request) (string, error)
|
||||
|
||||
// Poll reports on a previously delegated request.
|
||||
Poll(ctx context.Context, externalID string) (DelegateStatus, error)
|
||||
|
||||
// Withdraw asks the manager to drop the request. Best-effort.
|
||||
Withdraw(ctx context.Context, externalID string) error
|
||||
}
|
||||
|
||||
// Lister is a provider that keeps a persistent wanted list of its own —
|
||||
// Lidarr monitoring an artist, say. It is the fourth role, and it
|
||||
// exists because for those systems "I want this" is a durable statement
|
||||
// they already model, and mirroring it there means the user's intent
|
||||
// survives in the place they will look for it.
|
||||
//
|
||||
// Sync through this interface is one-directional in the loop: this app
|
||||
// pushes, the external system receives. Pulling happens only when the
|
||||
// user explicitly imports.
|
||||
type Lister interface {
|
||||
// PushWant records a want in the provider's own list and returns
|
||||
// the provider's identifier for it. Implementations must be
|
||||
// idempotent: pushing a want the provider already has returns the
|
||||
// existing identifier rather than duplicating it.
|
||||
PushWant(ctx context.Context, w Want) (string, error)
|
||||
|
||||
// RemoveWant drops a previously pushed want. Best-effort.
|
||||
RemoveWant(ctx context.Context, externalID string) error
|
||||
|
||||
// ListWants reads the provider's list back, for the deliberate
|
||||
// import path. LibraryID is filled in by the caller.
|
||||
ListWants(ctx context.Context) ([]Want, error)
|
||||
}
|
||||
|
||||
// ProviderInfo is a provider's identity as the frontend sees it.
|
||||
type ProviderInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Kind Kind `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
// Priority breaks ties between providers that found equally good
|
||||
// candidates. Higher wins; default 50.
|
||||
Priority int `json:"priority"`
|
||||
|
||||
Caps Caps `json:"caps"`
|
||||
}
|
||||
|
||||
// Config is a provider's stored settings. Secret values are not held
|
||||
// here — they live in the secrets store keyed by provider ID, so a
|
||||
// config blob can be logged or shown in the UI without redaction.
|
||||
type Config struct {
|
||||
ID int64 `json:"id"`
|
||||
Kind Kind `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Priority int `json:"priority"`
|
||||
Settings map[string]string `json:"settings"`
|
||||
}
|
||||
|
||||
// Setting returns a config value, or fallback when unset.
|
||||
func (c Config) Setting(key, fallback string) string {
|
||||
if v, ok := c.Settings[key]; ok && v != "" {
|
||||
return v
|
||||
}
|
||||
|
||||
return fallback
|
||||
}
|
||||
|
||||
// Constructor builds a provider from its stored config. The secret
|
||||
// lookup is passed in rather than the secret itself so a provider can
|
||||
// fetch several (username and password, say) and so nothing forces the
|
||||
// secret into a struct field that might get logged.
|
||||
type Constructor func(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error)
|
||||
|
||||
// SecretLookup retrieves a named secret for a provider.
|
||||
type SecretLookup func(name string) (string, error)
|
||||
|
||||
// registry maps provider kinds to their constructors. Adapters
|
||||
// register themselves in an init function, so adding a provider does
|
||||
// not require editing this file.
|
||||
var (
|
||||
registryMu sync.RWMutex
|
||||
constructors = map[Kind]Constructor{}
|
||||
descriptors = map[Kind]Descriptor{}
|
||||
)
|
||||
|
||||
// Descriptor is the static, instance-independent description of a
|
||||
// provider kind: what it is called, what it can do, and which settings
|
||||
// it needs. The settings page renders its form from this, so a new
|
||||
// provider gets a config UI without any frontend work.
|
||||
type Descriptor struct {
|
||||
Kind Kind `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
|
||||
// Summary is one line explaining what connecting this gets you.
|
||||
Summary string `json:"summary"`
|
||||
|
||||
// Caps are the kind's inherent capabilities, before configuration.
|
||||
Caps Caps `json:"caps"`
|
||||
|
||||
// Fields are the settings the user must supply.
|
||||
Fields []Field `json:"fields"`
|
||||
|
||||
// RequiresExternal names the software the user must run themselves
|
||||
// (a slskd daemon, a Lidarr instance), or is empty for providers
|
||||
// that need nothing but a binary on PATH.
|
||||
RequiresExternal string `json:"requiresExternal,omitempty"`
|
||||
}
|
||||
|
||||
// Field describes one provider setting for the settings form.
|
||||
type Field struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Placeholder string `json:"placeholder,omitempty"`
|
||||
Help string `json:"help,omitempty"`
|
||||
|
||||
// Secret marks a value stored in the secrets store rather than the
|
||||
// provider config row, and rendered as a password input.
|
||||
Secret bool `json:"secret"`
|
||||
|
||||
Required bool `json:"required"`
|
||||
Default string `json:"default,omitempty"`
|
||||
}
|
||||
|
||||
// Register makes a provider kind available. Called from adapter init
|
||||
// functions; panics on a duplicate kind because that is a build-time
|
||||
// programming error, not a runtime condition.
|
||||
func Register(d Descriptor, c Constructor) {
|
||||
registryMu.Lock()
|
||||
defer registryMu.Unlock()
|
||||
|
||||
if _, exists := constructors[d.Kind]; exists {
|
||||
panic("download: duplicate provider kind " + string(d.Kind))
|
||||
}
|
||||
|
||||
// Every provider that moves bytes gets a transfer limit it can be
|
||||
// tuned with, appended here rather than repeated in each adapter's
|
||||
// descriptor: the setting means the same thing everywhere, only the
|
||||
// sensible default differs.
|
||||
if d.Caps.CanTransport {
|
||||
d.Fields = append(d.Fields, concurrencyField(d.Kind))
|
||||
}
|
||||
|
||||
constructors[d.Kind] = c
|
||||
descriptors[d.Kind] = d
|
||||
}
|
||||
|
||||
// concurrencyField describes the per-provider transfer limit, with help
|
||||
// text explaining why the default is what it is — a user who raises
|
||||
// slskd from 1 to 8 and gets themselves queued behind every other
|
||||
// Soulseek user deserves to have been warned.
|
||||
func concurrencyField(k Kind) Field {
|
||||
help := "Maximum simultaneous transfers from this client."
|
||||
|
||||
if k == KindSlskd {
|
||||
help = "Maximum simultaneous transfers. Soulseek peers serve " +
|
||||
"one file at a time and queue or ban clients that ask for " +
|
||||
"more, so 1 is both the polite setting and usually the " +
|
||||
"fastest."
|
||||
}
|
||||
|
||||
return Field{
|
||||
Key: concurrencyKey,
|
||||
Label: "Simultaneous transfers",
|
||||
Help: help,
|
||||
Default: strconv.Itoa(kindConcurrency[k]),
|
||||
}
|
||||
}
|
||||
|
||||
// New builds a provider instance from stored config.
|
||||
func New(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
registryMu.RLock()
|
||||
|
||||
ctor, ok := constructors[cfg.Kind]
|
||||
|
||||
registryMu.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownKind, cfg.Kind)
|
||||
}
|
||||
|
||||
p, err := ctor(cfg, secrets, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build %s provider: %w", cfg.Kind, err)
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Descriptors returns every registered provider kind, name-ordered, for
|
||||
// the "add a download client" picker.
|
||||
func Descriptors() []Descriptor {
|
||||
registryMu.RLock()
|
||||
defer registryMu.RUnlock()
|
||||
|
||||
out := make([]Descriptor, 0, len(descriptors))
|
||||
|
||||
for _, d := range descriptors {
|
||||
if d.Kind == KindFake {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, d)
|
||||
}
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].Name < out[j].Name
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// DescriptorFor returns the descriptor for a kind.
|
||||
func DescriptorFor(k Kind) (Descriptor, bool) {
|
||||
registryMu.RLock()
|
||||
defer registryMu.RUnlock()
|
||||
|
||||
d, ok := descriptors[k]
|
||||
|
||||
return d, ok
|
||||
}
|
||||
|
||||
// asSearcher returns the provider's Searcher role, gated on Caps so a
|
||||
// type that implements the method but declares it unsupported (because
|
||||
// it is misconfigured) is not used.
|
||||
func asSearcher(p Provider) (Searcher, bool) {
|
||||
if !p.Info().Caps.CanSearch {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
s, ok := p.(Searcher)
|
||||
|
||||
return s, ok
|
||||
}
|
||||
|
||||
// asTransporter returns the provider's Transporter role.
|
||||
func asTransporter(p Provider) (Transporter, bool) {
|
||||
if !p.Info().Caps.CanTransport {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
t, ok := p.(Transporter)
|
||||
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// asDelegator returns the provider's Delegator role.
|
||||
func asDelegator(p Provider) (Delegator, bool) {
|
||||
if !p.Info().Caps.CanDelegate {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
d, ok := p.(Delegator)
|
||||
|
||||
return d, ok
|
||||
}
|
||||
|
||||
// asLister returns the provider's Lister role.
|
||||
func asLister(p Provider) (Lister, bool) {
|
||||
if !p.Info().Caps.CanList {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
l, ok := p.(Lister)
|
||||
|
||||
return l, ok
|
||||
}
|
||||
@@ -0,0 +1,548 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Lidarr is the delegate shape, and it is genuinely different from the
|
||||
// other two: we do not search, we do not move bytes, and we do not own
|
||||
// the import. We hand Lidarr an album and ask periodically whether it
|
||||
// is done.
|
||||
//
|
||||
// The consequence that shapes this adapter: when Lidarr finishes, the
|
||||
// files are already in Lidarr's library, tagged by Lidarr, at paths
|
||||
// Lidarr chose. Re-importing them would mean moving files out from
|
||||
// under a system that is actively managing them. So a completed
|
||||
// delegate returns the paths Lidarr reports and the pipeline skips
|
||||
// tagging and moving — it reconciles rather than imports.
|
||||
|
||||
// Lidarr provider errors.
|
||||
var (
|
||||
// ErrLidarrUnreachable means the instance did not answer.
|
||||
ErrLidarrUnreachable = errors.New("lidarr is unreachable")
|
||||
|
||||
// ErrLidarrAuth means the API key was rejected.
|
||||
ErrLidarrAuth = errors.New("lidarr rejected the API key")
|
||||
|
||||
// ErrLidarrNoMatch means Lidarr could not find the release.
|
||||
ErrLidarrNoMatch = errors.New("lidarr could not find this release")
|
||||
|
||||
// ErrLidarrNoRootFolder means no root folder is configured, so
|
||||
// Lidarr has nowhere to put anything it finds.
|
||||
ErrLidarrNoRootFolder = errors.New("lidarr has no root folder configured")
|
||||
)
|
||||
|
||||
// lidarrHTTPTimeout bounds one API call.
|
||||
const lidarrHTTPTimeout = 30 * time.Second
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindLidarr,
|
||||
Name: "Lidarr",
|
||||
Summary: "Hand album requests to an existing Lidarr instance " +
|
||||
"and let it do the searching and importing.",
|
||||
RequiresExternal: "Lidarr",
|
||||
Caps: Caps{
|
||||
CanDelegate: true,
|
||||
CanCancel: true,
|
||||
CanList: true,
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "url",
|
||||
Label: "Lidarr URL",
|
||||
Placeholder: "http://localhost:8686",
|
||||
Required: true,
|
||||
Default: "http://localhost:8686",
|
||||
},
|
||||
{
|
||||
Key: "apiKey",
|
||||
Label: "API key",
|
||||
Secret: true,
|
||||
Required: true,
|
||||
Help: "Lidarr → Settings → General → API Key.",
|
||||
},
|
||||
{
|
||||
Key: "qualityProfileId",
|
||||
Label: "Quality profile ID",
|
||||
Help: "Numeric ID of the Lidarr quality profile to use. " +
|
||||
"Leave blank to use the first one.",
|
||||
},
|
||||
{
|
||||
Key: "metadataProfileId",
|
||||
Label: "Metadata profile ID",
|
||||
Help: "Leave blank to use the first one.",
|
||||
},
|
||||
{
|
||||
Key: "rootFolderPath",
|
||||
Label: "Root folder",
|
||||
Help: "Leave blank to use Lidarr's first configured " +
|
||||
"root folder.",
|
||||
},
|
||||
},
|
||||
},
|
||||
newLidarr,
|
||||
)
|
||||
}
|
||||
|
||||
// lidarr is the Lidarr delegate provider.
|
||||
type lidarr struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
client *apiClient
|
||||
|
||||
qualityProfileID int
|
||||
metadataProfileID int
|
||||
rootFolderPath string
|
||||
}
|
||||
|
||||
// newLidarr builds the provider from config.
|
||||
func newLidarr(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
base := strings.TrimRight(cfg.Setting("url", ""), "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("%w: Lidarr URL is required", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey := ""
|
||||
|
||||
if secrets != nil {
|
||||
key, err := secrets("apiKey")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey = key
|
||||
}
|
||||
|
||||
quality, _ := strconv.Atoi(cfg.Setting("qualityProfileId", "0"))
|
||||
metadata, _ := strconv.Atoi(cfg.Setting("metadataProfileId", "0"))
|
||||
|
||||
return &lidarr{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindLidarr,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{
|
||||
CanDelegate: true,
|
||||
CanCancel: true,
|
||||
CanList: true,
|
||||
},
|
||||
},
|
||||
logger: logger.With("provider", "lidarr"),
|
||||
client: newAPIClient(
|
||||
base, "X-Api-Key", apiKey, lidarrHTTPTimeout,
|
||||
ErrLidarrUnreachable, ErrLidarrAuth,
|
||||
),
|
||||
qualityProfileID: quality,
|
||||
metadataProfileID: metadata,
|
||||
rootFolderPath: cfg.Setting("rootFolderPath", ""),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (l *lidarr) Info() ProviderInfo {
|
||||
return l.info
|
||||
}
|
||||
|
||||
// Close is a no-op.
|
||||
func (l *lidarr) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check verifies the instance answers and has somewhere to put music.
|
||||
func (l *lidarr) Check(ctx context.Context) error {
|
||||
var status struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
if err := l.client.get(ctx, "/api/v1/system/status", &status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
folders, err := l.rootFolders(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(folders) == 0 {
|
||||
return ErrLidarrNoRootFolder
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// API types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// lidarrAlbum is the subset of Lidarr's album resource used here.
|
||||
type lidarrAlbum struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
ForeignAlbum string `json:"foreignAlbumId"`
|
||||
Monitored bool `json:"monitored"`
|
||||
ArtistID int `json:"artistId"`
|
||||
|
||||
Artist lidarrArtist `json:"artist"`
|
||||
Statistics lidarrAlbumStats `json:"statistics"`
|
||||
}
|
||||
|
||||
// lidarrArtist is the artist an album belongs to. Lidarr models albums
|
||||
// as children of artists, so an album it does not yet know about cannot
|
||||
// be monitored until its artist exists.
|
||||
type lidarrArtist struct {
|
||||
ID int `json:"id"`
|
||||
ArtistName string `json:"artistName"`
|
||||
ForeignArtistID string `json:"foreignArtistId"`
|
||||
}
|
||||
|
||||
// lidarrAlbumStats is how Lidarr reports import progress.
|
||||
type lidarrAlbumStats struct {
|
||||
TrackFileCount int `json:"trackFileCount"`
|
||||
TrackCount int `json:"trackCount"`
|
||||
PercentOfTracks float64 `json:"percentOfTracks"`
|
||||
}
|
||||
|
||||
// lidarrRootFolder is a configured library root.
|
||||
type lidarrRootFolder struct {
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
// lidarrTrackFile is one imported file.
|
||||
type lidarrTrackFile struct {
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
AlbumID int `json:"albumId"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Delegate
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Delegate adds the album to Lidarr, monitors it and triggers a search.
|
||||
// The external ID returned is Lidarr's album ID, which is what Poll
|
||||
// needs and what survives a restart.
|
||||
func (l *lidarr) Delegate(ctx context.Context, req Request) (string, error) {
|
||||
album, err := l.findAlbum(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// An album Lidarr already knows about only needs monitoring turned
|
||||
// on; one it does not needs its artist added first, because Lidarr
|
||||
// models albums as children of artists.
|
||||
if album.ID == 0 {
|
||||
added, err := l.addArtistForAlbum(ctx, album)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
album = added
|
||||
}
|
||||
|
||||
if !album.Monitored {
|
||||
if err := l.monitorAlbum(ctx, album.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if err := l.command(ctx, map[string]any{
|
||||
"name": "AlbumSearch",
|
||||
"albumIds": []int{album.ID},
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return strconv.Itoa(album.ID), nil
|
||||
}
|
||||
|
||||
// findAlbum looks for the requested album, preferring the MusicBrainz
|
||||
// release-group ID because that is unambiguous where a title search is
|
||||
// not.
|
||||
func (l *lidarr) findAlbum(
|
||||
ctx context.Context,
|
||||
req Request,
|
||||
) (lidarrAlbum, error) {
|
||||
term := req.SearchText()
|
||||
|
||||
if req.ReleaseGroupMBID != "" {
|
||||
term = "lidarr:" + req.ReleaseGroupMBID
|
||||
}
|
||||
|
||||
var results []struct {
|
||||
Album lidarrAlbum `json:"album"`
|
||||
}
|
||||
|
||||
endpoint := "/api/v1/search?term=" + url.QueryEscape(term)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &results); err != nil {
|
||||
return lidarrAlbum{}, err
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
if r.Album.Title != "" {
|
||||
return r.Album, nil
|
||||
}
|
||||
}
|
||||
|
||||
return lidarrAlbum{}, fmt.Errorf("%w: %s", ErrLidarrNoMatch, req.SearchText())
|
||||
}
|
||||
|
||||
// addArtistForAlbum adds the album's artist so the album becomes a real
|
||||
// record Lidarr can monitor.
|
||||
func (l *lidarr) addArtistForAlbum(
|
||||
ctx context.Context,
|
||||
album lidarrAlbum,
|
||||
) (lidarrAlbum, error) {
|
||||
root := l.rootFolderPath
|
||||
|
||||
if root == "" {
|
||||
folders, err := l.rootFolders(ctx)
|
||||
if err != nil {
|
||||
return lidarrAlbum{}, err
|
||||
}
|
||||
|
||||
if len(folders) == 0 {
|
||||
return lidarrAlbum{}, ErrLidarrNoRootFolder
|
||||
}
|
||||
|
||||
root = folders[0].Path
|
||||
}
|
||||
|
||||
quality, metadata, err := l.profiles(ctx)
|
||||
if err != nil {
|
||||
return lidarrAlbum{}, err
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"foreignArtistId": album.Artist.ForeignArtistID,
|
||||
"artistName": album.Artist.ArtistName,
|
||||
"qualityProfileId": quality,
|
||||
"metadataProfileId": metadata,
|
||||
"rootFolderPath": root,
|
||||
"monitored": true,
|
||||
"addOptions": map[string]any{
|
||||
// Monitor nothing by default and turn on just the requested
|
||||
// album below. Adding an artist with everything monitored
|
||||
// would kick off downloads of their entire discography,
|
||||
// which is emphatically not what the user asked for.
|
||||
"monitor": "none",
|
||||
"searchForMissingAlbums": false,
|
||||
},
|
||||
}
|
||||
|
||||
var created struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
if err := l.client.post(ctx, "/api/v1/artist", body, &created); err != nil {
|
||||
return lidarrAlbum{}, err
|
||||
}
|
||||
|
||||
// Re-resolve the album now that its artist exists.
|
||||
var albums []lidarrAlbum
|
||||
|
||||
endpoint := "/api/v1/album?artistId=" + strconv.Itoa(created.ID)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &albums); err != nil {
|
||||
return lidarrAlbum{}, err
|
||||
}
|
||||
|
||||
for _, a := range albums {
|
||||
if a.ForeignAlbum == album.ForeignAlbum {
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
return lidarrAlbum{}, fmt.Errorf(
|
||||
"%w: album not present after adding artist", ErrLidarrNoMatch,
|
||||
)
|
||||
}
|
||||
|
||||
// monitorAlbum turns on monitoring for one album.
|
||||
func (l *lidarr) monitorAlbum(ctx context.Context, albumID int) error {
|
||||
body := map[string]any{
|
||||
"albumIds": []int{albumID},
|
||||
"monitored": true,
|
||||
}
|
||||
|
||||
return l.client.put(ctx, "/api/v1/album/monitor", body, nil)
|
||||
}
|
||||
|
||||
// Poll reports whether Lidarr has finished with the album.
|
||||
//
|
||||
// Completion is judged by imported track files rather than by queue
|
||||
// state: the queue empties when a download finishes, which is before
|
||||
// the import happens, and reporting success then would have the
|
||||
// pipeline reconcile files that are not there yet.
|
||||
func (l *lidarr) Poll(
|
||||
ctx context.Context,
|
||||
externalID string,
|
||||
) (DelegateStatus, error) {
|
||||
albumID, err := strconv.Atoi(externalID)
|
||||
if err != nil {
|
||||
return DelegateStatus{}, fmt.Errorf(
|
||||
"%w: bad album id %q", ErrLidarrNoMatch, externalID,
|
||||
)
|
||||
}
|
||||
|
||||
var album lidarrAlbum
|
||||
|
||||
endpoint := "/api/v1/album/" + strconv.Itoa(albumID)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &album); err != nil {
|
||||
return DelegateStatus{}, err
|
||||
}
|
||||
|
||||
total := album.Statistics.TrackCount
|
||||
got := album.Statistics.TrackFileCount
|
||||
|
||||
progress := -1.0
|
||||
if total > 0 {
|
||||
progress = float64(got) / float64(total)
|
||||
}
|
||||
|
||||
if total > 0 && got >= total {
|
||||
paths, err := l.trackFilePaths(ctx, albumID)
|
||||
if err != nil {
|
||||
return DelegateStatus{}, err
|
||||
}
|
||||
|
||||
return DelegateStatus{
|
||||
State: StateComplete,
|
||||
Progress: 1,
|
||||
ImportedPaths: paths,
|
||||
Message: fmt.Sprintf(
|
||||
"Lidarr imported %d of %d tracks", got, total,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return DelegateStatus{
|
||||
State: StateGrabbing,
|
||||
Progress: progress,
|
||||
Message: fmt.Sprintf(
|
||||
"Lidarr has %d of %d tracks", got, total,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// trackFilePaths returns the on-disk paths Lidarr imported.
|
||||
func (l *lidarr) trackFilePaths(
|
||||
ctx context.Context,
|
||||
albumID int,
|
||||
) ([]string, error) {
|
||||
var files []lidarrTrackFile
|
||||
|
||||
endpoint := "/api/v1/trackfile?albumId=" + strconv.Itoa(albumID)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &files); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
if f.Path != "" {
|
||||
out = append(out, f.Path)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Withdraw stops monitoring the album so Lidarr gives up on it. The
|
||||
// album record is left in place: deleting it would be a bigger action
|
||||
// than the user asked for, and it may predate this request.
|
||||
func (l *lidarr) Withdraw(ctx context.Context, externalID string) error {
|
||||
albumID, err := strconv.Atoi(externalID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: bad album id %q", ErrLidarrNoMatch, externalID)
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"albumIds": []int{albumID},
|
||||
"monitored": false,
|
||||
}
|
||||
|
||||
return l.client.put(ctx, "/api/v1/album/monitor", body, nil)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// rootFolders lists Lidarr's configured library roots.
|
||||
func (l *lidarr) rootFolders(ctx context.Context) ([]lidarrRootFolder, error) {
|
||||
var folders []lidarrRootFolder
|
||||
|
||||
if err := l.client.get(ctx, "/api/v1/rootfolder", &folders); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return folders, nil
|
||||
}
|
||||
|
||||
// profiles resolves the quality and metadata profile IDs to use,
|
||||
// falling back to the first of each when unconfigured.
|
||||
func (l *lidarr) profiles(ctx context.Context) (quality, metadata int, err error) {
|
||||
quality, metadata = l.qualityProfileID, l.metadataProfileID
|
||||
|
||||
if quality == 0 {
|
||||
var profiles []struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
if err := l.client.get(ctx, "/api/v1/qualityprofile", &profiles); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
if len(profiles) == 0 {
|
||||
return 0, 0, fmt.Errorf(
|
||||
"%w: no quality profiles configured", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
quality = profiles[0].ID
|
||||
}
|
||||
|
||||
if metadata == 0 {
|
||||
var profiles []struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
if err := l.client.get(ctx, "/api/v1/metadataprofile", &profiles); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
if len(profiles) == 0 {
|
||||
return 0, 0, fmt.Errorf(
|
||||
"%w: no metadata profiles configured", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
metadata = profiles[0].ID
|
||||
}
|
||||
|
||||
return quality, metadata, nil
|
||||
}
|
||||
|
||||
// command posts to Lidarr's command endpoint.
|
||||
func (l *lidarr) command(ctx context.Context, body map[string]any) error {
|
||||
return l.client.post(ctx, "/api/v1/command", body, nil)
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Lidarr's Lister role: mirroring this app's wanted list into Lidarr's
|
||||
// own monitoring.
|
||||
//
|
||||
// Lidarr already models exactly what a want is — a monitored artist or
|
||||
// a monitored album — so the mapping is direct and, more usefully, it
|
||||
// means an artist subscription made here keeps working through Lidarr's
|
||||
// own release-checking even while this app is closed. That is the
|
||||
// whole reason the Lister role exists: a desktop player is not running
|
||||
// most of the time, and a always-on system that already watches for new
|
||||
// releases is a better place for a subscription to live than a loop
|
||||
// that only ticks when someone opens the app.
|
||||
//
|
||||
// The mapping is deliberately lossy in one direction only:
|
||||
//
|
||||
// artist want -> Lidarr artist, monitored
|
||||
// release-group want -> Lidarr album, monitored (artist added if new)
|
||||
// release want -> same, at release-group granularity
|
||||
// recording want -> not pushed; Lidarr has no concept of wanting
|
||||
// one track, and monitoring the whole album to
|
||||
// get it would download far more than asked.
|
||||
//
|
||||
// Nothing here searches. Pushing a want expresses intent; Lidarr
|
||||
// decides when to act on it, which is the point of delegating.
|
||||
|
||||
// PushWant records a want in Lidarr's own monitoring.
|
||||
//
|
||||
// It is idempotent because Lidarr is: adding an artist that already
|
||||
// exists returns the existing record, and monitoring an already-
|
||||
// monitored album is a no-op. Callers rely on that — the reconciler
|
||||
// pushes on every pass until it gets an ID back.
|
||||
func (l *lidarr) PushWant(ctx context.Context, w Want) (string, error) {
|
||||
switch w.Entity {
|
||||
case EntityArtist:
|
||||
return l.pushArtistWant(ctx, w)
|
||||
case EntityReleaseGroup, EntityRelease:
|
||||
return l.pushAlbumWant(ctx, w)
|
||||
case EntityRecording:
|
||||
// Deliberately unsupported rather than approximated. See the
|
||||
// mapping note above.
|
||||
return "", nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: entity %q", ErrUnsupported, w.Entity)
|
||||
}
|
||||
}
|
||||
|
||||
// pushArtistWant makes Lidarr monitor an artist.
|
||||
//
|
||||
// Scope is honoured through Lidarr's own monitor option rather than by
|
||||
// pushing each album separately: "future" maps to monitoring new
|
||||
// releases only, "all" to monitoring everything missing. Letting
|
||||
// Lidarr apply the policy means it stays applied to albums released
|
||||
// after this push, which is what a subscription is for.
|
||||
func (l *lidarr) pushArtistWant(ctx context.Context, w Want) (string, error) {
|
||||
existing, err := l.findArtistByMBID(ctx, w.MBID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
monitor := "future"
|
||||
if w.Scope == ScopeAll {
|
||||
monitor = "missing"
|
||||
}
|
||||
|
||||
if existing.ID != 0 {
|
||||
return strconv.Itoa(existing.ID), nil
|
||||
}
|
||||
|
||||
root, err := l.resolveRootFolder(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
quality, metadata, err := l.profiles(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
name := w.Artist
|
||||
if name == "" {
|
||||
name = w.Title
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"foreignArtistId": w.MBID,
|
||||
"artistName": name,
|
||||
"qualityProfileId": quality,
|
||||
"metadataProfileId": metadata,
|
||||
"rootFolderPath": root,
|
||||
"monitored": true,
|
||||
"addOptions": map[string]any{
|
||||
"monitor": monitor,
|
||||
// Searching is left off even for a full-discography
|
||||
// subscription: adding an artist should not launch forty
|
||||
// simultaneous searches on a system the user shares with
|
||||
// their own queue. Lidarr picks the albums up on its next
|
||||
// scheduled search.
|
||||
"searchForMissingAlbums": false,
|
||||
},
|
||||
}
|
||||
|
||||
var created struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
if err := l.client.post(ctx, "/api/v1/artist", body, &created); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return strconv.Itoa(created.ID), nil
|
||||
}
|
||||
|
||||
// pushAlbumWant makes Lidarr monitor one album, adding its artist if
|
||||
// Lidarr has never heard of them.
|
||||
func (l *lidarr) pushAlbumWant(ctx context.Context, w Want) (string, error) {
|
||||
album, err := l.findAlbum(ctx, Request{
|
||||
ReleaseGroupMBID: w.MBID,
|
||||
Artist: w.Artist,
|
||||
Album: w.Title,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if album.ID == 0 {
|
||||
album, err = l.addArtistForAlbum(ctx, album)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if !album.Monitored {
|
||||
if err := l.monitorAlbum(ctx, album.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return strconv.Itoa(album.ID), nil
|
||||
}
|
||||
|
||||
// RemoveWant stops Lidarr monitoring something.
|
||||
//
|
||||
// It unmonitors rather than deletes: the user's Lidarr may have been
|
||||
// monitoring that artist long before this app existed, and removing a
|
||||
// want here is not permission to tear down their setup. An unmonitored
|
||||
// artist stays in their library with its files intact.
|
||||
func (l *lidarr) RemoveWant(ctx context.Context, externalID string) error {
|
||||
id, err := strconv.Atoi(externalID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: bad lidarr id %q", ErrLidarrNoMatch, externalID)
|
||||
}
|
||||
|
||||
// The ID may name an album or an artist and the caller does not
|
||||
// track which, so try the album endpoint first and fall back.
|
||||
if err := l.client.put(ctx, "/api/v1/album/monitor", map[string]any{
|
||||
"albumIds": []int{id},
|
||||
"monitored": false,
|
||||
}, nil); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var artist map[string]any
|
||||
|
||||
endpoint := "/api/v1/artist/" + strconv.Itoa(id)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &artist); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
artist["monitored"] = false
|
||||
|
||||
return l.client.put(ctx, endpoint, artist, nil)
|
||||
}
|
||||
|
||||
// ListWants reads Lidarr's monitored artists back, for the deliberate
|
||||
// "import what Lidarr is already watching" action.
|
||||
//
|
||||
// Only artists are imported, not their individual monitored albums: an
|
||||
// artist is the durable statement of intent, and importing every
|
||||
// monitored album alongside it would produce a wanted list that is
|
||||
// mostly redundant with the subscription that generated it.
|
||||
func (l *lidarr) ListWants(ctx context.Context) ([]Want, error) {
|
||||
var artists []struct {
|
||||
lidarrArtist
|
||||
|
||||
Monitored bool `json:"monitored"`
|
||||
}
|
||||
|
||||
if err := l.client.get(ctx, "/api/v1/artist", &artists); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]Want, 0, len(artists))
|
||||
|
||||
for _, a := range artists {
|
||||
if !a.Monitored || a.ForeignArtistID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, Want{
|
||||
MBID: a.ForeignArtistID,
|
||||
Entity: EntityArtist,
|
||||
Artist: a.ArtistName,
|
||||
Title: a.ArtistName,
|
||||
// Imported subscriptions take the conservative scope: the
|
||||
// user can widen it, but silently queueing a back catalogue
|
||||
// on import would be a nasty surprise.
|
||||
Scope: ScopeFuture,
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// findArtistByMBID looks up an artist Lidarr already has.
|
||||
func (l *lidarr) findArtistByMBID(
|
||||
ctx context.Context,
|
||||
mbid string,
|
||||
) (lidarrArtist, error) {
|
||||
var artists []lidarrArtist
|
||||
|
||||
endpoint := "/api/v1/artist?mbId=" + url.QueryEscape(mbid)
|
||||
|
||||
if err := l.client.get(ctx, endpoint, &artists); err != nil {
|
||||
return lidarrArtist{}, err
|
||||
}
|
||||
|
||||
for _, a := range artists {
|
||||
if a.ForeignArtistID == mbid {
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
return lidarrArtist{}, nil
|
||||
}
|
||||
|
||||
// resolveRootFolder returns the configured root folder, or Lidarr's
|
||||
// first if none is configured.
|
||||
func (l *lidarr) resolveRootFolder(ctx context.Context) (string, error) {
|
||||
if l.rootFolderPath != "" {
|
||||
return l.rootFolderPath, nil
|
||||
}
|
||||
|
||||
folders, err := l.rootFolders(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(folders) == 0 {
|
||||
return "", ErrLidarrNoRootFolder
|
||||
}
|
||||
|
||||
return folders[0].Path, nil
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An artist subscription becomes a monitored Lidarr artist, with the
|
||||
// scope translated into Lidarr's own monitor option — so the policy
|
||||
// keeps applying to albums released after the push, which is the whole
|
||||
// reason to mirror a subscription rather than a list of albums.
|
||||
func TestLidarrPushArtistWantMapsScope(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
scope WantScope
|
||||
wantMonitor string
|
||||
}{
|
||||
{name: "future", scope: ScopeFuture, wantMonitor: "future"},
|
||||
{name: "all", scope: ScopeAll, wantMonitor: "missing"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
id, err := l.PushWant(context.Background(), Want{
|
||||
MBID: "artist-mbid",
|
||||
Entity: EntityArtist,
|
||||
Artist: "Radiohead",
|
||||
Scope: tt.scope,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: PushWant: %v", tt.name, err)
|
||||
}
|
||||
|
||||
if id != "42" {
|
||||
t.Errorf("%s: external id = %q, want 42", tt.name, id)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
added := append([]map[string]any(nil), stub.addedArtists...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(added) != 1 {
|
||||
t.Fatalf("%s: added %d artists, want 1", tt.name, len(added))
|
||||
}
|
||||
|
||||
opts, ok := added[0]["addOptions"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s: no addOptions in the artist body", tt.name)
|
||||
}
|
||||
|
||||
if opts["monitor"] != tt.wantMonitor {
|
||||
t.Errorf(
|
||||
"%s: monitor = %v, want %q",
|
||||
tt.name, opts["monitor"], tt.wantMonitor,
|
||||
)
|
||||
}
|
||||
|
||||
// Adding an artist must never kick off a discography-wide
|
||||
// search on a system the user shares with their own queue.
|
||||
if opts["searchForMissingAlbums"] != false {
|
||||
t.Errorf("%s: push triggered a search", tt.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pushing a want Lidarr already has returns the existing ID instead of
|
||||
// adding a second copy — the reconciler pushes on every pass, so this
|
||||
// is load-bearing rather than tidy.
|
||||
func TestLidarrPushArtistWantIsIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.artists = []map[string]any{{
|
||||
"id": 7,
|
||||
"artistName": "Radiohead",
|
||||
"foreignArtistId": "artist-mbid",
|
||||
"monitored": true,
|
||||
}}
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
id, err := l.PushWant(context.Background(), Want{
|
||||
MBID: "artist-mbid",
|
||||
Entity: EntityArtist,
|
||||
Artist: "Radiohead",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PushWant: %v", err)
|
||||
}
|
||||
|
||||
if id != "7" {
|
||||
t.Errorf("external id = %q, want the existing artist's 7", id)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
added := len(stub.addedArtists)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if added != 0 {
|
||||
t.Errorf("added %d artists, want 0 — it already existed", added)
|
||||
}
|
||||
}
|
||||
|
||||
// Lidarr cannot express "I want one track", and monitoring the whole
|
||||
// album to get it would download far more than was asked for.
|
||||
func TestLidarrPushRecordingWantIsSkipped(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
id, err := l.PushWant(context.Background(), Want{
|
||||
MBID: "recording-mbid",
|
||||
Entity: EntityRecording,
|
||||
Title: "Paranoid Android",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PushWant: %v", err)
|
||||
}
|
||||
|
||||
if id != "" {
|
||||
t.Errorf("external id = %q, want empty (not pushed)", id)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
added := len(stub.addedArtists)
|
||||
monitors := len(stub.monitorCalls)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if added != 0 || monitors != 0 {
|
||||
t.Errorf(
|
||||
"track want touched Lidarr: %d artists, %d monitors",
|
||||
added, monitors,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Importing adopts monitored artists conservatively: a subscription
|
||||
// pulled in from elsewhere must not queue a back catalogue.
|
||||
func TestLidarrListWantsImportsMonitoredArtistsOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.artists = []map[string]any{
|
||||
{
|
||||
"id": 1,
|
||||
"artistName": "Radiohead",
|
||||
"foreignArtistId": "artist-1",
|
||||
"monitored": true,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"artistName": "Unmonitored Band",
|
||||
"foreignArtistId": "artist-2",
|
||||
"monitored": false,
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"artistName": "No MBID",
|
||||
"foreignArtistId": "",
|
||||
"monitored": true,
|
||||
},
|
||||
}
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
wants, err := l.ListWants(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListWants: %v", err)
|
||||
}
|
||||
|
||||
if len(wants) != 1 {
|
||||
t.Fatalf("imported %d wants, want 1", len(wants))
|
||||
}
|
||||
|
||||
w := wants[0]
|
||||
|
||||
if w.MBID != "artist-1" {
|
||||
t.Errorf("mbid = %q, want artist-1", w.MBID)
|
||||
}
|
||||
|
||||
if w.Entity != EntityArtist {
|
||||
t.Errorf("entity = %q, want artist", w.Entity)
|
||||
}
|
||||
|
||||
if w.Scope != ScopeFuture {
|
||||
t.Errorf("scope = %q, want the conservative future", w.Scope)
|
||||
}
|
||||
}
|
||||
|
||||
// Removing a want must not tear down a Lidarr setup that may predate
|
||||
// this app: it unmonitors, it does not delete.
|
||||
func TestLidarrRemoveWantUnmonitorsOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if err := l.RemoveWant(context.Background(), "55"); err != nil {
|
||||
t.Fatalf("RemoveWant: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
monitors := append([]map[string]any(nil), stub.monitorCalls...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(monitors) != 1 {
|
||||
t.Fatalf("got %d monitor calls, want 1", len(monitors))
|
||||
}
|
||||
|
||||
if monitors[0]["monitored"] != false {
|
||||
t.Errorf("monitored = %v, want false", monitors[0]["monitored"])
|
||||
}
|
||||
}
|
||||
|
||||
// The Lister role has to be declared, not just implemented, or the
|
||||
// reconciler never finds it.
|
||||
func TestLidarrDeclaresListerCapability(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if _, ok := asLister(l); !ok {
|
||||
t.Error("lidarr does not present as a Lister")
|
||||
}
|
||||
|
||||
desc, ok := DescriptorFor(KindLidarr)
|
||||
if !ok {
|
||||
t.Fatal("no descriptor registered for lidarr")
|
||||
}
|
||||
|
||||
if !desc.Caps.CanList {
|
||||
t.Error("lidarr's descriptor does not declare CanList")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// lidarrStub is a fake Lidarr instance.
|
||||
type lidarrStub struct {
|
||||
server *httptest.Server
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
// searchAlbum is what /api/v1/search returns.
|
||||
searchAlbum lidarrAlbum
|
||||
|
||||
// album is what /api/v1/album/{id} returns, in order; the last
|
||||
// entry repeats.
|
||||
albumStates []lidarrAlbum
|
||||
pollCount int
|
||||
|
||||
// trackFiles is what /api/v1/trackfile returns.
|
||||
trackFiles []lidarrTrackFile
|
||||
|
||||
// rootFolders is what /api/v1/rootfolder returns.
|
||||
rootFolders []lidarrRootFolder
|
||||
|
||||
// commands records the commands that were issued.
|
||||
commands []string
|
||||
|
||||
// monitorCalls records album-monitor toggles.
|
||||
monitorCalls []map[string]any
|
||||
|
||||
// addedArtists records artist additions.
|
||||
addedArtists []map[string]any
|
||||
|
||||
// artists is what a GET of /api/v1/artist returns, which is how the
|
||||
// Lister role looks up and enumerates monitored artists.
|
||||
artists []map[string]any
|
||||
|
||||
unauthorized bool
|
||||
}
|
||||
|
||||
func newLidarrStub(t *testing.T) *lidarrStub {
|
||||
t.Helper()
|
||||
|
||||
s := &lidarrStub{
|
||||
rootFolders: []lidarrRootFolder{{ID: 1, Path: "/music"}},
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/api/v1/system/status", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, map[string]any{"version": "2.0.0"})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/rootfolder", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
folders := s.rootFolders
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, folders)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/qualityprofile", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, []map[string]any{{"id": 7}})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/metadataprofile", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, []map[string]any{{"id": 3}})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
album := s.searchAlbum
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, []map[string]any{{"album": album}})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/artist", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodGet {
|
||||
s.mu.Lock()
|
||||
artists := s.artists
|
||||
s.mu.Unlock()
|
||||
|
||||
if artists == nil {
|
||||
artists = []map[string]any{}
|
||||
}
|
||||
|
||||
writeJSON(t, w, artists)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode artist body: %v", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.addedArtists = append(s.addedArtists, body)
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{"id": 42})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/album/monitor", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode monitor body: %v", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.monitorCalls = append(s.monitorCalls, body)
|
||||
s.mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/album", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
album := s.searchAlbum
|
||||
s.mu.Unlock()
|
||||
|
||||
album.ID = 99
|
||||
|
||||
writeJSON(t, w, []lidarrAlbum{album})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/album/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
idx := s.pollCount
|
||||
if idx >= len(s.albumStates) {
|
||||
idx = len(s.albumStates) - 1
|
||||
} else {
|
||||
s.pollCount++
|
||||
}
|
||||
|
||||
var album lidarrAlbum
|
||||
if idx >= 0 && len(s.albumStates) > 0 {
|
||||
album = s.albumStates[idx]
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, album)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/trackfile", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
files := s.trackFiles
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, files)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/command", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode command body: %v", err)
|
||||
}
|
||||
|
||||
name, _ := body["name"].(string)
|
||||
|
||||
s.mu.Lock()
|
||||
s.commands = append(s.commands, name)
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{"id": 1})
|
||||
})
|
||||
|
||||
s.server = httptest.NewServer(mux)
|
||||
t.Cleanup(s.server.Close)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *lidarrStub) reject(w http.ResponseWriter, r *http.Request) bool {
|
||||
s.mu.Lock()
|
||||
unauthorized := s.unauthorized
|
||||
s.mu.Unlock()
|
||||
|
||||
if unauthorized || r.Header.Get("X-Api-Key") != "test-key" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func newStubLidarr(t *testing.T, stub *lidarrStub) *lidarr {
|
||||
t.Helper()
|
||||
|
||||
p, err := newLidarr(
|
||||
Config{
|
||||
ID: 1,
|
||||
Kind: KindLidarr,
|
||||
Name: "lidarr",
|
||||
Enabled: true,
|
||||
Settings: map[string]string{
|
||||
"url": stub.server.URL,
|
||||
},
|
||||
},
|
||||
func(string) (string, error) { return "test-key", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newLidarr: %v", err)
|
||||
}
|
||||
|
||||
l, ok := p.(*lidarr)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *lidarr", p)
|
||||
}
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
func TestLidarrCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if err := l.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLidarrCheckRejectsBadKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.unauthorized = true
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if err := l.Check(context.Background()); !errors.Is(err, ErrLidarrAuth) {
|
||||
t.Errorf("error = %v, want ErrLidarrAuth", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Lidarr with nowhere to put music cannot fulfil anything, and that
|
||||
// should be visible at configuration time.
|
||||
func TestLidarrCheckRequiresRootFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.rootFolders = nil
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if err := l.Check(context.Background()); !errors.Is(
|
||||
err, ErrLidarrNoRootFolder,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrLidarrNoRootFolder", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An album Lidarr already tracks only needs monitoring and a search.
|
||||
func TestLidarrDelegateExistingAlbum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.searchAlbum = lidarrAlbum{
|
||||
ID: 55,
|
||||
Title: "OK Computer",
|
||||
ForeignAlbum: "rg-mbid",
|
||||
Monitored: false,
|
||||
}
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
externalID, err := l.Delegate(context.Background(), Request{
|
||||
ReleaseGroupMBID: "rg-mbid",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Delegate: %v", err)
|
||||
}
|
||||
|
||||
if externalID != "55" {
|
||||
t.Errorf("external id = %q, want 55", externalID)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
commands := append([]string(nil), stub.commands...)
|
||||
monitors := len(stub.monitorCalls)
|
||||
added := len(stub.addedArtists)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if added != 0 {
|
||||
t.Errorf("added %d artists, want 0 for an album Lidarr already has", added)
|
||||
}
|
||||
|
||||
if monitors != 1 {
|
||||
t.Errorf("monitor calls = %d, want 1", monitors)
|
||||
}
|
||||
|
||||
if len(commands) != 1 || commands[0] != "AlbumSearch" {
|
||||
t.Errorf("commands = %v, want [AlbumSearch]", commands)
|
||||
}
|
||||
}
|
||||
|
||||
// Adding an artist must not kick off their entire discography.
|
||||
func TestLidarrDelegateNewArtistMonitorsNothingByDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.searchAlbum = lidarrAlbum{
|
||||
Title: "OK Computer",
|
||||
ForeignAlbum: "rg-mbid",
|
||||
}
|
||||
stub.searchAlbum.Artist.ForeignArtistID = "artist-mbid"
|
||||
stub.searchAlbum.Artist.ArtistName = "Radiohead"
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if _, err := l.Delegate(context.Background(), Request{
|
||||
ReleaseGroupMBID: "rg-mbid",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
}); err != nil {
|
||||
t.Fatalf("Delegate: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
added := append([]map[string]any(nil), stub.addedArtists...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(added) != 1 {
|
||||
t.Fatalf("added %d artists, want 1", len(added))
|
||||
}
|
||||
|
||||
opts, ok := added[0]["addOptions"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("addOptions missing from %v", added[0])
|
||||
}
|
||||
|
||||
if opts["monitor"] != "none" {
|
||||
t.Errorf("monitor = %v, want none", opts["monitor"])
|
||||
}
|
||||
|
||||
if opts["searchForMissingAlbums"] != false {
|
||||
t.Errorf(
|
||||
"searchForMissingAlbums = %v, want false",
|
||||
opts["searchForMissingAlbums"],
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLidarrDelegateNoMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
// searchAlbum stays zero-valued: no title means no match.
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
_, err := l.Delegate(context.Background(), Request{
|
||||
Artist: "Nobody",
|
||||
Album: "Nothing",
|
||||
})
|
||||
|
||||
if !errors.Is(err, ErrLidarrNoMatch) {
|
||||
t.Errorf("error = %v, want ErrLidarrNoMatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Completion is judged by imported files, not by an empty queue: the
|
||||
// queue drains when the download finishes, which is before the import.
|
||||
func TestLidarrPollWaitsForImportedFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
stub.albumStates = []lidarrAlbum{
|
||||
{ID: 55, Statistics: lidarrAlbumStats{TrackCount: 12}},
|
||||
{ID: 55, Statistics: lidarrAlbumStats{TrackFileCount: 6, TrackCount: 12}},
|
||||
{ID: 55, Statistics: lidarrAlbumStats{TrackFileCount: 12, TrackCount: 12}},
|
||||
}
|
||||
stub.trackFiles = []lidarrTrackFile{
|
||||
{ID: 1, Path: "/music/Radiohead/OK Computer/01 Airbag.flac"},
|
||||
{ID: 2, Path: "/music/Radiohead/OK Computer/02 Paranoid Android.flac"},
|
||||
}
|
||||
|
||||
l := newStubLidarr(t, stub)
|
||||
ctx := context.Background()
|
||||
|
||||
// Nothing imported yet.
|
||||
first, err := l.Poll(ctx, "55")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll: %v", err)
|
||||
}
|
||||
|
||||
if first.State != StateGrabbing {
|
||||
t.Errorf("state = %q, want grabbing", first.State)
|
||||
}
|
||||
|
||||
// Half done.
|
||||
second, err := l.Poll(ctx, "55")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll: %v", err)
|
||||
}
|
||||
|
||||
if second.State != StateGrabbing {
|
||||
t.Errorf("state = %q, want grabbing", second.State)
|
||||
}
|
||||
|
||||
if second.Progress <= first.Progress {
|
||||
t.Errorf(
|
||||
"progress did not advance: %f then %f",
|
||||
first.Progress, second.Progress,
|
||||
)
|
||||
}
|
||||
|
||||
// Complete, with the paths Lidarr imported to.
|
||||
third, err := l.Poll(ctx, "55")
|
||||
if err != nil {
|
||||
t.Fatalf("Poll: %v", err)
|
||||
}
|
||||
|
||||
if third.State != StateComplete {
|
||||
t.Fatalf("state = %q, want complete", third.State)
|
||||
}
|
||||
|
||||
if len(third.ImportedPaths) != 2 {
|
||||
t.Errorf("imported paths = %v, want 2", third.ImportedPaths)
|
||||
}
|
||||
|
||||
for _, p := range third.ImportedPaths {
|
||||
if !strings.HasPrefix(p, "/music/") {
|
||||
t.Errorf("path %q is not in Lidarr's library", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLidarrPollRejectsBadExternalID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if _, err := l.Poll(context.Background(), "not-a-number"); !errors.Is(
|
||||
err, ErrLidarrNoMatch,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrLidarrNoMatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Withdrawing stops monitoring; it must not delete the album, which may
|
||||
// predate this request.
|
||||
func TestLidarrWithdrawUnmonitorsOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newLidarrStub(t)
|
||||
l := newStubLidarr(t, stub)
|
||||
|
||||
if err := l.Withdraw(context.Background(), "55"); err != nil {
|
||||
t.Fatalf("Withdraw: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
calls := append([]map[string]any(nil), stub.monitorCalls...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("monitor calls = %d, want 1", len(calls))
|
||||
}
|
||||
|
||||
if calls[0]["monitored"] != false {
|
||||
t.Errorf("monitored = %v, want false", calls[0]["monitored"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLidarrRequiresConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("no url", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newLidarr(
|
||||
Config{},
|
||||
func(string) (string, error) { return "k", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrNotConfigured) {
|
||||
t.Errorf("error = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no api key", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newLidarr(
|
||||
Config{Settings: map[string]string{"url": "http://localhost:8686"}},
|
||||
func(string) (string, error) { return "", ErrSecretNotFound },
|
||||
slogDiscard(),
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrNotConfigured) {
|
||||
t.Errorf("error = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Prowlarr is the one provider that cannot finish a job by itself: it
|
||||
// searches dozens of indexers and hands back magnet links and NZB URLs,
|
||||
// which some other client has to actually fetch. That is the whole
|
||||
// reason the role interfaces are separate — Prowlarr implements
|
||||
// Searcher and nothing else, and the pipeline pairs its results with
|
||||
// whichever enabled transport handles the protocol.
|
||||
//
|
||||
// Its search results also carry no file list. A torrent is one opaque
|
||||
// blob until it is fetched, so match scoring here has only the release
|
||||
// title to work with, and candidates are marked accordingly rather than
|
||||
// pretending to know what is inside.
|
||||
|
||||
// Prowlarr provider errors.
|
||||
var (
|
||||
// ErrProwlarrUnreachable means the instance did not answer.
|
||||
ErrProwlarrUnreachable = errors.New("prowlarr is unreachable")
|
||||
|
||||
// ErrProwlarrAuth means the API key was rejected.
|
||||
ErrProwlarrAuth = errors.New("prowlarr rejected the API key")
|
||||
|
||||
// ErrProwlarrNoIndexers means nothing is configured to search.
|
||||
ErrProwlarrNoIndexers = errors.New("prowlarr has no enabled indexers")
|
||||
)
|
||||
|
||||
// prowlarrHTTPTimeout bounds one API call. Indexer fan-out is slow, so
|
||||
// this is longer than the other adapters'.
|
||||
const prowlarrHTTPTimeout = 45 * time.Second
|
||||
|
||||
// prowlarrMusicCategory is Newznab's music category. Searching without
|
||||
// it returns every match across film and software too.
|
||||
const prowlarrMusicCategory = "3000"
|
||||
|
||||
// prowlarrMaxResults caps how many results are turned into candidates.
|
||||
const prowlarrMaxResults = 40
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindProwlarr,
|
||||
Name: "Prowlarr",
|
||||
Summary: "Search many torrent and usenet indexers at once. " +
|
||||
"Needs a download client (qBittorrent or SABnzbd) to fetch results.",
|
||||
RequiresExternal: "Prowlarr",
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "url",
|
||||
Label: "Prowlarr URL",
|
||||
Placeholder: "http://localhost:9696",
|
||||
Required: true,
|
||||
Default: "http://localhost:9696",
|
||||
},
|
||||
{
|
||||
Key: "apiKey",
|
||||
Label: "API key",
|
||||
Secret: true,
|
||||
Required: true,
|
||||
Help: "Prowlarr → Settings → General → API Key.",
|
||||
},
|
||||
{
|
||||
Key: "indexerIds",
|
||||
Label: "Indexer IDs",
|
||||
Help: "Comma-separated numeric IDs to restrict the search to. " +
|
||||
"Leave blank to search all enabled indexers.",
|
||||
},
|
||||
{
|
||||
Key: "minSeeders",
|
||||
Label: "Minimum seeders",
|
||||
Help: "Torrent results below this are hidden. " +
|
||||
"Defaults to 1.",
|
||||
Default: "1",
|
||||
},
|
||||
},
|
||||
},
|
||||
newProwlarr,
|
||||
)
|
||||
}
|
||||
|
||||
// prowlarr is the Prowlarr search provider.
|
||||
type prowlarr struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
client *apiClient
|
||||
|
||||
indexerIDs []string
|
||||
minSeeders int
|
||||
}
|
||||
|
||||
// newProwlarr builds the provider from config.
|
||||
func newProwlarr(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
base := strings.TrimRight(cfg.Setting("url", ""), "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("%w: Prowlarr URL is required", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey := ""
|
||||
|
||||
if secrets != nil {
|
||||
key, err := secrets("apiKey")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey = key
|
||||
}
|
||||
|
||||
minSeeders, err := strconv.Atoi(cfg.Setting("minSeeders", "1"))
|
||||
if err != nil {
|
||||
minSeeders = 1
|
||||
}
|
||||
|
||||
var indexers []string
|
||||
|
||||
for _, id := range strings.Split(cfg.Setting("indexerIds", ""), ",") {
|
||||
if trimmed := strings.TrimSpace(id); trimmed != "" {
|
||||
indexers = append(indexers, trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
return &prowlarr{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindProwlarr,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{CanSearch: true},
|
||||
},
|
||||
logger: logger.With("provider", "prowlarr"),
|
||||
client: newAPIClient(
|
||||
base, "X-Api-Key", apiKey, prowlarrHTTPTimeout,
|
||||
ErrProwlarrUnreachable, ErrProwlarrAuth,
|
||||
),
|
||||
indexerIDs: indexers,
|
||||
minSeeders: minSeeders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (p *prowlarr) Info() ProviderInfo {
|
||||
return p.info
|
||||
}
|
||||
|
||||
// Close is a no-op.
|
||||
func (p *prowlarr) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check verifies the instance answers and has something to search.
|
||||
func (p *prowlarr) Check(ctx context.Context) error {
|
||||
var status struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
if err := p.client.get(ctx, "/api/v1/system/status", &status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var indexers []struct {
|
||||
ID int `json:"id"`
|
||||
Enable bool `json:"enable"`
|
||||
}
|
||||
|
||||
if err := p.client.get(ctx, "/api/v1/indexer", &indexers); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, i := range indexers {
|
||||
if i.Enable {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return ErrProwlarrNoIndexers
|
||||
}
|
||||
|
||||
// prowlarrResult is one indexer hit.
|
||||
type prowlarrResult struct {
|
||||
GUID string `json:"guid"`
|
||||
Title string `json:"title"`
|
||||
Indexer string `json:"indexer"`
|
||||
Size int64 `json:"size"`
|
||||
Seeders int `json:"seeders"`
|
||||
Leechers int `json:"leechers"`
|
||||
Protocol string `json:"protocol"` // "torrent" or "usenet"
|
||||
DownloadURL string `json:"downloadUrl"`
|
||||
MagnetURL string `json:"magnetUrl"`
|
||||
InfoHash string `json:"infoHash"`
|
||||
}
|
||||
|
||||
// Search queries every configured indexer through Prowlarr.
|
||||
func (p *prowlarr) Search(
|
||||
ctx context.Context,
|
||||
req Request,
|
||||
) ([]Candidate, error) {
|
||||
query := url.Values{}
|
||||
query.Set("query", req.SearchText())
|
||||
query.Set("categories", prowlarrMusicCategory)
|
||||
query.Set("type", "search")
|
||||
|
||||
for _, id := range p.indexerIDs {
|
||||
query.Add("indexerIds", id)
|
||||
}
|
||||
|
||||
var results []prowlarrResult
|
||||
|
||||
endpoint := "/api/v1/search?" + query.Encode()
|
||||
|
||||
if err := p.client.get(ctx, endpoint, &results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]Candidate, 0, len(results))
|
||||
|
||||
for _, r := range results {
|
||||
if len(out) >= prowlarrMaxResults {
|
||||
break
|
||||
}
|
||||
|
||||
protocol := protocolFor(r.Protocol)
|
||||
if protocol == ProtocolDirect {
|
||||
continue
|
||||
}
|
||||
|
||||
// A torrent with no seeders will never finish. Offering it
|
||||
// wastes the user's pick on something that cannot complete.
|
||||
if protocol == ProtocolTorrent && r.Seeders < p.minSeeders {
|
||||
continue
|
||||
}
|
||||
|
||||
link := r.MagnetURL
|
||||
if link == "" {
|
||||
link = r.DownloadURL
|
||||
}
|
||||
|
||||
if link == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, Candidate{
|
||||
ID: "prowlarr:" + r.GUID,
|
||||
Kind: KindProwlarr,
|
||||
Protocol: protocol,
|
||||
Title: r.Title,
|
||||
Origin: r.Indexer,
|
||||
// Indexer results are opaque before they are fetched: there
|
||||
// is no file list, so no per-file scoring is possible and
|
||||
// the ranker works from the release title alone.
|
||||
Files: nil,
|
||||
TotalSize: r.Size,
|
||||
Health: swarmHealth(protocol, r.Seeders),
|
||||
Payload: map[string]string{
|
||||
"link": link,
|
||||
"indexer": r.Indexer,
|
||||
"infoHash": r.InfoHash,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// protocolFor maps Prowlarr's protocol string onto ours.
|
||||
func protocolFor(s string) Protocol {
|
||||
switch strings.ToLower(s) {
|
||||
case "torrent":
|
||||
return ProtocolTorrent
|
||||
case "usenet":
|
||||
return ProtocolUsenet
|
||||
default:
|
||||
return ProtocolDirect
|
||||
}
|
||||
}
|
||||
|
||||
// swarmHealth scores availability, in 0..1. Usenet has no swarm: a
|
||||
// retained article either downloads at full speed or is gone, so it
|
||||
// gets a flat, confident score.
|
||||
func swarmHealth(protocol Protocol, seeders int) float64 {
|
||||
if protocol == ProtocolUsenet {
|
||||
return 0.85
|
||||
}
|
||||
|
||||
// Seeder counts have sharply diminishing returns — the difference
|
||||
// between 1 and 10 is enormous, between 100 and 500 irrelevant.
|
||||
switch {
|
||||
case seeders <= 0:
|
||||
return 0.05
|
||||
case seeders == 1:
|
||||
return 0.3
|
||||
case seeders < 5:
|
||||
return 0.5
|
||||
case seeders < 20:
|
||||
return 0.75
|
||||
case seeders < 100:
|
||||
return 0.9
|
||||
default:
|
||||
return 1.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// prowlarrStub is a fake Prowlarr instance.
|
||||
type prowlarrStub struct {
|
||||
server *httptest.Server
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
results []prowlarrResult
|
||||
indexers []map[string]any
|
||||
|
||||
// lastQuery records the search query string for assertions.
|
||||
lastQuery string
|
||||
|
||||
unauthorized bool
|
||||
}
|
||||
|
||||
func newProwlarrStub(t *testing.T) *prowlarrStub {
|
||||
t.Helper()
|
||||
|
||||
s := &prowlarrStub{
|
||||
indexers: []map[string]any{{"id": 1, "enable": true}},
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/api/v1/system/status", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, map[string]any{"version": "1.0.0"})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/indexer", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
indexers := s.indexers
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, indexers)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v1/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.lastQuery = r.URL.RawQuery
|
||||
results := s.results
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, results)
|
||||
})
|
||||
|
||||
s.server = httptest.NewServer(mux)
|
||||
t.Cleanup(s.server.Close)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *prowlarrStub) reject(w http.ResponseWriter, r *http.Request) bool {
|
||||
s.mu.Lock()
|
||||
unauthorized := s.unauthorized
|
||||
s.mu.Unlock()
|
||||
|
||||
if unauthorized || r.Header.Get("X-Api-Key") != "test-key" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func newStubProwlarr(t *testing.T, stub *prowlarrStub, settings map[string]string) *prowlarr {
|
||||
t.Helper()
|
||||
|
||||
if settings == nil {
|
||||
settings = map[string]string{}
|
||||
}
|
||||
|
||||
settings["url"] = stub.server.URL
|
||||
|
||||
p, err := newProwlarr(
|
||||
Config{ID: 1, Kind: KindProwlarr, Name: "prowlarr", Settings: settings},
|
||||
func(string) (string, error) { return "test-key", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newProwlarr: %v", err)
|
||||
}
|
||||
|
||||
pr, ok := p.(*prowlarr)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *prowlarr", p)
|
||||
}
|
||||
|
||||
return pr
|
||||
}
|
||||
|
||||
func TestProwlarrCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newProwlarrStub(t)
|
||||
p := newStubProwlarr(t, stub, nil)
|
||||
|
||||
if err := p.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prowlarr with every indexer disabled will silently return nothing
|
||||
// forever, which is worth surfacing at configuration time.
|
||||
func TestProwlarrCheckRequiresEnabledIndexer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newProwlarrStub(t)
|
||||
stub.indexers = []map[string]any{{"id": 1, "enable": false}}
|
||||
|
||||
p := newStubProwlarr(t, stub, nil)
|
||||
|
||||
if err := p.Check(context.Background()); !errors.Is(
|
||||
err, ErrProwlarrNoIndexers,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrProwlarrNoIndexers", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prowlarr fills only the Searcher role, so its candidates must carry a
|
||||
// protocol the pipeline can pair with a transport.
|
||||
func TestProwlarrSearchMarksProtocols(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newProwlarrStub(t)
|
||||
stub.results = []prowlarrResult{
|
||||
{
|
||||
GUID: "a",
|
||||
Title: "Radiohead - OK Computer [FLAC]",
|
||||
Indexer: "SomeTracker",
|
||||
Protocol: "torrent",
|
||||
Seeders: 50,
|
||||
Size: 400_000_000,
|
||||
MagnetURL: "magnet:?xt=urn:btih:abc123",
|
||||
},
|
||||
{
|
||||
GUID: "b",
|
||||
Title: "Radiohead - OK Computer [MP3]",
|
||||
Indexer: "SomeUsenet",
|
||||
Protocol: "usenet",
|
||||
Size: 90_000_000,
|
||||
DownloadURL: "https://example.com/x.nzb",
|
||||
},
|
||||
}
|
||||
|
||||
p := newStubProwlarr(t, stub, nil)
|
||||
|
||||
got, err := p.Search(context.Background(), Request{
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2", len(got))
|
||||
}
|
||||
|
||||
byProtocol := map[Protocol]Candidate{}
|
||||
for _, c := range got {
|
||||
byProtocol[c.Protocol] = c
|
||||
}
|
||||
|
||||
torrent, ok := byProtocol[ProtocolTorrent]
|
||||
if !ok {
|
||||
t.Fatal("no torrent candidate")
|
||||
}
|
||||
|
||||
if torrent.Payload["link"] != "magnet:?xt=urn:btih:abc123" {
|
||||
t.Errorf("torrent link = %q, want the magnet", torrent.Payload["link"])
|
||||
}
|
||||
|
||||
usenet, ok := byProtocol[ProtocolUsenet]
|
||||
if !ok {
|
||||
t.Fatal("no usenet candidate")
|
||||
}
|
||||
|
||||
if usenet.Payload["link"] != "https://example.com/x.nzb" {
|
||||
t.Errorf("usenet link = %q, want the NZB URL", usenet.Payload["link"])
|
||||
}
|
||||
|
||||
// Indexer results are opaque before fetching; claiming a file list
|
||||
// would be inventing information.
|
||||
if len(torrent.Files) != 0 {
|
||||
t.Errorf("torrent candidate has %d files, want none", len(torrent.Files))
|
||||
}
|
||||
}
|
||||
|
||||
// A torrent nobody is seeding will never finish, so offering it wastes
|
||||
// the user's choice.
|
||||
func TestProwlarrFiltersDeadTorrents(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newProwlarrStub(t)
|
||||
stub.results = []prowlarrResult{
|
||||
{
|
||||
GUID: "dead", Title: "Dead", Protocol: "torrent",
|
||||
Seeders: 0, MagnetURL: "magnet:?xt=urn:btih:dead",
|
||||
},
|
||||
{
|
||||
GUID: "alive", Title: "Alive", Protocol: "torrent",
|
||||
Seeders: 10, MagnetURL: "magnet:?xt=urn:btih:alive",
|
||||
},
|
||||
}
|
||||
|
||||
p := newStubProwlarr(t, stub, map[string]string{"minSeeders": "1"})
|
||||
|
||||
got, err := p.Search(context.Background(), Request{Query: "x"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1", len(got))
|
||||
}
|
||||
|
||||
if got[0].Title != "Alive" {
|
||||
t.Errorf("kept %q, want the seeded torrent", got[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
// Searching without a category constraint returns films and software
|
||||
// alongside music.
|
||||
func TestProwlarrSearchesMusicCategory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newProwlarrStub(t)
|
||||
p := newStubProwlarr(t, stub, nil)
|
||||
|
||||
if _, err := p.Search(
|
||||
context.Background(), Request{Query: "radiohead"},
|
||||
); err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
query := stub.lastQuery
|
||||
stub.mu.Unlock()
|
||||
|
||||
if !strings.Contains(query, "categories="+prowlarrMusicCategory) {
|
||||
t.Errorf("query %q does not constrain to the music category", query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwarmHealth(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// More seeders is never worse.
|
||||
prev := -1.0
|
||||
|
||||
for _, seeders := range []int{0, 1, 3, 10, 50, 500} {
|
||||
got := swarmHealth(ProtocolTorrent, seeders)
|
||||
|
||||
if got < prev {
|
||||
t.Errorf("health fell at %d seeders: %f after %f", seeders, got, prev)
|
||||
}
|
||||
|
||||
if got < 0 || got > 1 {
|
||||
t.Errorf("health %f out of range at %d seeders", got, seeders)
|
||||
}
|
||||
|
||||
prev = got
|
||||
}
|
||||
|
||||
// Usenet has no swarm, so seeder count is meaningless there.
|
||||
if a, b := swarmHealth(ProtocolUsenet, 0), swarmHealth(ProtocolUsenet, 99); a != b {
|
||||
t.Errorf("usenet health varied with seeders: %f vs %f", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolFor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := map[string]Protocol{
|
||||
"torrent": ProtocolTorrent,
|
||||
"Torrent": ProtocolTorrent,
|
||||
"usenet": ProtocolUsenet,
|
||||
"USENET": ProtocolUsenet,
|
||||
"weird": ProtocolDirect,
|
||||
"": ProtocolDirect,
|
||||
}
|
||||
|
||||
for in, want := range tests {
|
||||
if got := protocolFor(in); got != want {
|
||||
t.Errorf("protocolFor(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,595 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// qBittorrent is a pure transport: it never searches, it just takes a
|
||||
// magnet link Prowlarr found and moves the bytes. That is the point of
|
||||
// splitting Transporter out — this adapter knows nothing about music.
|
||||
//
|
||||
// Two things make it different from the others. Its auth is a session
|
||||
// cookie rather than a header, so it needs its own client. And it can
|
||||
// be told where to save, which means the transfer lands directly in our
|
||||
// staging directory instead of needing to be collected afterwards —
|
||||
// provided qBittorrent sees the same filesystem we do.
|
||||
|
||||
// qBittorrent provider errors.
|
||||
var (
|
||||
// ErrQbitUnreachable means the instance did not answer.
|
||||
ErrQbitUnreachable = errors.New("qbittorrent is unreachable")
|
||||
|
||||
// ErrQbitAuth means the credentials were rejected.
|
||||
ErrQbitAuth = errors.New("qbittorrent rejected the credentials")
|
||||
|
||||
// ErrQbitNoHash means the candidate carried no usable torrent
|
||||
// identifier, so the transfer could not be tracked.
|
||||
ErrQbitNoHash = errors.New("candidate has no torrent hash")
|
||||
|
||||
// ErrQbitTransferFailed means the torrent errored or stalled out.
|
||||
ErrQbitTransferFailed = errors.New("qbittorrent transfer failed")
|
||||
)
|
||||
|
||||
// qBittorrent tuning.
|
||||
const (
|
||||
// qbitHTTPTimeout bounds one API call.
|
||||
qbitHTTPTimeout = 30 * time.Second
|
||||
|
||||
// qbitPollInterval is how often torrent state is checked.
|
||||
qbitPollInterval = 5 * time.Second
|
||||
|
||||
// qbitAppearWait is how long to wait for a just-added torrent to
|
||||
// show up in the torrent list before concluding it was rejected.
|
||||
qbitAppearWait = 60 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindQBittorrent,
|
||||
Name: "qBittorrent",
|
||||
Summary: "Download torrents found by an indexer. " +
|
||||
"Pairs with Prowlarr; does not search on its own.",
|
||||
RequiresExternal: "qBittorrent",
|
||||
Caps: Caps{
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
CanResume: true,
|
||||
ReportsSize: true,
|
||||
Transports: []Protocol{ProtocolTorrent},
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "url",
|
||||
Label: "qBittorrent URL",
|
||||
Placeholder: "http://localhost:8080",
|
||||
Required: true,
|
||||
Default: "http://localhost:8080",
|
||||
},
|
||||
{
|
||||
Key: "username",
|
||||
Label: "Username",
|
||||
Required: true,
|
||||
Default: "admin",
|
||||
},
|
||||
{
|
||||
Key: "password",
|
||||
Label: "Password",
|
||||
Secret: true,
|
||||
Required: true,
|
||||
},
|
||||
{
|
||||
Key: "category",
|
||||
Label: "Category",
|
||||
Help: "qBittorrent category to tag these downloads with. " +
|
||||
"Useful for keeping them out of your other rules.",
|
||||
Default: "yellowjacket",
|
||||
},
|
||||
},
|
||||
},
|
||||
newQBittorrent,
|
||||
)
|
||||
}
|
||||
|
||||
// qbittorrent is the qBittorrent transport.
|
||||
type qbittorrent struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
client *http.Client
|
||||
|
||||
baseURL string
|
||||
username string
|
||||
password string
|
||||
category string
|
||||
|
||||
// authMu guards the lazy login, so concurrent grabs share one
|
||||
// session instead of racing to create several.
|
||||
authMu sync.Mutex
|
||||
authenticated bool
|
||||
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
// newQBittorrent builds the transport from config.
|
||||
func newQBittorrent(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
base := strings.TrimRight(cfg.Setting("url", ""), "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: qBittorrent URL is required", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
password := ""
|
||||
|
||||
if secrets != nil {
|
||||
pw, err := secrets("password")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: no password stored", ErrNotConfigured)
|
||||
}
|
||||
|
||||
password = pw
|
||||
}
|
||||
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create cookie jar: %w", err)
|
||||
}
|
||||
|
||||
return &qbittorrent{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindQBittorrent,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
CanResume: true,
|
||||
ReportsSize: true,
|
||||
Transports: []Protocol{ProtocolTorrent},
|
||||
},
|
||||
},
|
||||
logger: logger.With("provider", "qbittorrent"),
|
||||
client: &http.Client{
|
||||
Timeout: qbitHTTPTimeout,
|
||||
Jar: jar,
|
||||
},
|
||||
baseURL: base,
|
||||
username: cfg.Setting("username", "admin"),
|
||||
password: password,
|
||||
category: cfg.Setting("category", "yellowjacket"),
|
||||
pollInterval: qbitPollInterval,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (q *qbittorrent) Info() ProviderInfo {
|
||||
return q.info
|
||||
}
|
||||
|
||||
// Close is a no-op; the session expires on its own.
|
||||
func (q *qbittorrent) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check logs in and asks for the version.
|
||||
func (q *qbittorrent) Check(ctx context.Context) error {
|
||||
if err := q.login(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := q.call(ctx, "/api/v2/app/version", nil)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// login establishes a session cookie. qBittorrent answers a bad login
|
||||
// with 200 and the body "Fails.", not a 401, so the body is what has to
|
||||
// be checked.
|
||||
func (q *qbittorrent) login(ctx context.Context) error {
|
||||
q.authMu.Lock()
|
||||
defer q.authMu.Unlock()
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("username", q.username)
|
||||
form.Set("password", q.password)
|
||||
|
||||
body, err := q.post(ctx, "/api/v2/auth/login", form)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.Contains(strings.ToLower(body), "ok") {
|
||||
q.authenticated = false
|
||||
|
||||
return ErrQbitAuth
|
||||
}
|
||||
|
||||
q.authenticated = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureAuth logs in if this client has not yet done so.
|
||||
func (q *qbittorrent) ensureAuth(ctx context.Context) error {
|
||||
q.authMu.Lock()
|
||||
done := q.authenticated
|
||||
q.authMu.Unlock()
|
||||
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
|
||||
return q.login(ctx)
|
||||
}
|
||||
|
||||
// qbitTorrent is the subset of qBittorrent's torrent list used here.
|
||||
type qbitTorrent struct {
|
||||
Hash string `json:"hash"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Progress float64 `json:"progress"`
|
||||
Size int64 `json:"size"`
|
||||
Completed int64 `json:"completed"`
|
||||
ContentPath string `json:"content_path"`
|
||||
SavePath string `json:"save_path"`
|
||||
}
|
||||
|
||||
// finished reports whether the torrent has all its data.
|
||||
func (t qbitTorrent) finished() bool {
|
||||
switch t.State {
|
||||
case "uploading", "stalledUP", "queuedUP", "pausedUP", "forcedUP",
|
||||
"checkingUP":
|
||||
return true
|
||||
default:
|
||||
return t.Progress >= 1.0
|
||||
}
|
||||
}
|
||||
|
||||
// failed reports whether the torrent is in an unrecoverable state.
|
||||
func (t qbitTorrent) failed() bool {
|
||||
return t.State == "error" || t.State == "missingFiles"
|
||||
}
|
||||
|
||||
// Grab adds the torrent, waits for it to complete, and collects its
|
||||
// files into dst.
|
||||
func (q *qbittorrent) Grab(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error) {
|
||||
if err := q.ensureAuth(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
link := c.Payload["link"]
|
||||
if link == "" {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: candidate has no magnet or torrent URL", ErrQbitNoHash,
|
||||
)
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("urls", link)
|
||||
form.Set("savepath", dst)
|
||||
form.Set("category", q.category)
|
||||
// Skip qBittorrent's own "move on completion" rules: the file must
|
||||
// stay where we put it until the import step decides otherwise.
|
||||
form.Set("autoTMM", "false")
|
||||
|
||||
if _, err := q.post(ctx, "/api/v2/torrents/add", form); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
hash, err := q.resolveHash(ctx, c, link)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
torrent, err := q.await(ctx, hash, c, onProgress)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return collectTree(torrent.ContentPath, dst)
|
||||
}
|
||||
|
||||
// resolveHash finds the torrent's hash, preferring the one the indexer
|
||||
// supplied and falling back to matching the newest torrent in our
|
||||
// category — qBittorrent's add endpoint returns nothing useful.
|
||||
func (q *qbittorrent) resolveHash(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
link string,
|
||||
) (string, error) {
|
||||
if h := c.Payload["infoHash"]; h != "" {
|
||||
return strings.ToLower(h), nil
|
||||
}
|
||||
|
||||
if h := infoHashFromMagnet(link); h != "" {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// Poll briefly for a torrent in our category that was not there
|
||||
// before; the add is asynchronous.
|
||||
deadline := time.Now().Add(qbitAppearWait)
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("%w: cancelled", ErrQbitTransferFailed)
|
||||
case <-time.After(q.pollInterval):
|
||||
}
|
||||
|
||||
torrents, err := q.list(ctx, "")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, t := range torrents {
|
||||
if strings.EqualFold(t.Name, c.Title) {
|
||||
return t.Hash, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf(
|
||||
"%w: torrent never appeared in qBittorrent", ErrQbitNoHash,
|
||||
)
|
||||
}
|
||||
|
||||
// await polls until the torrent finishes or fails.
|
||||
func (q *qbittorrent) await(
|
||||
ctx context.Context,
|
||||
hash string,
|
||||
c Candidate,
|
||||
onProgress ProgressFunc,
|
||||
) (qbitTorrent, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return qbitTorrent{}, fmt.Errorf(
|
||||
"%w: cancelled", ErrQbitTransferFailed,
|
||||
)
|
||||
case <-time.After(q.pollInterval):
|
||||
}
|
||||
|
||||
torrents, err := q.list(ctx, hash)
|
||||
if err != nil {
|
||||
q.logger.Debug("qbittorrent poll failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if len(torrents) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
t := torrents[0]
|
||||
|
||||
if onProgress != nil {
|
||||
total := t.Size
|
||||
if total == 0 {
|
||||
total = c.TotalSize
|
||||
}
|
||||
|
||||
onProgress(Progress{
|
||||
Current: t.Completed,
|
||||
Total: total,
|
||||
Phase: "Downloading torrent (" + t.State + ")",
|
||||
})
|
||||
}
|
||||
|
||||
if t.failed() {
|
||||
return qbitTorrent{}, fmt.Errorf(
|
||||
"%w: state %s", ErrQbitTransferFailed, t.State,
|
||||
)
|
||||
}
|
||||
|
||||
if t.finished() {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// list returns torrents, optionally filtered to one hash.
|
||||
func (q *qbittorrent) list(
|
||||
ctx context.Context,
|
||||
hash string,
|
||||
) ([]qbitTorrent, error) {
|
||||
params := url.Values{}
|
||||
if hash != "" {
|
||||
params.Set("hashes", hash)
|
||||
}
|
||||
|
||||
var out []qbitTorrent
|
||||
|
||||
if err := q.getJSON(ctx, "/api/v2/torrents/info", params, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// infoHashFromMagnet extracts the btih hash from a magnet URI.
|
||||
func infoHashFromMagnet(magnet string) string {
|
||||
if !strings.HasPrefix(magnet, "magnet:") {
|
||||
return ""
|
||||
}
|
||||
|
||||
u, err := url.Parse(magnet)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, xt := range u.Query()["xt"] {
|
||||
if after, ok := strings.CutPrefix(xt, "urn:btih:"); ok {
|
||||
return strings.ToLower(after)
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// collectTree gathers every file under root into dst. A torrent may be
|
||||
// a single file or a directory tree; either way the importer wants a
|
||||
// flat set of paths inside the staging directory.
|
||||
func collectTree(root, dst string) (Result, error) {
|
||||
result := Result{Dir: dst, Files: []string{}}
|
||||
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("stat downloaded content: %w", err)
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
target := filepath.Join(dst, filepath.Base(root))
|
||||
|
||||
if root != target {
|
||||
if err := movePath(root, target); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
result.Files = append(result.Files, target)
|
||||
result.BytesTransferred = info.Size()
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err //nolint:wrapcheck // walk error passthrough
|
||||
}
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil || fi.Size() == 0 {
|
||||
return nil //nolint:nilerr // skip unreadable entries
|
||||
}
|
||||
|
||||
target := filepath.Join(dst, filepath.Base(path))
|
||||
|
||||
if path != target {
|
||||
if err := movePath(path, target); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
result.Files = append(result.Files, target)
|
||||
result.BytesTransferred += fi.Size()
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("collect downloaded files: %w", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// call performs a GET and returns the raw body.
|
||||
func (q *qbittorrent) call(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
params url.Values,
|
||||
) (string, error) {
|
||||
target := q.baseURL + endpoint
|
||||
if len(params) > 0 {
|
||||
target += "?" + params.Encode()
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build qbittorrent request: %w", err)
|
||||
}
|
||||
|
||||
return q.send(req)
|
||||
}
|
||||
|
||||
// getJSON performs a GET and decodes JSON.
|
||||
func (q *qbittorrent) getJSON(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
params url.Values,
|
||||
out any,
|
||||
) error {
|
||||
body, err := q.call(ctx, endpoint, params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := decodeJSON(body, out); err != nil {
|
||||
return fmt.Errorf("decode qbittorrent response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// post performs a form POST and returns the raw body.
|
||||
func (q *qbittorrent) post(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
form url.Values,
|
||||
) (string, error) {
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
q.baseURL+endpoint,
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build qbittorrent request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
// qBittorrent rejects cross-origin requests unless Referer matches.
|
||||
req.Header.Set("Referer", q.baseURL)
|
||||
|
||||
return q.send(req)
|
||||
}
|
||||
|
||||
// send executes a request and normalizes failures.
|
||||
func (q *qbittorrent) send(req *http.Request) (string, error) {
|
||||
resp, err := q.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %w", ErrQbitUnreachable, err)
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusForbidden:
|
||||
return "", ErrQbitAuth
|
||||
case resp.StatusCode >= 400:
|
||||
return "", fmt.Errorf(
|
||||
"%w: HTTP %d: %s",
|
||||
ErrQbitUnreachable, resp.StatusCode, strings.TrimSpace(string(body)),
|
||||
)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SABnzbd is the usenet half of the split-role pair: Prowlarr finds an
|
||||
// NZB, SABnzbd fetches and unpacks it.
|
||||
//
|
||||
// Like slskd, it writes to its own completed-downloads directory rather
|
||||
// than one we choose, and unlike qBittorrent there is no per-job save
|
||||
// path we can set reliably across versions. It does, however, report
|
||||
// the final storage path in its history, so the collect step reads that
|
||||
// rather than guessing.
|
||||
|
||||
// SABnzbd provider errors.
|
||||
var (
|
||||
// ErrSabUnreachable means the instance did not answer.
|
||||
ErrSabUnreachable = errors.New("sabnzbd is unreachable")
|
||||
|
||||
// ErrSabAuth means the API key was rejected.
|
||||
ErrSabAuth = errors.New("sabnzbd rejected the API key")
|
||||
|
||||
// ErrSabTransferFailed means the job failed or was removed.
|
||||
ErrSabTransferFailed = errors.New("sabnzbd job failed")
|
||||
|
||||
// ErrSabNoJob means the queued job vanished from both queue and
|
||||
// history without completing.
|
||||
ErrSabNoJob = errors.New("sabnzbd job disappeared")
|
||||
)
|
||||
|
||||
// SABnzbd tuning.
|
||||
const (
|
||||
// sabHTTPTimeout bounds one API call.
|
||||
sabHTTPTimeout = 30 * time.Second
|
||||
|
||||
// sabPollInterval is how often job state is checked.
|
||||
sabPollInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindSABnzbd,
|
||||
Name: "SABnzbd",
|
||||
Summary: "Download usenet releases found by an indexer. " +
|
||||
"Pairs with Prowlarr; does not search on its own.",
|
||||
RequiresExternal: "SABnzbd",
|
||||
Caps: Caps{
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
Transports: []Protocol{ProtocolUsenet},
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "url",
|
||||
Label: "SABnzbd URL",
|
||||
Placeholder: "http://localhost:8080",
|
||||
Required: true,
|
||||
Default: "http://localhost:8080",
|
||||
},
|
||||
{
|
||||
Key: "apiKey",
|
||||
Label: "API key",
|
||||
Secret: true,
|
||||
Required: true,
|
||||
Help: "SABnzbd → Config → General → API Key.",
|
||||
},
|
||||
{
|
||||
Key: "category",
|
||||
Label: "Category",
|
||||
Help: "SABnzbd category for these downloads. " +
|
||||
"Its folder must be readable from this machine.",
|
||||
Default: "music",
|
||||
},
|
||||
},
|
||||
},
|
||||
newSABnzbd,
|
||||
)
|
||||
}
|
||||
|
||||
// sabnzbd is the SABnzbd transport.
|
||||
type sabnzbd struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
client *apiClient
|
||||
|
||||
apiKey string
|
||||
category string
|
||||
|
||||
pollInterval time.Duration
|
||||
}
|
||||
|
||||
// newSABnzbd builds the transport from config.
|
||||
func newSABnzbd(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
base := strings.TrimRight(cfg.Setting("url", ""), "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("%w: SABnzbd URL is required", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey := ""
|
||||
|
||||
if secrets != nil {
|
||||
key, err := secrets("apiKey")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey = key
|
||||
}
|
||||
|
||||
return &sabnzbd{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindSABnzbd,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
Transports: []Protocol{ProtocolUsenet},
|
||||
},
|
||||
},
|
||||
logger: logger.With("provider", "sabnzbd"),
|
||||
// SABnzbd authenticates by query parameter, not header, so the
|
||||
// shared client carries no auth header here.
|
||||
client: newAPIClient(
|
||||
base, "", "", sabHTTPTimeout, ErrSabUnreachable, ErrSabAuth,
|
||||
),
|
||||
apiKey: apiKey,
|
||||
category: cfg.Setting("category", "music"),
|
||||
pollInterval: sabPollInterval,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (s *sabnzbd) Info() ProviderInfo {
|
||||
return s.info
|
||||
}
|
||||
|
||||
// Close is a no-op.
|
||||
func (s *sabnzbd) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sabResponse is SABnzbd's common envelope. It answers a bad API key
|
||||
// with HTTP 200 and status:false, so the body has to be inspected.
|
||||
type sabResponse struct {
|
||||
Status bool `json:"status"`
|
||||
Error string `json:"error"`
|
||||
NzoIDs []string `json:"nzo_ids"`
|
||||
}
|
||||
|
||||
// sabQueue is the queue view.
|
||||
type sabQueue struct {
|
||||
Queue struct {
|
||||
Slots []sabQueueSlot `json:"slots"`
|
||||
} `json:"queue"`
|
||||
}
|
||||
|
||||
// sabQueueSlot is one in-flight job.
|
||||
type sabQueueSlot struct {
|
||||
NzoID string `json:"nzo_id"`
|
||||
Filename string `json:"filename"`
|
||||
Status string `json:"status"`
|
||||
Percentage string `json:"percentage"`
|
||||
MB string `json:"mb"`
|
||||
MBLeft string `json:"mbleft"`
|
||||
}
|
||||
|
||||
// sabHistory is the history view.
|
||||
type sabHistory struct {
|
||||
History struct {
|
||||
Slots []sabHistorySlot `json:"slots"`
|
||||
} `json:"history"`
|
||||
}
|
||||
|
||||
// sabHistorySlot is one finished job. Storage is the unpacked path,
|
||||
// which is the only reliable way to find what SABnzbd produced.
|
||||
type sabHistorySlot struct {
|
||||
NzoID string `json:"nzo_id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Storage string `json:"storage"`
|
||||
FailMsg string `json:"fail_message"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
// Check verifies the instance answers and the key is accepted.
|
||||
func (s *sabnzbd) Check(ctx context.Context) error {
|
||||
var resp struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
if err := s.call(ctx, url.Values{"mode": {"version"}}, &resp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// version answers without auth, so make one authenticated call too.
|
||||
var queue sabQueue
|
||||
|
||||
return s.call(ctx, url.Values{"mode": {"queue"}}, &queue)
|
||||
}
|
||||
|
||||
// Grab adds the NZB, waits for SABnzbd to finish, and moves the
|
||||
// unpacked files into dst.
|
||||
func (s *sabnzbd) Grab(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error) {
|
||||
link := c.Payload["link"]
|
||||
if link == "" {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: candidate has no NZB URL", ErrSabTransferFailed,
|
||||
)
|
||||
}
|
||||
|
||||
if err := validateHTTPURL(link); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
var added sabResponse
|
||||
|
||||
if err := s.call(ctx, url.Values{
|
||||
"mode": {"addurl"},
|
||||
"name": {link},
|
||||
"cat": {s.category},
|
||||
"nzbname": {c.Title},
|
||||
"priority": {"0"},
|
||||
}, &added); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
if !added.Status || len(added.NzoIDs) == 0 {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: %s", ErrSabTransferFailed, added.Error,
|
||||
)
|
||||
}
|
||||
|
||||
nzoID := added.NzoIDs[0]
|
||||
|
||||
slot, err := s.await(ctx, nzoID, onProgress)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
if !strings.EqualFold(slot.Status, "Completed") {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: %s: %s", ErrSabTransferFailed, slot.Status, slot.FailMsg,
|
||||
)
|
||||
}
|
||||
|
||||
return collectTree(slot.Storage, dst)
|
||||
}
|
||||
|
||||
// await polls the queue until the job leaves it, then reads history for
|
||||
// the outcome. SABnzbd moves a job from queue to history when it
|
||||
// finishes post-processing, so history is where completion is truthful.
|
||||
func (s *sabnzbd) await(
|
||||
ctx context.Context,
|
||||
nzoID string,
|
||||
onProgress ProgressFunc,
|
||||
) (sabHistorySlot, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return sabHistorySlot{}, fmt.Errorf(
|
||||
"%w: cancelled", ErrSabTransferFailed,
|
||||
)
|
||||
case <-time.After(s.pollInterval):
|
||||
}
|
||||
|
||||
inQueue, slot, err := s.queueSlot(ctx, nzoID)
|
||||
if err != nil {
|
||||
s.logger.Debug("sabnzbd queue poll failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if inQueue {
|
||||
if onProgress != nil {
|
||||
onProgress(Progress{
|
||||
Current: parseMB(slot.MB) - parseMB(slot.MBLeft),
|
||||
Total: parseMB(slot.MB),
|
||||
Phase: "Downloading from usenet (" + slot.Status + ")",
|
||||
})
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
found, hist, err := s.historySlot(ctx, nzoID)
|
||||
if err != nil {
|
||||
s.logger.Debug("sabnzbd history poll failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
return hist, nil
|
||||
}
|
||||
|
||||
// Not in the queue and not in history: the job was removed out
|
||||
// from under us.
|
||||
return sabHistorySlot{}, ErrSabNoJob
|
||||
}
|
||||
}
|
||||
|
||||
// queueSlot looks for a job in the queue.
|
||||
func (s *sabnzbd) queueSlot(
|
||||
ctx context.Context,
|
||||
nzoID string,
|
||||
) (bool, sabQueueSlot, error) {
|
||||
var queue sabQueue
|
||||
|
||||
if err := s.call(ctx, url.Values{"mode": {"queue"}}, &queue); err != nil {
|
||||
return false, sabQueueSlot{}, err
|
||||
}
|
||||
|
||||
for _, slot := range queue.Queue.Slots {
|
||||
if slot.NzoID == nzoID {
|
||||
return true, slot, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, sabQueueSlot{}, nil
|
||||
}
|
||||
|
||||
// historySlot looks for a job in history.
|
||||
func (s *sabnzbd) historySlot(
|
||||
ctx context.Context,
|
||||
nzoID string,
|
||||
) (bool, sabHistorySlot, error) {
|
||||
var history sabHistory
|
||||
|
||||
if err := s.call(
|
||||
ctx, url.Values{"mode": {"history"}}, &history,
|
||||
); err != nil {
|
||||
return false, sabHistorySlot{}, err
|
||||
}
|
||||
|
||||
for _, slot := range history.History.Slots {
|
||||
if slot.NzoID == nzoID {
|
||||
return true, slot, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, sabHistorySlot{}, nil
|
||||
}
|
||||
|
||||
// call performs one API request. SABnzbd puts everything on the query
|
||||
// string of a single endpoint.
|
||||
func (s *sabnzbd) call(ctx context.Context, params url.Values, out any) error {
|
||||
params.Set("apikey", s.apiKey)
|
||||
params.Set("output", "json")
|
||||
|
||||
return s.client.get(ctx, "/api?"+params.Encode(), out)
|
||||
}
|
||||
|
||||
// parseMB converts SABnzbd's megabyte strings to bytes. Values are
|
||||
// decimal strings like "1024.5"; a malformed one yields 0 rather than
|
||||
// failing a transfer that is otherwise fine.
|
||||
func parseMB(s string) int64 {
|
||||
const bytesPerMB = 1024 * 1024
|
||||
|
||||
var mb float64
|
||||
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(s), "%f", &mb); err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return int64(mb * bytesPerMB)
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Soulseek is reached through a user-run slskd daemon rather than the
|
||||
// wire protocol. That trades a setup step for not having to implement
|
||||
// peer connections, distributed search, and upload obligations — and
|
||||
// keeps the user's Soulseek credentials in their daemon instead of in
|
||||
// this process.
|
||||
//
|
||||
// One wrinkle shapes this adapter: slskd downloads into its own
|
||||
// configured directory, not one we hand it. There is no API to stream
|
||||
// a finished file back. So the user tells us where that directory is,
|
||||
// and Grab waits for the transfer, then moves the files into staging.
|
||||
// When slskd runs on another machine, that path has to be a mount —
|
||||
// which is why Check verifies it exists rather than discovering the
|
||||
// problem after a two-hour transfer.
|
||||
|
||||
// slskd provider errors.
|
||||
var (
|
||||
// ErrSlskdUnreachable means the daemon did not answer.
|
||||
ErrSlskdUnreachable = errors.New("slskd is unreachable")
|
||||
|
||||
// ErrSlskdAuth means the API key was rejected.
|
||||
ErrSlskdAuth = errors.New("slskd rejected the API key")
|
||||
|
||||
// ErrSlskdDownloadsPath means the configured downloads directory is
|
||||
// missing or unreadable from this machine.
|
||||
ErrSlskdDownloadsPath = errors.New(
|
||||
"slskd downloads directory is not readable from here",
|
||||
)
|
||||
|
||||
// ErrSlskdTransferFailed means a peer transfer ended badly.
|
||||
ErrSlskdTransferFailed = errors.New("slskd transfer failed")
|
||||
|
||||
// ErrSlskdTimeout means a search or transfer outlived its budget.
|
||||
ErrSlskdTimeout = errors.New("slskd timed out")
|
||||
)
|
||||
|
||||
// slskd tuning.
|
||||
const (
|
||||
// slskdSearchPoll is how often an in-flight search is polled.
|
||||
slskdSearchPoll = 1 * time.Second
|
||||
|
||||
// slskdSearchWait bounds a single search. Soulseek searches return
|
||||
// results progressively; waiting the full budget gets noticeably
|
||||
// more peers than bailing at the first response.
|
||||
slskdSearchWait = 12 * time.Second
|
||||
|
||||
// slskdTransferPoll is how often transfer state is polled.
|
||||
slskdTransferPoll = 3 * time.Second
|
||||
|
||||
// slskdMinFiles is the fewest audio files a folder needs before it
|
||||
// is offered as a candidate. Soulseek returns a lot of one-file
|
||||
// noise for common queries.
|
||||
slskdMinFiles = 2
|
||||
|
||||
// slskdHTTPTimeout bounds one API call.
|
||||
slskdHTTPTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindSlskd,
|
||||
Name: "Soulseek (slskd)",
|
||||
Summary: "Search and download from the Soulseek network " +
|
||||
"through your own slskd daemon.",
|
||||
RequiresExternal: "slskd",
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "url",
|
||||
Label: "slskd URL",
|
||||
Placeholder: "http://localhost:5030",
|
||||
Required: true,
|
||||
Default: "http://localhost:5030",
|
||||
},
|
||||
{
|
||||
Key: "apiKey",
|
||||
Label: "API key",
|
||||
Secret: true,
|
||||
Required: true,
|
||||
Help: "From your slskd configuration under web.authentication.",
|
||||
},
|
||||
{
|
||||
Key: "downloadsPath",
|
||||
Label: "slskd downloads folder",
|
||||
Placeholder: "/var/lib/slskd/downloads",
|
||||
Required: true,
|
||||
Help: "The folder slskd saves to, as this machine sees it. " +
|
||||
"If slskd runs elsewhere, this must be a mounted share.",
|
||||
},
|
||||
},
|
||||
},
|
||||
newSlskd,
|
||||
)
|
||||
}
|
||||
|
||||
// slskd is the Soulseek provider.
|
||||
type slskd struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
client *apiClient
|
||||
|
||||
downloadsPath string
|
||||
|
||||
// Poll intervals are fields rather than constants so tests can run
|
||||
// the full search-and-transfer flow without sleeping through it.
|
||||
searchPoll time.Duration
|
||||
searchWait time.Duration
|
||||
transferPoll time.Duration
|
||||
}
|
||||
|
||||
// newSlskd builds the provider from config.
|
||||
func newSlskd(
|
||||
cfg Config,
|
||||
secrets SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
base := strings.TrimRight(cfg.Setting("url", ""), "/")
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("%w: slskd URL is required", ErrNotConfigured)
|
||||
}
|
||||
|
||||
downloads := cfg.Setting("downloadsPath", "")
|
||||
if downloads == "" {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: slskd downloads folder is required", ErrNotConfigured,
|
||||
)
|
||||
}
|
||||
|
||||
apiKey := ""
|
||||
|
||||
if secrets != nil {
|
||||
key, err := secrets("apiKey")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured)
|
||||
}
|
||||
|
||||
apiKey = key
|
||||
}
|
||||
|
||||
return &slskd{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindSlskd,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
},
|
||||
},
|
||||
logger: logger.With("provider", "slskd"),
|
||||
client: newAPIClient(
|
||||
base, "X-Api-Key", apiKey, slskdHTTPTimeout,
|
||||
ErrSlskdUnreachable, ErrSlskdAuth,
|
||||
),
|
||||
downloadsPath: downloads,
|
||||
searchPoll: slskdSearchPoll,
|
||||
searchWait: slskdSearchWait,
|
||||
transferPoll: slskdTransferPoll,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (s *slskd) Info() ProviderInfo {
|
||||
return s.info
|
||||
}
|
||||
|
||||
// Close is a no-op; the HTTP client holds no session.
|
||||
func (s *slskd) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check verifies the daemon answers, the key is accepted, and the
|
||||
// downloads directory is readable from this machine.
|
||||
func (s *slskd) Check(ctx context.Context) error {
|
||||
var app map[string]any
|
||||
|
||||
if err := s.client.get(ctx, "/api/v0/application", &app); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, err := os.Stat(s.downloadsPath)
|
||||
if err != nil || !info.IsDir() {
|
||||
return fmt.Errorf("%w: %s", ErrSlskdDownloadsPath, s.downloadsPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// API types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// slskdSearch is a search as slskd reports it.
|
||||
type slskdSearch struct {
|
||||
ID string `json:"id"`
|
||||
IsComplete bool `json:"isComplete"`
|
||||
Responses []slskdResponse `json:"responses"`
|
||||
}
|
||||
|
||||
// slskdResponse is one peer's answer to a search.
|
||||
type slskdResponse struct {
|
||||
Username string `json:"username"`
|
||||
HasFreeUploadSlot bool `json:"hasFreeUploadSlot"`
|
||||
QueueLength int `json:"queueLength"`
|
||||
UploadSpeed int64 `json:"uploadSpeed"`
|
||||
Files []slskdFile `json:"files"`
|
||||
LockedFileCount int `json:"lockedFileCount"`
|
||||
FileCount int `json:"fileCount"`
|
||||
FreeUploadSlotFlag bool `json:"freeUploadSlots"`
|
||||
}
|
||||
|
||||
// slskdFile is one file a peer is offering.
|
||||
type slskdFile struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
BitRate int `json:"bitRate"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
|
||||
// slskdTransfer is one download's state.
|
||||
type slskdTransfer struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
Filename string `json:"filename"`
|
||||
State string `json:"state"`
|
||||
Size int64 `json:"size"`
|
||||
BytesTransferred int64 `json:"bytesTransferred"`
|
||||
}
|
||||
|
||||
// done reports whether the transfer reached a terminal state, and
|
||||
// whether it succeeded. slskd reports compound states such as
|
||||
// "Completed, Succeeded" and "Completed, Errored".
|
||||
func (t slskdTransfer) done() (finished, ok bool) {
|
||||
if !strings.Contains(t.State, "Completed") {
|
||||
return false, false
|
||||
}
|
||||
|
||||
return true, strings.Contains(t.State, "Succeeded")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Search
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Search runs a Soulseek search and groups the results into per-peer,
|
||||
// per-folder candidates. A folder from one peer is the unit a user
|
||||
// actually wants: Soulseek has no album concept, but people organise
|
||||
// their shares by album directory.
|
||||
func (s *slskd) Search(ctx context.Context, req Request) ([]Candidate, error) {
|
||||
searchID := newID()
|
||||
|
||||
body := map[string]any{
|
||||
"id": searchID,
|
||||
"searchText": req.SearchText(),
|
||||
}
|
||||
|
||||
if err := s.client.post(ctx, "/api/v0/searches", body, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
search, err := s.awaitSearch(ctx, searchID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Best effort cleanup; a left-behind search is harmless but clutters
|
||||
// the slskd UI.
|
||||
defer func() {
|
||||
_ = s.client.delete(
|
||||
context.WithoutCancel(ctx), "/api/v0/searches/"+searchID,
|
||||
)
|
||||
}()
|
||||
|
||||
return s.candidatesFrom(search), nil
|
||||
}
|
||||
|
||||
// awaitSearch polls until the search completes or the budget runs out.
|
||||
// A timeout is not an error: partial Soulseek results are normal and
|
||||
// often good enough.
|
||||
func (s *slskd) awaitSearch(
|
||||
ctx context.Context,
|
||||
searchID string,
|
||||
) (slskdSearch, error) {
|
||||
deadline := time.Now().Add(s.searchWait)
|
||||
|
||||
var last slskdSearch
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, fmt.Errorf("%w: search cancelled", ErrSlskdTimeout)
|
||||
case <-time.After(s.searchPoll):
|
||||
}
|
||||
|
||||
var search slskdSearch
|
||||
|
||||
if err := s.client.get(
|
||||
ctx,
|
||||
"/api/v0/searches/"+searchID+"?includeResponses=true",
|
||||
&search,
|
||||
); err != nil {
|
||||
return last, err
|
||||
}
|
||||
|
||||
last = search
|
||||
|
||||
if search.IsComplete {
|
||||
return search, nil
|
||||
}
|
||||
}
|
||||
|
||||
return last, nil
|
||||
}
|
||||
|
||||
// candidatesFrom groups a search's responses into candidates.
|
||||
func (s *slskd) candidatesFrom(search slskdSearch) []Candidate {
|
||||
out := make([]Candidate, 0, len(search.Responses))
|
||||
|
||||
for _, resp := range search.Responses {
|
||||
for folder, files := range groupByFolder(resp.Files) {
|
||||
audio := 0
|
||||
|
||||
cfiles := make([]CandidateFile, 0, len(files))
|
||||
|
||||
var total int64
|
||||
|
||||
for _, f := range files {
|
||||
format, isAudio := FormatForPath(f.Filename)
|
||||
if isAudio {
|
||||
audio++
|
||||
}
|
||||
|
||||
cfiles = append(cfiles, CandidateFile{
|
||||
Path: f.Filename,
|
||||
Size: f.Size,
|
||||
Format: format,
|
||||
Bitrate: f.BitRate,
|
||||
IsAudio: isAudio,
|
||||
})
|
||||
|
||||
total += f.Size
|
||||
}
|
||||
|
||||
if audio < slskdMinFiles {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, Candidate{
|
||||
ID: "slskd:" + resp.Username + ":" + folder,
|
||||
Kind: KindSlskd,
|
||||
Protocol: ProtocolDirect,
|
||||
Title: path.Base(strings.ReplaceAll(folder, `\`, "/")),
|
||||
Origin: resp.Username,
|
||||
Files: cfiles,
|
||||
TotalSize: total,
|
||||
Health: peerHealth(resp),
|
||||
Payload: map[string]string{"username": resp.Username},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// groupByFolder buckets a peer's files by their containing directory.
|
||||
func groupByFolder(files []slskdFile) map[string][]slskdFile {
|
||||
out := map[string][]slskdFile{}
|
||||
|
||||
for _, f := range files {
|
||||
norm := strings.ReplaceAll(f.Filename, `\`, "/")
|
||||
out[path.Dir(norm)] = append(out[path.Dir(norm)], f)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// peerHealth scores how likely a peer is to actually deliver, in 0..1.
|
||||
// On Soulseek this matters more than it does for torrents: a queue of
|
||||
// 40 behind a single upload slot means the transfer starts tomorrow,
|
||||
// and that is the difference between a good candidate and a bad one no
|
||||
// matter how good the files look.
|
||||
func peerHealth(r slskdResponse) float64 {
|
||||
score := 0.35
|
||||
|
||||
if r.HasFreeUploadSlot || r.FreeUploadSlotFlag {
|
||||
score += 0.4
|
||||
}
|
||||
|
||||
switch {
|
||||
case r.QueueLength == 0:
|
||||
score += 0.15
|
||||
case r.QueueLength <= 3:
|
||||
score += 0.08
|
||||
case r.QueueLength > 20:
|
||||
score -= 0.2
|
||||
}
|
||||
|
||||
// Anything above roughly 1 MB/s is fast enough that more speed does
|
||||
// not change the experience.
|
||||
const fastEnough = 1_000_000
|
||||
|
||||
if r.UploadSpeed > 0 {
|
||||
ratio := float64(r.UploadSpeed) / fastEnough
|
||||
if ratio > 1 {
|
||||
ratio = 1
|
||||
}
|
||||
|
||||
score += 0.1 * ratio
|
||||
}
|
||||
|
||||
return clamp01(score)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Transfer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Grab enqueues a candidate's files with slskd, waits for the peer to
|
||||
// send them, then moves them out of slskd's download directory into the
|
||||
// staging directory.
|
||||
func (s *slskd) Grab(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error) {
|
||||
username := c.Payload["username"]
|
||||
if username == "" {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: candidate has no peer username", ErrSlskdTransferFailed,
|
||||
)
|
||||
}
|
||||
|
||||
wanted := make([]map[string]any, 0, len(c.Files))
|
||||
for _, f := range c.Files {
|
||||
wanted = append(wanted, map[string]any{
|
||||
"filename": f.Path,
|
||||
"size": f.Size,
|
||||
})
|
||||
}
|
||||
|
||||
if err := s.client.post(
|
||||
ctx, "/api/v0/transfers/downloads/"+username, wanted, nil,
|
||||
); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
if err := s.awaitTransfers(ctx, username, c, onProgress); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return s.collect(c, dst)
|
||||
}
|
||||
|
||||
// awaitTransfers polls until every requested file reaches a terminal
|
||||
// state. Soulseek queues are measured in hours, so the only deadline
|
||||
// is the caller's context.
|
||||
func (s *slskd) awaitTransfers(
|
||||
ctx context.Context,
|
||||
username string,
|
||||
c Candidate,
|
||||
onProgress ProgressFunc,
|
||||
) error {
|
||||
wanted := make(map[string]bool, len(c.Files))
|
||||
for _, f := range c.Files {
|
||||
wanted[f.Path] = true
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%w: transfer cancelled", ErrSlskdTimeout)
|
||||
case <-time.After(s.transferPoll):
|
||||
}
|
||||
|
||||
transfers, err := s.transfersFor(ctx, username)
|
||||
if err != nil {
|
||||
// A blip talking to the daemon should not abandon a
|
||||
// transfer that may be hours in.
|
||||
s.logger.Debug("slskd transfer poll failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
done, failed int
|
||||
current int64
|
||||
)
|
||||
|
||||
for _, t := range transfers {
|
||||
if !wanted[t.Filename] {
|
||||
continue
|
||||
}
|
||||
|
||||
current += t.BytesTransferred
|
||||
|
||||
finished, ok := t.done()
|
||||
if !finished {
|
||||
continue
|
||||
}
|
||||
|
||||
if ok {
|
||||
done++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
|
||||
if onProgress != nil {
|
||||
onProgress(Progress{
|
||||
Current: current,
|
||||
Total: c.TotalSize,
|
||||
Phase: fmt.Sprintf(
|
||||
"Transferring from %s (%d/%d)", username, done, len(wanted),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
if done+failed < len(wanted) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Some files failing is normal — a peer goes offline mid-folder.
|
||||
// Let the importer's completeness check decide whether what
|
||||
// arrived is enough, rather than discarding it here.
|
||||
if done == 0 {
|
||||
return fmt.Errorf(
|
||||
"%w: all %d files failed", ErrSlskdTransferFailed, failed,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// transfersFor returns a peer's current downloads. slskd nests
|
||||
// transfers under directories, so this flattens them.
|
||||
func (s *slskd) transfersFor(
|
||||
ctx context.Context,
|
||||
username string,
|
||||
) ([]slskdTransfer, error) {
|
||||
var raw struct {
|
||||
Directories []struct {
|
||||
Files []slskdTransfer `json:"files"`
|
||||
} `json:"directories"`
|
||||
}
|
||||
|
||||
if err := s.client.get(
|
||||
ctx, "/api/v0/transfers/downloads/"+username, &raw,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]slskdTransfer, 0, len(raw.Directories))
|
||||
for _, d := range raw.Directories {
|
||||
out = append(out, d.Files...)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// collect moves finished files out of slskd's download directory into
|
||||
// staging. slskd lays them out as <downloads>/<folder>/<file>, so each
|
||||
// wanted file is looked up by its base name under the folder slskd
|
||||
// derived from the remote path.
|
||||
func (s *slskd) collect(c Candidate, dst string) (Result, error) {
|
||||
result := Result{Dir: dst, Files: make([]string, 0, len(c.Files))}
|
||||
|
||||
for _, f := range c.Files {
|
||||
norm := strings.ReplaceAll(f.Path, `\`, "/")
|
||||
folder := path.Base(path.Dir(norm))
|
||||
base := path.Base(norm)
|
||||
|
||||
src := filepath.Join(s.downloadsPath, folder, base)
|
||||
|
||||
info, err := os.Stat(src)
|
||||
if err != nil || info.Size() == 0 {
|
||||
// Not every requested file arrives; that is expected and
|
||||
// handled by completeness scoring downstream.
|
||||
continue
|
||||
}
|
||||
|
||||
target := filepath.Join(dst, base)
|
||||
|
||||
if err := movePath(src, target); err != nil {
|
||||
return Result{}, fmt.Errorf("collect %s: %w", base, err)
|
||||
}
|
||||
|
||||
result.Files = append(result.Files, target)
|
||||
result.BytesTransferred += info.Size()
|
||||
}
|
||||
|
||||
if len(result.Files) == 0 {
|
||||
return Result{}, fmt.Errorf(
|
||||
"%w: nothing found under %s",
|
||||
ErrSlskdDownloadsPath, s.downloadsPath,
|
||||
)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,567 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// slskd tests run against an httptest server shaped like the real API.
|
||||
// No daemon, no Soulseek account, no network.
|
||||
|
||||
// slskdStub is a fake slskd daemon.
|
||||
type slskdStub struct {
|
||||
server *httptest.Server
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
// responses is what a search returns.
|
||||
responses []slskdResponse
|
||||
|
||||
// transfers is what the downloads endpoint reports, in order; the
|
||||
// last entry repeats.
|
||||
transfers [][]slskdTransfer
|
||||
pollCount int
|
||||
|
||||
// enqueued records what was requested for download.
|
||||
enqueued []map[string]any
|
||||
|
||||
// unauthorized makes every call return 401.
|
||||
unauthorized bool
|
||||
}
|
||||
|
||||
func newSlskdStub(t *testing.T) *slskdStub {
|
||||
t.Helper()
|
||||
|
||||
s := &slskdStub{}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/api/v0/application", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, map[string]any{"version": "0.21.0"})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v0/searches", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v0/searches/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodDelete {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
responses := s.responses
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, slskdSearch{
|
||||
ID: "search-1",
|
||||
IsComplete: true,
|
||||
Responses: responses,
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v0/transfers/downloads/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.reject(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodPost {
|
||||
var body []map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode enqueue body: %v", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.enqueued = body
|
||||
s.mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
idx := s.pollCount
|
||||
if idx >= len(s.transfers) {
|
||||
idx = len(s.transfers) - 1
|
||||
} else {
|
||||
s.pollCount++
|
||||
}
|
||||
|
||||
var batch []slskdTransfer
|
||||
if idx >= 0 && len(s.transfers) > 0 {
|
||||
batch = s.transfers[idx]
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{
|
||||
"directories": []map[string]any{{"files": batch}},
|
||||
})
|
||||
})
|
||||
|
||||
s.server = httptest.NewServer(mux)
|
||||
t.Cleanup(s.server.Close)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// reject enforces API-key auth like the real daemon.
|
||||
func (s *slskdStub) reject(w http.ResponseWriter, r *http.Request) bool {
|
||||
s.mu.Lock()
|
||||
unauthorized := s.unauthorized
|
||||
s.mu.Unlock()
|
||||
|
||||
if unauthorized || r.Header.Get("X-Api-Key") != "test-key" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func writeJSON(t *testing.T, w http.ResponseWriter, v any) {
|
||||
t.Helper()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||
t.Errorf("encode response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// newStubSlskd builds the provider pointed at the stub, with a real
|
||||
// temp directory standing in for slskd's downloads folder.
|
||||
func newStubSlskd(t *testing.T, stub *slskdStub) (*slskd, string) {
|
||||
t.Helper()
|
||||
|
||||
downloads := t.TempDir()
|
||||
|
||||
p, err := newSlskd(
|
||||
Config{
|
||||
ID: 1,
|
||||
Kind: KindSlskd,
|
||||
Name: "slskd",
|
||||
Enabled: true,
|
||||
Settings: map[string]string{
|
||||
"url": stub.server.URL,
|
||||
"downloadsPath": downloads,
|
||||
},
|
||||
},
|
||||
func(string) (string, error) { return "test-key", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newSlskd: %v", err)
|
||||
}
|
||||
|
||||
s, ok := p.(*slskd)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *slskd", p)
|
||||
}
|
||||
|
||||
// Real intervals are tuned for Soulseek's pace; tests only care
|
||||
// about the state machine, so run it at full speed.
|
||||
s.searchPoll = time.Millisecond
|
||||
s.searchWait = 200 * time.Millisecond
|
||||
s.transferPoll = time.Millisecond
|
||||
|
||||
return s, downloads
|
||||
}
|
||||
|
||||
func TestSlskdCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
if err := s.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlskdCheckRejectsBadKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.unauthorized = true
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
if err := s.Check(context.Background()); !errors.Is(err, ErrSlskdAuth) {
|
||||
t.Errorf("error = %v, want ErrSlskdAuth", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A downloads folder that is not readable from this machine is the
|
||||
// classic slskd-on-a-NAS misconfiguration, and must surface at
|
||||
// configuration time rather than after a long transfer.
|
||||
func TestSlskdCheckRejectsUnreadableDownloadsPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
|
||||
p, err := newSlskd(
|
||||
Config{
|
||||
ID: 1,
|
||||
Settings: map[string]string{
|
||||
"url": stub.server.URL,
|
||||
"downloadsPath": "/definitely/not/a/real/path",
|
||||
},
|
||||
},
|
||||
func(string) (string, error) { return "test-key", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newSlskd: %v", err)
|
||||
}
|
||||
|
||||
if err := p.Check(context.Background()); !errors.Is(
|
||||
err, ErrSlskdDownloadsPath,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrSlskdDownloadsPath", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Soulseek has no album concept, so candidates are built by grouping a
|
||||
// peer's files into the folders they live in.
|
||||
func TestSlskdGroupsResultsByPeerAndFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.responses = []slskdResponse{
|
||||
{
|
||||
Username: "peer-one",
|
||||
HasFreeUploadSlot: true,
|
||||
QueueLength: 0,
|
||||
UploadSpeed: 2_000_000,
|
||||
Files: []slskdFile{
|
||||
{Filename: `@@x\Music\OK Computer\01 Airbag.flac`, Size: 30_000_000},
|
||||
{Filename: `@@x\Music\OK Computer\02 Paranoid Android.flac`, Size: 40_000_000},
|
||||
{Filename: `@@x\Music\Kid A\01 Everything.flac`, Size: 30_000_000},
|
||||
{Filename: `@@x\Music\Kid A\02 Kid A.flac`, Size: 30_000_000},
|
||||
},
|
||||
},
|
||||
{
|
||||
Username: "peer-two",
|
||||
HasFreeUploadSlot: false,
|
||||
QueueLength: 40,
|
||||
Files: []slskdFile{
|
||||
{
|
||||
Filename: `\share\OK Computer [320]\01 - Airbag.mp3`,
|
||||
Size: 8_000_000,
|
||||
BitRate: 320,
|
||||
},
|
||||
{
|
||||
Filename: `\share\OK Computer [320]\02 - Paranoid Android.mp3`,
|
||||
Size: 9_000_000,
|
||||
BitRate: 320,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
got, err := s.Search(context.Background(), Request{
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
// Two folders from peer-one, one from peer-two.
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("got %d candidates, want 3", len(got))
|
||||
}
|
||||
|
||||
byOrigin := map[string]int{}
|
||||
for _, c := range got {
|
||||
byOrigin[c.Origin]++
|
||||
}
|
||||
|
||||
if byOrigin["peer-one"] != 2 {
|
||||
t.Errorf("peer-one folders = %d, want 2", byOrigin["peer-one"])
|
||||
}
|
||||
|
||||
if byOrigin["peer-two"] != 1 {
|
||||
t.Errorf("peer-two folders = %d, want 1", byOrigin["peer-two"])
|
||||
}
|
||||
}
|
||||
|
||||
// A busy peer behind a long queue is a worse bet than a free one, no
|
||||
// matter how good the files look.
|
||||
func TestSlskdPeerHealthReflectsAvailability(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
free := peerHealth(slskdResponse{
|
||||
HasFreeUploadSlot: true,
|
||||
QueueLength: 0,
|
||||
UploadSpeed: 2_000_000,
|
||||
})
|
||||
|
||||
busy := peerHealth(slskdResponse{
|
||||
HasFreeUploadSlot: false,
|
||||
QueueLength: 40,
|
||||
})
|
||||
|
||||
if free <= busy {
|
||||
t.Errorf("free peer health %f should exceed busy peer %f", free, busy)
|
||||
}
|
||||
|
||||
if free > 1 || busy < 0 {
|
||||
t.Errorf("health out of range: free=%f busy=%f", free, busy)
|
||||
}
|
||||
}
|
||||
|
||||
// Folders with almost nothing in them are Soulseek noise, not albums.
|
||||
func TestSlskdSkipsTinyFolders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.responses = []slskdResponse{{
|
||||
Username: "peer",
|
||||
Files: []slskdFile{
|
||||
{Filename: `\share\Random\one.mp3`, Size: 5_000_000},
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
got, err := s.Search(context.Background(), Request{Query: "x"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 0 {
|
||||
t.Errorf("got %d candidates, want 0 for a single-file folder", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlskdGrabCollectsFromDownloadsFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{
|
||||
{
|
||||
{
|
||||
Filename: `\share\OK Computer\01 Airbag.flac`,
|
||||
State: "InProgress",
|
||||
BytesTransferred: 100,
|
||||
},
|
||||
{
|
||||
Filename: `\share\OK Computer\02 Paranoid Android.flac`,
|
||||
State: "InProgress",
|
||||
},
|
||||
},
|
||||
{
|
||||
{
|
||||
Filename: `\share\OK Computer\01 Airbag.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
{
|
||||
Filename: `\share\OK Computer\02 Paranoid Android.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
// slskd writes into <downloads>/<folder>/<file>.
|
||||
folder := filepath.Join(downloads, "OK Computer")
|
||||
if err := os.MkdirAll(folder, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"01 Airbag.flac",
|
||||
"02 Paranoid Android.flac",
|
||||
} {
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(folder, name), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
ID: "slskd:peer:OK Computer",
|
||||
Protocol: ProtocolDirect,
|
||||
Files: []CandidateFile{
|
||||
{Path: `\share\OK Computer\01 Airbag.flac`, Size: 500, IsAudio: true},
|
||||
{Path: `\share\OK Computer\02 Paranoid Android.flac`, Size: 500, IsAudio: true},
|
||||
},
|
||||
TotalSize: 1000,
|
||||
Payload: map[string]string{"username": "peer"},
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := s.Grab(context.Background(), c, dst, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
for _, f := range got.Files {
|
||||
if !strings.HasPrefix(f, dst) {
|
||||
t.Errorf("file %s is not inside the staging dir %s", f, dst)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(f); err != nil {
|
||||
t.Errorf("collected file missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The enqueue request named the files the candidate listed.
|
||||
stub.mu.Lock()
|
||||
enqueued := len(stub.enqueued)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if enqueued != 2 {
|
||||
t.Errorf("enqueued %d files, want 2", enqueued)
|
||||
}
|
||||
}
|
||||
|
||||
// A peer that drops mid-folder is normal; partial results go forward
|
||||
// and the importer's completeness check decides.
|
||||
func TestSlskdGrabToleratesPartialFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{
|
||||
Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
{Filename: `\s\Album\02 B.flac`, State: "Completed, Errored"},
|
||||
}}
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
folder := filepath.Join(downloads, "Album")
|
||||
if err := os.MkdirAll(folder, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(folder, "01 A.flac"), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
Files: []CandidateFile{
|
||||
{Path: `\s\Album\01 A.flac`, Size: 500, IsAudio: true},
|
||||
{Path: `\s\Album\02 B.flac`, Size: 500, IsAudio: true},
|
||||
},
|
||||
Payload: map[string]string{"username": "peer"},
|
||||
}
|
||||
|
||||
got, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Errorf("collected %d files, want the 1 that succeeded", len(got.Files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlskdGrabFailsWhenEverythingFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{Filename: `\s\Album\01 A.flac`, State: "Completed, Errored"},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
c := Candidate{
|
||||
Files: []CandidateFile{{Path: `\s\Album\01 A.flac`, IsAudio: true}},
|
||||
Payload: map[string]string{"username": "peer"},
|
||||
}
|
||||
|
||||
_, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSlskdTransferFailed) {
|
||||
t.Errorf("error = %v, want ErrSlskdTransferFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlskdRequiresConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
settings map[string]string
|
||||
secrets SecretLookup
|
||||
}{
|
||||
{
|
||||
name: "no url",
|
||||
settings: map[string]string{"downloadsPath": "/tmp"},
|
||||
secrets: func(string) (string, error) { return "k", nil },
|
||||
},
|
||||
{
|
||||
name: "no downloads path",
|
||||
settings: map[string]string{"url": "http://localhost:5030"},
|
||||
secrets: func(string) (string, error) { return "k", nil },
|
||||
},
|
||||
{
|
||||
name: "no api key",
|
||||
settings: map[string]string{
|
||||
"url": "http://localhost:5030", "downloadsPath": "/tmp",
|
||||
},
|
||||
secrets: func(string) (string, error) {
|
||||
return "", ErrSecretNotFound
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newSlskd(
|
||||
Config{Settings: tt.settings}, tt.secrets, slogDiscard(),
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrNotConfigured) {
|
||||
t.Errorf("error = %v, want ErrNotConfigured", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// yt-dlp is the local-subprocess shape: no server for the user to run,
|
||||
// no credentials, but everything comes back as text from a binary whose
|
||||
// output format is not a stable contract. The defences are: pin a
|
||||
// minimum version and check it before use, ask for JSON rather than
|
||||
// parsing human output, and never build a shell command — every
|
||||
// invocation is an argv slice, so a track title containing `; rm -rf`
|
||||
// is an argument and not a command.
|
||||
|
||||
// yt-dlp provider errors.
|
||||
var (
|
||||
// ErrYtDlpMissing means the binary was not found.
|
||||
ErrYtDlpMissing = errors.New("yt-dlp was not found")
|
||||
|
||||
// ErrYtDlpTooOld means the installed version predates the output
|
||||
// format this adapter relies on.
|
||||
ErrYtDlpTooOld = errors.New("yt-dlp is too old")
|
||||
|
||||
// ErrYtDlpFailed wraps a non-zero exit.
|
||||
ErrYtDlpFailed = errors.New("yt-dlp failed")
|
||||
|
||||
// ErrUnsafeURL rejects a URL that is not plain http(s). yt-dlp
|
||||
// accepts things like file:// that must never come from a search
|
||||
// result.
|
||||
ErrUnsafeURL = errors.New("refusing to fetch a non-http URL")
|
||||
)
|
||||
|
||||
// minYtDlpVersion is the oldest release known to support the
|
||||
// --progress-template and --dump-json output this adapter parses.
|
||||
// yt-dlp versions are date-stamped, so this compares lexically.
|
||||
const minYtDlpVersion = "2023.01.01"
|
||||
|
||||
// ytSearchCount is how many results to ask for per query.
|
||||
const ytSearchCount = 5
|
||||
|
||||
// ytTrackConcurrency bounds parallel per-track searches when assembling
|
||||
// an album. YouTube throttles aggressively; three is fast enough to
|
||||
// finish inside the search timeout without tripping it.
|
||||
const ytTrackConcurrency = 3
|
||||
|
||||
func init() {
|
||||
Register(
|
||||
Descriptor{
|
||||
Kind: KindYtDlp,
|
||||
Name: "yt-dlp",
|
||||
Summary: "Download audio from YouTube, SoundCloud, Bandcamp and other sites yt-dlp supports.",
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
},
|
||||
Fields: []Field{
|
||||
{
|
||||
Key: "binary",
|
||||
Label: "yt-dlp path",
|
||||
Placeholder: "yt-dlp",
|
||||
Help: "Leave blank to find yt-dlp on your PATH.",
|
||||
Default: "yt-dlp",
|
||||
},
|
||||
{
|
||||
Key: "audioFormat",
|
||||
Label: "Audio format",
|
||||
Help: "flac, mp3, opus, m4a, or 'best' to keep the source format.",
|
||||
Default: "flac",
|
||||
},
|
||||
{
|
||||
Key: "searchPrefix",
|
||||
Label: "Search source",
|
||||
Help: "ytsearch for YouTube, ytmsearch for YouTube Music. " +
|
||||
"Defaults to ytsearch.",
|
||||
Default: "ytsearch",
|
||||
},
|
||||
},
|
||||
},
|
||||
newYtDlp,
|
||||
)
|
||||
}
|
||||
|
||||
// ytDlp is the yt-dlp provider.
|
||||
type ytDlp struct {
|
||||
info ProviderInfo
|
||||
logger *slog.Logger
|
||||
|
||||
binary string
|
||||
audioFormat string
|
||||
searchPrefix string
|
||||
}
|
||||
|
||||
// newYtDlp builds a yt-dlp provider from config.
|
||||
func newYtDlp(
|
||||
cfg Config,
|
||||
_ SecretLookup,
|
||||
logger *slog.Logger,
|
||||
) (Provider, error) {
|
||||
binary := cfg.Setting("binary", "yt-dlp")
|
||||
|
||||
resolved, err := exec.LookPath(binary)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrYtDlpMissing, binary)
|
||||
}
|
||||
|
||||
return &ytDlp{
|
||||
info: ProviderInfo{
|
||||
ID: cfg.ID,
|
||||
Kind: KindYtDlp,
|
||||
Name: cfg.Name,
|
||||
Enabled: cfg.Enabled,
|
||||
Priority: cfg.Priority,
|
||||
Caps: Caps{
|
||||
CanSearch: true,
|
||||
CanTransport: true,
|
||||
CanCancel: true,
|
||||
ReportsSize: true,
|
||||
},
|
||||
},
|
||||
logger: logger.With("provider", "yt-dlp"),
|
||||
binary: resolved,
|
||||
audioFormat: cfg.Setting("audioFormat", "flac"),
|
||||
searchPrefix: cfg.Setting("searchPrefix", "ytsearch"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Info returns the provider's identity.
|
||||
func (y *ytDlp) Info() ProviderInfo {
|
||||
return y.info
|
||||
}
|
||||
|
||||
// Close is a no-op; each invocation is its own process.
|
||||
func (y *ytDlp) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check verifies the binary runs and is new enough. Version drift is
|
||||
// yt-dlp's defining trait, so this is the difference between a clear
|
||||
// error at configuration time and garbled output at download time.
|
||||
func (y *ytDlp) Check(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
out, err := exec.CommandContext(ctx, y.binary, "--version").Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrYtDlpFailed, err)
|
||||
}
|
||||
|
||||
version := strings.TrimSpace(string(out))
|
||||
if version < minYtDlpVersion {
|
||||
return fmt.Errorf(
|
||||
"%w: found %s, need %s or newer",
|
||||
ErrYtDlpTooOld, version, minYtDlpVersion,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ytEntry is the subset of yt-dlp's --dump-json output this adapter
|
||||
// uses. yt-dlp emits far more; naming only what is needed means a
|
||||
// field being added or reordered upstream cannot break parsing.
|
||||
type ytEntry struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
WebURL string `json:"webpage_url"`
|
||||
Uploader string `json:"uploader"`
|
||||
Duration float64 `json:"duration"`
|
||||
Filesize int64 `json:"filesize_approx"`
|
||||
}
|
||||
|
||||
// link returns the entry's best usable URL.
|
||||
func (e ytEntry) link() string {
|
||||
if e.WebURL != "" {
|
||||
return e.WebURL
|
||||
}
|
||||
|
||||
return e.URL
|
||||
}
|
||||
|
||||
// Search assembles candidates. With an expected tracklist it searches
|
||||
// per track and offers the assembled album as one candidate, which is
|
||||
// how yt-dlp is actually useful for albums — a single "full album"
|
||||
// video is one file and cannot be imported as tracks. Without a
|
||||
// tracklist it falls back to returning the top individual results.
|
||||
func (y *ytDlp) Search(ctx context.Context, req Request) ([]Candidate, error) {
|
||||
if len(req.Expected) > 0 {
|
||||
c, err := y.assembleAlbum(ctx, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(c.Files) > 0 {
|
||||
return []Candidate{c}, nil
|
||||
}
|
||||
}
|
||||
|
||||
entries, err := y.search(ctx, req.SearchText(), ytSearchCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]Candidate, 0, len(entries))
|
||||
|
||||
for _, e := range entries {
|
||||
link := e.link()
|
||||
if link == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
name := sanitizePathPart(e.Title) + "." + y.extension()
|
||||
|
||||
out = append(out, Candidate{
|
||||
ID: "ytdlp:" + e.ID,
|
||||
Kind: KindYtDlp,
|
||||
Protocol: ProtocolDirect,
|
||||
Title: e.Title,
|
||||
Artist: e.Uploader,
|
||||
Origin: "yt-dlp",
|
||||
Files: []CandidateFile{{
|
||||
Path: name,
|
||||
Size: e.Filesize,
|
||||
IsAudio: true,
|
||||
}},
|
||||
TotalSize: e.Filesize,
|
||||
// yt-dlp results are always available; there is no peer to
|
||||
// be offline, so health carries no information here.
|
||||
Health: 0.75,
|
||||
Payload: map[string]string{name: link},
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// assembleAlbum searches once per expected track and builds a single
|
||||
// multi-file candidate. Tracks that find no result are left out; the
|
||||
// completeness score then reflects the gap, and the importer's
|
||||
// threshold decides whether what arrived is enough.
|
||||
func (y *ytDlp) assembleAlbum(
|
||||
ctx context.Context,
|
||||
req Request,
|
||||
) (Candidate, error) {
|
||||
type hit struct {
|
||||
index int
|
||||
entry ytEntry
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
hits []hit
|
||||
)
|
||||
|
||||
group, gctx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(ytTrackConcurrency)
|
||||
|
||||
for i, track := range req.Expected {
|
||||
group.Go(func() error {
|
||||
query := strings.TrimSpace(
|
||||
req.Artist + " " + track.Title,
|
||||
)
|
||||
|
||||
entries, err := y.search(gctx, query, 1)
|
||||
if err != nil || len(entries) == 0 {
|
||||
// One missing track is not a failed search. Recording
|
||||
// nothing lets completeness scoring speak for it.
|
||||
return nil //nolint:nilerr // partial results are expected
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
|
||||
hits = append(hits, hit{index: i, entry: entries[0]})
|
||||
|
||||
mu.Unlock()
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := group.Wait(); err != nil {
|
||||
return Candidate{}, fmt.Errorf("assemble album: %w", err)
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
ID: "ytdlp:album:" + req.ID,
|
||||
Kind: KindYtDlp,
|
||||
Protocol: ProtocolDirect,
|
||||
Title: req.Album,
|
||||
Artist: req.Artist,
|
||||
Origin: "yt-dlp (assembled per track)",
|
||||
Health: 0.75,
|
||||
Payload: map[string]string{},
|
||||
Files: make([]CandidateFile, 0, len(hits)),
|
||||
}
|
||||
|
||||
for _, h := range hits {
|
||||
track := req.Expected[h.index]
|
||||
|
||||
link := h.entry.link()
|
||||
if link == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Name the staged file after the expected track, not the video
|
||||
// title: the video is called whatever the uploader felt like,
|
||||
// and the import step matches on filename.
|
||||
name := trackToken(track.Position) + " - " +
|
||||
sanitizePathPart(track.Title) + "." + y.extension()
|
||||
|
||||
c.Files = append(c.Files, CandidateFile{
|
||||
Path: name,
|
||||
Size: h.entry.Filesize,
|
||||
IsAudio: true,
|
||||
MatchedTo: track.Position,
|
||||
})
|
||||
|
||||
c.TotalSize += h.entry.Filesize
|
||||
c.Payload[name] = link
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// search runs one yt-dlp search and decodes its JSON lines.
|
||||
func (y *ytDlp) search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
) ([]ytEntry, error) {
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// The search term is one argv element; yt-dlp parses the
|
||||
// "ytsearchN:" prefix itself. No shell is involved at any point.
|
||||
target := y.searchPrefix + strconv.Itoa(count) + ":" + query
|
||||
|
||||
args := []string{
|
||||
"--dump-json",
|
||||
"--flat-playlist",
|
||||
"--no-warnings",
|
||||
"--no-playlist",
|
||||
"--ignore-config",
|
||||
"--socket-timeout", "15",
|
||||
target,
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, y.binary, args...)
|
||||
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: search: %w", ErrYtDlpFailed, err)
|
||||
}
|
||||
|
||||
return decodeYtEntries(strings.NewReader(string(out))), nil
|
||||
}
|
||||
|
||||
// decodeYtEntries reads newline-delimited JSON, skipping lines that do
|
||||
// not parse. yt-dlp mixes warnings into stdout in some versions, and
|
||||
// one bad line must not discard the rest of the results.
|
||||
func decodeYtEntries(r io.Reader) []ytEntry {
|
||||
var out []ytEntry
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || !strings.HasPrefix(line, "{") {
|
||||
continue
|
||||
}
|
||||
|
||||
var e ytEntry
|
||||
if err := json.Unmarshal([]byte(line), &e); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if e.ID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, e)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Grab downloads each of the candidate's files into dst.
|
||||
func (y *ytDlp) Grab(
|
||||
ctx context.Context,
|
||||
c Candidate,
|
||||
dst string,
|
||||
onProgress ProgressFunc,
|
||||
) (Result, error) {
|
||||
result := Result{Dir: dst, Files: make([]string, 0, len(c.Files))}
|
||||
|
||||
for i, f := range c.Files {
|
||||
link, ok := c.Payload[f.Path]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := validateHTTPURL(link); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
if onProgress != nil {
|
||||
onProgress(Progress{
|
||||
Current: int64(i),
|
||||
Total: int64(len(c.Files)),
|
||||
Phase: fmt.Sprintf(
|
||||
"Downloading %d of %d", i+1, len(c.Files),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
path, err := y.fetchOne(ctx, link, dst, f.Path)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
result.Files = append(result.Files, path)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// fetchOne downloads a single URL to a known filename inside dst.
|
||||
func (y *ytDlp) fetchOne(
|
||||
ctx context.Context,
|
||||
link, dst, name string,
|
||||
) (string, error) {
|
||||
// Strip the extension from the output template: yt-dlp appends the
|
||||
// real one after extraction, and forcing it here produces
|
||||
// double-extensioned files.
|
||||
stem := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
template := filepath.Join(dst, stem) + ".%(ext)s"
|
||||
|
||||
args := []string{
|
||||
"--extract-audio",
|
||||
"--no-playlist",
|
||||
"--no-warnings",
|
||||
"--ignore-config",
|
||||
"--newline",
|
||||
"--no-part",
|
||||
"--socket-timeout", "30",
|
||||
"--output", template,
|
||||
}
|
||||
|
||||
if y.audioFormat != "" && y.audioFormat != "best" {
|
||||
args = append(args, "--audio-format", y.audioFormat)
|
||||
}
|
||||
|
||||
args = append(args, "--", link)
|
||||
|
||||
cmd := exec.CommandContext(ctx, y.binary, args...)
|
||||
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w: download: %w: %s",
|
||||
ErrYtDlpFailed, err, lastLine(string(out)),
|
||||
)
|
||||
}
|
||||
|
||||
// The extracted extension is whatever yt-dlp produced, so find the
|
||||
// file by stem rather than assuming.
|
||||
matches, err := filepath.Glob(filepath.Join(dst, stem) + ".*")
|
||||
if err != nil || len(matches) == 0 {
|
||||
return "", fmt.Errorf(
|
||||
"%w: no output file for %s", ErrYtDlpFailed, stem,
|
||||
)
|
||||
}
|
||||
|
||||
return matches[0], nil
|
||||
}
|
||||
|
||||
// extension returns the file extension downloads will have.
|
||||
func (y *ytDlp) extension() string {
|
||||
if y.audioFormat == "" || y.audioFormat == "best" {
|
||||
return "opus"
|
||||
}
|
||||
|
||||
return y.audioFormat
|
||||
}
|
||||
|
||||
// validateHTTPURL rejects anything that is not plain http(s). yt-dlp
|
||||
// happily accepts file:// and other schemes, and a search result is
|
||||
// untrusted input.
|
||||
func validateHTTPURL(raw string) error {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s", ErrUnsafeURL, raw)
|
||||
}
|
||||
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("%w: %s", ErrUnsafeURL, raw)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastLine returns the final non-empty line of output, which is where
|
||||
// yt-dlp puts its error message.
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
if line := strings.TrimSpace(lines[i]); line != "" {
|
||||
return line
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// yt-dlp tests drive a stub shell script rather than the real binary:
|
||||
// the adapter's contract is "what argv do we build and what do we do
|
||||
// with the output", and a stub tests exactly that without a network,
|
||||
// a YouTube account, or a 40MB dependency.
|
||||
|
||||
// stubYtDlp writes an executable script that echoes the given stdout
|
||||
// and returns it as a provider config binary path.
|
||||
func stubYtDlp(t *testing.T, script string) string {
|
||||
t.Helper()
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("stub binary test uses a shell script")
|
||||
}
|
||||
|
||||
path := filepath.Join(t.TempDir(), "yt-dlp")
|
||||
|
||||
if err := os.WriteFile(
|
||||
path, []byte("#!/bin/sh\n"+script), 0o700,
|
||||
); err != nil {
|
||||
t.Fatalf("write stub: %v", err)
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// newStubYtDlp builds the provider over a stub binary.
|
||||
func newStubYtDlp(t *testing.T, script string) *ytDlp {
|
||||
t.Helper()
|
||||
|
||||
p, err := newYtDlp(
|
||||
Config{
|
||||
ID: 1,
|
||||
Kind: KindYtDlp,
|
||||
Name: "yt-dlp",
|
||||
Enabled: true,
|
||||
Priority: 50,
|
||||
Settings: map[string]string{
|
||||
"binary": stubYtDlp(t, script),
|
||||
"audioFormat": "flac",
|
||||
},
|
||||
},
|
||||
nil,
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newYtDlp: %v", err)
|
||||
}
|
||||
|
||||
y, ok := p.(*ytDlp)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *ytDlp", p)
|
||||
}
|
||||
|
||||
return y
|
||||
}
|
||||
|
||||
func TestYtDlpCheckVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("recent version passes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `echo "2024.08.06"`)
|
||||
|
||||
if err := y.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("old version is rejected", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `echo "2021.01.01"`)
|
||||
|
||||
if err := y.Check(context.Background()); !errors.Is(
|
||||
err, ErrYtDlpTooOld,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrYtDlpTooOld", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-zero exit is reported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `exit 1`)
|
||||
|
||||
if err := y.Check(context.Background()); !errors.Is(
|
||||
err, ErrYtDlpFailed,
|
||||
) {
|
||||
t.Errorf("error = %v, want ErrYtDlpFailed", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestYtDlpMissingBinary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newYtDlp(
|
||||
Config{Settings: map[string]string{
|
||||
"binary": "definitely-not-a-real-binary-xyzzy",
|
||||
}},
|
||||
nil,
|
||||
slogDiscard(),
|
||||
)
|
||||
|
||||
if !errors.Is(err, ErrYtDlpMissing) {
|
||||
t.Errorf("error = %v, want ErrYtDlpMissing", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYtDlpSearchParsesJSONLines(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `
|
||||
cat <<'EOF'
|
||||
{"id":"aaa","title":"Airbag","webpage_url":"https://example.com/aaa","uploader":"Radiohead","duration":284,"filesize_approx":5000000}
|
||||
{"id":"bbb","title":"Paranoid Android","webpage_url":"https://example.com/bbb","uploader":"Radiohead","duration":383}
|
||||
EOF
|
||||
`)
|
||||
|
||||
got, err := y.Search(context.Background(), Request{
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2", len(got))
|
||||
}
|
||||
|
||||
if got[0].Title != "Airbag" {
|
||||
t.Errorf("title = %q, want Airbag", got[0].Title)
|
||||
}
|
||||
|
||||
if got[0].Protocol != ProtocolDirect {
|
||||
t.Errorf("protocol = %q, want direct", got[0].Protocol)
|
||||
}
|
||||
|
||||
if len(got[0].Files) != 1 {
|
||||
t.Fatalf("got %d files, want 1", len(got[0].Files))
|
||||
}
|
||||
|
||||
link := got[0].Payload[got[0].Files[0].Path]
|
||||
if link != "https://example.com/aaa" {
|
||||
t.Errorf("payload url = %q, want the webpage_url", link)
|
||||
}
|
||||
}
|
||||
|
||||
// Warning text mixed into stdout must not discard valid results.
|
||||
func TestYtDlpSearchSkipsUnparseableLines(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `
|
||||
cat <<'EOF'
|
||||
WARNING: something happened
|
||||
{"id":"aaa","title":"Airbag","webpage_url":"https://example.com/aaa"}
|
||||
not json at all
|
||||
{"id":"bbb","title":"Lucky","webpage_url":"https://example.com/bbb"}
|
||||
EOF
|
||||
`)
|
||||
|
||||
got, err := y.Search(context.Background(), Request{Query: "radiohead"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d candidates, want 2 valid ones", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// With a tracklist the adapter assembles an album from per-track
|
||||
// searches, because a single "full album" video cannot be imported as
|
||||
// separate tracks.
|
||||
func TestYtDlpAssemblesAlbumFromTracklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `
|
||||
echo '{"id":"x","title":"whatever the uploader called it","webpage_url":"https://example.com/x","filesize_approx":4000000}'
|
||||
`)
|
||||
|
||||
req := Request{
|
||||
ID: "req-1",
|
||||
ReleaseMBID: "mbid-1",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
Expected: []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
{Position: 3, Title: "Subterranean Homesick Alien"},
|
||||
},
|
||||
}
|
||||
|
||||
got, err := y.Search(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1 assembled album", len(got))
|
||||
}
|
||||
|
||||
album := got[0]
|
||||
|
||||
if len(album.Files) != 3 {
|
||||
t.Fatalf("got %d files, want 3", len(album.Files))
|
||||
}
|
||||
|
||||
// Files are named after the expected tracks, not the video titles,
|
||||
// because the import step matches on filename.
|
||||
names := make([]string, 0, len(album.Files))
|
||||
for _, f := range album.Files {
|
||||
names = append(names, f.Path)
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
"01 - Airbag.flac",
|
||||
"02 - Paranoid Android.flac",
|
||||
"03 - Subterranean Homesick Alien.flac",
|
||||
} {
|
||||
if !containsString(names, want) {
|
||||
t.Errorf("missing %q in %v", want, names)
|
||||
}
|
||||
}
|
||||
|
||||
if album.TotalSize != 12_000_000 {
|
||||
t.Errorf("total size = %d, want 12000000", album.TotalSize)
|
||||
}
|
||||
}
|
||||
|
||||
// A track with no search result is left out, so completeness scoring
|
||||
// can speak for the gap instead of the search failing outright.
|
||||
func TestYtDlpAssembleToleratesMissingTracks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `
|
||||
case "$*" in
|
||||
*Airbag*) echo '{"id":"a","title":"Airbag","webpage_url":"https://example.com/a"}' ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
`)
|
||||
|
||||
got, err := y.Search(context.Background(), Request{
|
||||
ID: "req-1",
|
||||
ReleaseMBID: "mbid-1",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
Expected: []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want 1", len(got))
|
||||
}
|
||||
|
||||
if len(got[0].Files) != 1 {
|
||||
t.Errorf("got %d files, want just the one that was found", len(got[0].Files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestYtDlpGrabWritesIntoStaging(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The stub writes a file at whatever --output stem it is given,
|
||||
// mimicking yt-dlp's post-extraction naming.
|
||||
y := newStubYtDlp(t, `
|
||||
out=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--output) out="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
target=$(printf '%s' "$out" | sed 's/%(ext)s/flac/')
|
||||
printf 'audio' > "$target"
|
||||
`)
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
c := Candidate{
|
||||
ID: "ytdlp:album:req-1",
|
||||
Protocol: ProtocolDirect,
|
||||
Files: []CandidateFile{
|
||||
{Path: "01 - Airbag.flac", IsAudio: true},
|
||||
},
|
||||
Payload: map[string]string{
|
||||
"01 - Airbag.flac": "https://example.com/a",
|
||||
},
|
||||
}
|
||||
|
||||
got, err := y.Grab(context.Background(), c, dst, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Fatalf("got %d files, want 1", len(got.Files))
|
||||
}
|
||||
|
||||
if _, err := os.Stat(got.Files[0]); err != nil {
|
||||
t.Errorf("downloaded file missing: %v", err)
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(got.Files[0], ".flac") {
|
||||
t.Errorf("file = %s, want a .flac", got.Files[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A search result is untrusted input, and yt-dlp accepts schemes that
|
||||
// would read the local filesystem.
|
||||
func TestYtDlpGrabRejectsNonHTTPURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
y := newStubYtDlp(t, `exit 0`)
|
||||
|
||||
c := Candidate{
|
||||
Files: []CandidateFile{{Path: "x.flac", IsAudio: true}},
|
||||
Payload: map[string]string{"x.flac": "file:///etc/passwd"},
|
||||
}
|
||||
|
||||
_, err := y.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrUnsafeURL) {
|
||||
t.Errorf("error = %v, want ErrUnsafeURL", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateHTTPURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
url string
|
||||
wantErr bool
|
||||
}{
|
||||
{"https://example.com/a", false},
|
||||
{"http://example.com/a", false},
|
||||
{"file:///etc/passwd", true},
|
||||
{"ftp://example.com/a", true},
|
||||
{"javascript:alert(1)", true},
|
||||
{"://nonsense", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.url, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := validateHTTPURL(tt.url)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("validateHTTPURL(%q) error = %v, wantErr %v",
|
||||
tt.url, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(haystack []string, needle string) bool {
|
||||
for _, h := range haystack {
|
||||
if h == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"yellowjacket/backend/autotag"
|
||||
)
|
||||
|
||||
// Ranking keeps two questions apart:
|
||||
//
|
||||
// match — is this the release the user asked for?
|
||||
// quality — is it a good copy of it?
|
||||
//
|
||||
// They are reported separately because they fail differently and trade
|
||||
// off against each other: a flawless FLAC of the wrong album is useless,
|
||||
// a 128kbps rip of the right one is merely disappointing, and only the
|
||||
// user knows which they will accept. A single blended number cannot be
|
||||
// explained, and the review UI has to explain itself.
|
||||
|
||||
// Ranking weights. Match dominates, because a wrong album at any
|
||||
// bitrate is a failed download.
|
||||
const (
|
||||
weightMatch = 0.72
|
||||
weightQuality = 0.28
|
||||
)
|
||||
|
||||
// Match sub-weights.
|
||||
const (
|
||||
weightTitleFit = 0.40
|
||||
weightCompleteness = 0.30
|
||||
weightAlbumFit = 0.18
|
||||
weightArtistFit = 0.12
|
||||
)
|
||||
|
||||
// Quality sub-weights.
|
||||
const (
|
||||
weightFormat = 0.45
|
||||
weightBitrate = 0.25
|
||||
weightHealth = 0.20
|
||||
weightPriority = 0.10
|
||||
)
|
||||
|
||||
// unanchoredCap bounds the match score of a free-text request. Without
|
||||
// an MBID there is no tracklist to be right about, so a confident-
|
||||
// looking score would be a lie — and auto-pick keys off this.
|
||||
const unanchoredCap = 0.65
|
||||
|
||||
// Score fills a candidate's Match, Quality and Score fields.
|
||||
func Score(req Request, c Candidate, priority int) Candidate {
|
||||
c.Files = AnnotateFiles(c.Files)
|
||||
|
||||
audio := c.AudioFiles()
|
||||
|
||||
matched, titleFit := matchFiles(audio, req.Expected)
|
||||
|
||||
// Write the alignment back so the picker can show which file maps
|
||||
// to which track.
|
||||
c.Files = mergeMatched(c.Files, matched)
|
||||
|
||||
c.Match = scoreMatch(req, c, audio, titleFit)
|
||||
c.Quality = scoreQuality(c, audio, priority)
|
||||
|
||||
c.Score = weightMatch*c.Match.Overall + weightQuality*c.Quality.Overall
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// scoreMatch answers whether this candidate is the requested release.
|
||||
func scoreMatch(
|
||||
req Request,
|
||||
c Candidate,
|
||||
audio []CandidateFile,
|
||||
titleFit float64,
|
||||
) MatchScore {
|
||||
m := MatchScore{
|
||||
Anchored: req.Anchored(),
|
||||
TitleFit: titleFit,
|
||||
}
|
||||
|
||||
m.Completeness = completeness(len(audio), len(req.Expected))
|
||||
|
||||
// The candidate's own title, and the folder its files sit in, are
|
||||
// two independent guesses at the album name. Take the better one:
|
||||
// providers vary in which is meaningful.
|
||||
folder := ""
|
||||
if len(audio) > 0 {
|
||||
folder = ParsePath(audio[0].Path).Folder
|
||||
}
|
||||
|
||||
m.AlbumFit = math.Max(
|
||||
autotag.TitleSimilarity(req.Album, c.Title),
|
||||
autotag.TitleSimilarity(req.Album, folder),
|
||||
)
|
||||
|
||||
m.ArtistFit = artistFit(req.Artist, c)
|
||||
|
||||
// With no expected tracklist there is no title signal at all, so
|
||||
// redistribute its weight onto the album/artist evidence rather
|
||||
// than scoring every free-text result as half-wrong.
|
||||
if len(req.Expected) == 0 {
|
||||
m.Overall = 0.55*m.AlbumFit + 0.45*m.ArtistFit
|
||||
} else {
|
||||
m.Overall = weightTitleFit*m.TitleFit +
|
||||
weightCompleteness*m.Completeness +
|
||||
weightAlbumFit*m.AlbumFit +
|
||||
weightArtistFit*m.ArtistFit
|
||||
}
|
||||
|
||||
if !m.Anchored {
|
||||
m.Overall = math.Min(m.Overall, unanchoredCap)
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// artistFit compares the requested artist against the candidate's
|
||||
// artist field, its title, and the path of its first audio file, taking
|
||||
// the best. Providers disagree about where the artist name lands.
|
||||
func artistFit(want string, c Candidate) float64 {
|
||||
if strings.TrimSpace(want) == "" {
|
||||
return 0.5
|
||||
}
|
||||
|
||||
best := autotag.TitleSimilarity(want, c.Artist)
|
||||
|
||||
if s := autotag.TitleSimilarity(want, c.Title); s > best {
|
||||
best = s
|
||||
}
|
||||
|
||||
// A path containing the artist name anywhere is weak but real
|
||||
// evidence — most folders are "Artist - Album".
|
||||
norm := autotag.Normalize(want)
|
||||
if norm != "" {
|
||||
for _, f := range c.Files {
|
||||
if strings.Contains(autotag.Normalize(f.Path), norm) {
|
||||
if best < 0.8 {
|
||||
best = 0.8
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
// completeness scores audio file count against the expected track
|
||||
// count. Extra files are penalized far more gently than missing ones:
|
||||
// a folder with bonus tracks or a stray intro is still the album, while
|
||||
// a folder missing half the tracks is not.
|
||||
func completeness(got, want int) float64 {
|
||||
if want == 0 {
|
||||
if got > 0 {
|
||||
return 0.5
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
if got == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
if got >= want {
|
||||
extra := float64(got-want) / float64(want)
|
||||
|
||||
return math.Max(0.75, 1.0-0.25*extra)
|
||||
}
|
||||
|
||||
return float64(got) / float64(want)
|
||||
}
|
||||
|
||||
// scoreQuality answers whether this is a good copy.
|
||||
func scoreQuality(
|
||||
c Candidate,
|
||||
audio []CandidateFile,
|
||||
priority int,
|
||||
) QualityScore {
|
||||
q := QualityScore{
|
||||
Health: clamp01(c.Health),
|
||||
Priority: clamp01(float64(priority) / 100.0),
|
||||
}
|
||||
|
||||
if len(audio) == 0 {
|
||||
return q
|
||||
}
|
||||
|
||||
// Format: score the worst file, not the average. A folder that is
|
||||
// mostly FLAC with three MP3s transcoded in is a worse copy than
|
||||
// its average suggests, and that is exactly what the user would
|
||||
// want flagged.
|
||||
worst := 1.0
|
||||
first := audio[0].Format
|
||||
|
||||
for _, f := range audio {
|
||||
if r := formatRank(f.Format); r < worst {
|
||||
worst = r
|
||||
}
|
||||
|
||||
if f.Format != first {
|
||||
q.Mixed = true
|
||||
}
|
||||
}
|
||||
|
||||
q.FormatRank = worst
|
||||
q.Bitrate = bitrateScore(audio)
|
||||
|
||||
q.Overall = weightFormat*q.FormatRank +
|
||||
weightBitrate*q.Bitrate +
|
||||
weightHealth*q.Health +
|
||||
weightPriority*q.Priority
|
||||
|
||||
if q.Mixed {
|
||||
q.Overall *= 0.9
|
||||
}
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
// formatRank scores a format on its own terms, in 0..1. Lossless
|
||||
// formats top out; lossy formats sit below and are further separated by
|
||||
// bitrate. Formats the player cannot decode are penalized but not
|
||||
// zeroed — the user may be acquiring them deliberately.
|
||||
func formatRank(f Format) float64 {
|
||||
base := 0.0
|
||||
|
||||
switch f {
|
||||
case FormatFLAC:
|
||||
base = 1.0
|
||||
case FormatALAC:
|
||||
base = 0.95
|
||||
case FormatWAV:
|
||||
base = 0.85 // lossless, but untaggable and huge
|
||||
case FormatMP3:
|
||||
base = 0.6
|
||||
case FormatAAC, FormatOpus:
|
||||
base = 0.6
|
||||
case FormatOGG:
|
||||
base = 0.55
|
||||
case FormatWMA:
|
||||
base = 0.3
|
||||
case FormatUnknown:
|
||||
base = 0.2
|
||||
default:
|
||||
base = 0.2
|
||||
}
|
||||
|
||||
if !f.Supported() && f != FormatUnknown {
|
||||
base *= 0.8
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
// bitrateScore maps the mean stated bitrate of lossy files onto 0..1.
|
||||
// Lossless files score 1.0 and are excluded from the mean. Returns a
|
||||
// neutral 0.5 when nothing states a bitrate, which is the common case
|
||||
// for Soulseek results.
|
||||
func bitrateScore(audio []CandidateFile) float64 {
|
||||
var (
|
||||
sum float64
|
||||
count int
|
||||
)
|
||||
|
||||
for _, f := range audio {
|
||||
if f.Format.Lossless() {
|
||||
sum += 1.0
|
||||
count++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if f.Bitrate == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
sum += lossyBitrateScore(f.Bitrate)
|
||||
count++
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return 0.5
|
||||
}
|
||||
|
||||
return sum / float64(count)
|
||||
}
|
||||
|
||||
// lossyBitrateScore maps kbps onto 0..1 with the knee where it belongs
|
||||
// perceptually: the gap between 128 and 192 matters much more than the
|
||||
// gap between 256 and 320.
|
||||
func lossyBitrateScore(kbps int) float64 {
|
||||
switch {
|
||||
case kbps >= 320:
|
||||
return 1.0
|
||||
case kbps >= 256:
|
||||
return 0.9
|
||||
case kbps >= 224:
|
||||
return 0.82
|
||||
case kbps >= 192:
|
||||
return 0.72
|
||||
case kbps >= 160:
|
||||
return 0.55
|
||||
case kbps >= 128:
|
||||
return 0.4
|
||||
case kbps >= 96:
|
||||
return 0.2
|
||||
default:
|
||||
return 0.1
|
||||
}
|
||||
}
|
||||
|
||||
// Rank scores every candidate and returns them best-first. Ties break
|
||||
// on match, then on provider priority, then on file count, so the order
|
||||
// is stable across runs rather than map-iteration dependent.
|
||||
func Rank(
|
||||
req Request,
|
||||
candidates []Candidate,
|
||||
priority func(providerID int64) int,
|
||||
) []Candidate {
|
||||
out := make([]Candidate, 0, len(candidates))
|
||||
|
||||
for _, c := range candidates {
|
||||
p := 50
|
||||
if priority != nil {
|
||||
p = priority(c.ProviderID)
|
||||
}
|
||||
|
||||
out = append(out, Score(req, c, p))
|
||||
}
|
||||
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Score != out[j].Score {
|
||||
return out[i].Score > out[j].Score
|
||||
}
|
||||
|
||||
if out[i].Match.Overall != out[j].Match.Overall {
|
||||
return out[i].Match.Overall > out[j].Match.Overall
|
||||
}
|
||||
|
||||
if out[i].Quality.Priority != out[j].Quality.Priority {
|
||||
return out[i].Quality.Priority > out[j].Quality.Priority
|
||||
}
|
||||
|
||||
return len(out[i].Files) > len(out[j].Files)
|
||||
})
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// AutoPickable reports whether a ranked list has a clear enough winner
|
||||
// to grab without asking. It demands an anchored request, a high match,
|
||||
// decent quality, and daylight between first and second place — if two
|
||||
// candidates are close, the choice is the user's.
|
||||
func AutoPickable(req Request, ranked []Candidate) bool {
|
||||
const (
|
||||
minMatch = 0.85
|
||||
minQuality = 0.5
|
||||
minLead = 0.08
|
||||
)
|
||||
|
||||
if !req.Anchored() || len(ranked) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// An anchor with no tracklist behind it is an anchor in name only:
|
||||
// the match score then rests on album and artist text alone, which
|
||||
// is exactly the evidence a wrong-album candidate also has. This
|
||||
// matters most for the wanted list, where nobody is watching.
|
||||
if len(req.Expected) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
best := ranked[0]
|
||||
if best.Match.Overall < minMatch || best.Quality.Overall < minQuality {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(ranked) > 1 && best.Score-ranked[1].Score < minLead {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// mergeMatched copies MatchedTo assignments from the audio-only slice
|
||||
// back onto the full file list.
|
||||
func mergeMatched(all, matched []CandidateFile) []CandidateFile {
|
||||
if len(matched) == 0 {
|
||||
return all
|
||||
}
|
||||
|
||||
byPath := make(map[string]int, len(matched))
|
||||
for _, m := range matched {
|
||||
byPath[m.Path] = m.MatchedTo
|
||||
}
|
||||
|
||||
out := make([]CandidateFile, len(all))
|
||||
copy(out, all)
|
||||
|
||||
for i := range out {
|
||||
if pos, ok := byPath[out[i].Path]; ok {
|
||||
out[i].MatchedTo = pos
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// clamp01 bounds a value to 0..1.
|
||||
func clamp01(v float64) float64 {
|
||||
return math.Max(0, math.Min(1, v))
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package download
|
||||
|
||||
import "testing"
|
||||
|
||||
// okComputer is the reference request used across ranking tests.
|
||||
func okComputer() Request {
|
||||
return Request{
|
||||
ReleaseMBID: "mbid-ok-computer",
|
||||
Artist: "Radiohead",
|
||||
Album: "OK Computer",
|
||||
Expected: []ExpectedTrack{
|
||||
{Position: 1, Title: "Airbag"},
|
||||
{Position: 2, Title: "Paranoid Android"},
|
||||
{Position: 3, Title: "Subterranean Homesick Alien"},
|
||||
{Position: 4, Title: "Exit Music (For a Film)"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// candidateFor builds a candidate whose files follow "NN - Title.ext".
|
||||
func candidateFor(id string, titles []string, ext string, size int64) Candidate {
|
||||
files := make([]CandidateFile, 0, len(titles))
|
||||
|
||||
for i, tt := range titles {
|
||||
files = append(files, CandidateFile{
|
||||
Path: "Radiohead - OK Computer/" +
|
||||
trackToken(i+1) + " - " + tt + ext,
|
||||
Size: size,
|
||||
})
|
||||
}
|
||||
|
||||
return Candidate{
|
||||
ID: id,
|
||||
Protocol: ProtocolDirect,
|
||||
Title: "Radiohead - OK Computer",
|
||||
Artist: "Radiohead",
|
||||
Files: files,
|
||||
Health: 0.5,
|
||||
}
|
||||
}
|
||||
|
||||
func allTitles() []string {
|
||||
return []string{
|
||||
"Airbag",
|
||||
"Paranoid Android",
|
||||
"Subterranean Homesick Alien",
|
||||
"Exit Music (For a Film)",
|
||||
}
|
||||
}
|
||||
|
||||
// The headline behaviour: a well-matched FLAC beats a well-matched
|
||||
// 128kbps MP3, but a mismatched FLAC loses to both.
|
||||
func TestRankPrefersQualityAtEqualMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
|
||||
flac := candidateFor("flac", allTitles(), ".flac", 30_000_000)
|
||||
mp3 := candidateFor("mp3", allTitles(), ".mp3", 3_000_000)
|
||||
|
||||
for i := range mp3.Files {
|
||||
mp3.Files[i].Bitrate = 128
|
||||
}
|
||||
|
||||
ranked := Rank(req, []Candidate{mp3, flac}, nil)
|
||||
|
||||
if ranked[0].ID != "flac" {
|
||||
t.Fatalf("winner = %s, want flac", ranked[0].ID)
|
||||
}
|
||||
|
||||
if ranked[0].Match.Overall < 0.9 {
|
||||
t.Errorf("flac match = %f, want high", ranked[0].Match.Overall)
|
||||
}
|
||||
|
||||
if ranked[0].Quality.Overall <= ranked[1].Quality.Overall {
|
||||
t.Errorf(
|
||||
"flac quality %f should exceed mp3 %f",
|
||||
ranked[0].Quality.Overall, ranked[1].Quality.Overall,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankMatchDominatesQuality(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
|
||||
// Right album, poor bitrate.
|
||||
right := candidateFor("right", allTitles(), ".mp3", 2_000_000)
|
||||
for i := range right.Files {
|
||||
right.Files[i].Bitrate = 128
|
||||
}
|
||||
|
||||
// Wrong album, pristine FLAC.
|
||||
wrong := candidateFor("wrong", []string{
|
||||
"Enter Sandman", "Sad But True", "Holier Than Thou", "The Unforgiven",
|
||||
}, ".flac", 30_000_000)
|
||||
wrong.Title = "Metallica - Metallica"
|
||||
wrong.Artist = "Metallica"
|
||||
|
||||
for i := range wrong.Files {
|
||||
wrong.Files[i].Path = "Metallica - Metallica/" +
|
||||
trackToken(i+1) + " - x.flac"
|
||||
}
|
||||
|
||||
ranked := Rank(req, []Candidate{wrong, right}, nil)
|
||||
|
||||
if ranked[0].ID != "right" {
|
||||
t.Fatalf(
|
||||
"winner = %s (score %f vs %f), want the correctly matched album",
|
||||
ranked[0].ID, ranked[0].Score, ranked[1].Score,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncompleteCandidateScoresLower(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
|
||||
full := candidateFor("full", allTitles(), ".flac", 30_000_000)
|
||||
partial := candidateFor("partial", allTitles()[:2], ".flac", 30_000_000)
|
||||
|
||||
ranked := Rank(req, []Candidate{partial, full}, nil)
|
||||
|
||||
if ranked[0].ID != "full" {
|
||||
t.Fatalf("winner = %s, want full", ranked[0].ID)
|
||||
}
|
||||
|
||||
if ranked[1].Match.Completeness >= ranked[0].Match.Completeness {
|
||||
t.Errorf(
|
||||
"partial completeness %f should be below full %f",
|
||||
ranked[1].Match.Completeness, ranked[0].Match.Completeness,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMixedFormatIsPenalized(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
|
||||
clean := candidateFor("clean", allTitles(), ".flac", 30_000_000)
|
||||
|
||||
mixed := candidateFor("mixed", allTitles(), ".flac", 30_000_000)
|
||||
mixed.Files[2].Path = "Radiohead - OK Computer/03 - x.mp3"
|
||||
mixed.Files[2].Format = FormatUnknown
|
||||
|
||||
ranked := Rank(req, []Candidate{mixed, clean}, nil)
|
||||
|
||||
var mixedScore QualityScore
|
||||
|
||||
for _, c := range ranked {
|
||||
if c.ID == "mixed" {
|
||||
mixedScore = c.Quality
|
||||
}
|
||||
}
|
||||
|
||||
if !mixedScore.Mixed {
|
||||
t.Error("mixed-format candidate not flagged")
|
||||
}
|
||||
|
||||
if ranked[0].ID != "clean" {
|
||||
t.Errorf("winner = %s, want clean", ranked[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Without an MBID there is no tracklist to be right about, so the match
|
||||
// score must not look confident regardless of how good the strings are.
|
||||
func TestUnanchoredMatchIsCapped(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := Request{Artist: "Radiohead", Album: "OK Computer"}
|
||||
c := candidateFor("c", allTitles(), ".flac", 30_000_000)
|
||||
|
||||
scored := Score(req, c, 50)
|
||||
|
||||
if scored.Match.Anchored {
|
||||
t.Error("free-text request reported as anchored")
|
||||
}
|
||||
|
||||
if scored.Match.Overall > unanchoredCap {
|
||||
t.Errorf(
|
||||
"unanchored match = %f, want <= %f",
|
||||
scored.Match.Overall, unanchoredCap,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoPickableRequiresAnchorAndLead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
best := Score(req, candidateFor("a", allTitles(), ".flac", 30_000_000), 50)
|
||||
|
||||
t.Run("clear winner is auto-pickable", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
weak := Score(
|
||||
req,
|
||||
candidateFor("b", allTitles()[:2], ".mp3", 1_000_000),
|
||||
50,
|
||||
)
|
||||
|
||||
if !AutoPickable(req, []Candidate{best, weak}) {
|
||||
t.Errorf(
|
||||
"want auto-pickable: match %f quality %f lead %f",
|
||||
best.Match.Overall, best.Quality.Overall, best.Score-weak.Score,
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("two close candidates are not", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
twin := best
|
||||
twin.ID = "twin"
|
||||
|
||||
if AutoPickable(req, []Candidate{best, twin}) {
|
||||
t.Error("identical candidates must not auto-pick")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("free text is never auto-pickable", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
free := Request{Artist: "Radiohead", Album: "OK Computer"}
|
||||
|
||||
if AutoPickable(free, []Candidate{best}) {
|
||||
t.Error("unanchored request must not auto-pick")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty list is not", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if AutoPickable(req, nil) {
|
||||
t.Error("empty candidate list must not auto-pick")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCompleteness(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
got int
|
||||
want int
|
||||
minScore float64
|
||||
maxScore float64
|
||||
}{
|
||||
{"exact", 10, 10, 1.0, 1.0},
|
||||
{"half missing", 5, 10, 0.49, 0.51},
|
||||
{"one bonus track", 11, 10, 0.95, 1.0},
|
||||
{"double", 20, 10, 0.74, 0.76},
|
||||
{"nothing", 0, 10, 0, 0},
|
||||
{"no expectation", 5, 0, 0.5, 0.5},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := completeness(tt.got, tt.want)
|
||||
if got < tt.minScore || got > tt.maxScore {
|
||||
t.Errorf(
|
||||
"completeness(%d, %d) = %f, want in [%f, %f]",
|
||||
tt.got, tt.want, got, tt.minScore, tt.maxScore,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPriorityBreaksTies(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := okComputer()
|
||||
|
||||
a := candidateFor("a", allTitles(), ".flac", 30_000_000)
|
||||
a.ProviderID = 1
|
||||
|
||||
b := candidateFor("b", allTitles(), ".flac", 30_000_000)
|
||||
b.ProviderID = 2
|
||||
|
||||
priority := func(id int64) int {
|
||||
if id == 2 {
|
||||
return 90
|
||||
}
|
||||
|
||||
return 10
|
||||
}
|
||||
|
||||
ranked := Rank(req, []Candidate{a, b}, priority)
|
||||
|
||||
if ranked[0].ID != "b" {
|
||||
t.Errorf("winner = %s, want b (higher provider priority)", ranked[0].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,746 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The reconciler is the only thing that turns wants into downloads.
|
||||
//
|
||||
// It runs on a slow loop rather than reacting to events, because
|
||||
// everything it cares about changes slowly: a release the user wants
|
||||
// appears on a source days or weeks after they asked, an artist puts
|
||||
// out an album once a year, and a library gains files by scan rather
|
||||
// than by notification. A loop that wakes a few times a day is
|
||||
// sufficient for all of it and costs nothing, where an event-driven
|
||||
// design here would mean subscribing to three subsystems to learn the
|
||||
// same facts later anyway.
|
||||
//
|
||||
// Each pass does four things, in this order and for this reason:
|
||||
//
|
||||
// 1. Expand artist subscriptions into per-album wants, so step 2 sees
|
||||
// them this pass rather than next.
|
||||
// 2. Retire wants the library already owns — including ones the user
|
||||
// satisfied by other means, which is why ownership is checked
|
||||
// rather than assumed from our own downloads.
|
||||
// 3. Push the list to clients that keep their own (Lidarr), so the
|
||||
// user's intent is expressed in both places.
|
||||
// 4. Attempt a bounded batch of due wants.
|
||||
//
|
||||
// Nothing here fails a want. A want that cannot be found gets an
|
||||
// attempt recorded and a longer backoff, and stays exactly as wanted as
|
||||
// it was.
|
||||
|
||||
// CatalogPort is what the reconciler needs to know about the world of
|
||||
// music, kept narrow so the download package does not depend on the
|
||||
// explore package (and so tests can answer these four questions from a
|
||||
// map).
|
||||
type CatalogPort interface {
|
||||
// ReleaseGroupsForArtist returns an artist's discography. An empty
|
||||
// result is not an error: the explore index fetches discographies
|
||||
// lazily, so the honest answer is often "not yet".
|
||||
ReleaseGroupsForArtist(
|
||||
ctx context.Context,
|
||||
artistMBID string,
|
||||
) ([]CatalogItem, error)
|
||||
|
||||
// Tracklist resolves a release group or release to the tracks it
|
||||
// should contain. This is what makes a want's download safe to
|
||||
// complete unattended, so a want with no tracklist is never
|
||||
// auto-grabbed.
|
||||
Tracklist(
|
||||
ctx context.Context,
|
||||
entity Entity,
|
||||
mbid string,
|
||||
) ([]ExpectedTrack, error)
|
||||
|
||||
// Owns reports whether the library already has the thing an MBID
|
||||
// names.
|
||||
Owns(ctx context.Context, entity Entity, mbid string) (bool, error)
|
||||
|
||||
// Describe fills in display text for a want the user added by MBID
|
||||
// alone. Best-effort: an unknown MBID returns false and the want
|
||||
// is still perfectly valid.
|
||||
Describe(
|
||||
ctx context.Context,
|
||||
entity Entity,
|
||||
mbid string,
|
||||
) (CatalogItem, bool)
|
||||
}
|
||||
|
||||
// CatalogItem is one thing the catalog knows about, in the download
|
||||
// package's own terms.
|
||||
type CatalogItem struct {
|
||||
MBID string
|
||||
Title string
|
||||
Artist string
|
||||
ArtistMBID string
|
||||
|
||||
// PrimaryType is the MusicBrainz release-group type ("Album",
|
||||
// "Single", "EP").
|
||||
PrimaryType string
|
||||
|
||||
// SecondaryTypes carries "Compilation", "Live", "Remix" and
|
||||
// friends. Their presence is what an artist want's default scope
|
||||
// filters out.
|
||||
SecondaryTypes []string
|
||||
|
||||
// FirstReleaseDate is a MusicBrainz partial date: "1997",
|
||||
// "1997-04", or "1997-04-22".
|
||||
FirstReleaseDate string
|
||||
|
||||
InLibrary bool
|
||||
}
|
||||
|
||||
// Reconciler defaults.
|
||||
const (
|
||||
// defaultReconcileInterval is how often the wanted list is worked.
|
||||
// Four times a day is far more often than new music appears and far
|
||||
// less often than any provider would object to.
|
||||
defaultReconcileInterval = 6 * time.Hour
|
||||
|
||||
// startupDelay lets the app finish starting — library scan, explore
|
||||
// index, provider construction — before the first pass. A wanted
|
||||
// list worked against an index that has not loaded yet would record
|
||||
// a pile of pointless attempts.
|
||||
startupDelay = 3 * time.Minute
|
||||
|
||||
// maxExpandPerArtist bounds how many child wants one artist
|
||||
// subscription creates in a single pass, so switching an artist to
|
||||
// full-discography scope does not enqueue four hundred albums at
|
||||
// once. The remainder is picked up next pass.
|
||||
maxExpandPerArtist = 40
|
||||
)
|
||||
|
||||
// Reconciler works the wanted list.
|
||||
type Reconciler struct {
|
||||
logger *slog.Logger
|
||||
store *Store
|
||||
manager *Manager
|
||||
catalog CatalogPort
|
||||
|
||||
interval time.Duration
|
||||
batch int
|
||||
|
||||
// now is injectable so tests can drive backoff without waiting.
|
||||
now func() time.Time
|
||||
|
||||
// trigger is a nudge for an out-of-band pass, buffered to one
|
||||
// because more than one pending "run now" is the same as one.
|
||||
trigger chan struct{}
|
||||
|
||||
// onChange fires after any pass that altered the list, so the UI
|
||||
// can refresh without polling.
|
||||
onChange func()
|
||||
|
||||
stopOnce sync.Once
|
||||
stop chan struct{}
|
||||
|
||||
// runMu serializes passes: two reconcilers racing would search for
|
||||
// the same want twice.
|
||||
runMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewReconciler builds a reconciler. catalog may be nil, in which case
|
||||
// the wanted list still stores and lists wants but never acts on them —
|
||||
// which is the right behaviour when the explore index is unavailable.
|
||||
func NewReconciler(
|
||||
logger *slog.Logger,
|
||||
store *Store,
|
||||
manager *Manager,
|
||||
catalog CatalogPort,
|
||||
) *Reconciler {
|
||||
return &Reconciler{
|
||||
logger: logger,
|
||||
store: store,
|
||||
manager: manager,
|
||||
catalog: catalog,
|
||||
interval: defaultReconcileInterval,
|
||||
batch: defaultDueBatch,
|
||||
now: time.Now,
|
||||
trigger: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// SetInterval overrides the pass interval.
|
||||
func (r *Reconciler) SetInterval(d time.Duration) {
|
||||
if d > 0 {
|
||||
r.interval = d
|
||||
}
|
||||
}
|
||||
|
||||
// SetBatch overrides how many wants one pass attempts.
|
||||
func (r *Reconciler) SetBatch(n int) {
|
||||
if n > 0 {
|
||||
r.batch = n
|
||||
}
|
||||
}
|
||||
|
||||
// SetOnChange registers a callback fired after a pass that changed the
|
||||
// list.
|
||||
func (r *Reconciler) SetOnChange(fn func()) {
|
||||
r.onChange = fn
|
||||
}
|
||||
|
||||
// Start runs the reconcile loop until ctx is done or Stop is called.
|
||||
func (r *Reconciler) Start(ctx context.Context) {
|
||||
go r.loop(ctx)
|
||||
}
|
||||
|
||||
// Stop ends the loop.
|
||||
func (r *Reconciler) Stop() {
|
||||
r.stopOnce.Do(func() { close(r.stop) })
|
||||
}
|
||||
|
||||
// Trigger asks for a pass as soon as possible without blocking the
|
||||
// caller. Used when the user adds a want and expects something to
|
||||
// happen.
|
||||
func (r *Reconciler) Trigger() {
|
||||
select {
|
||||
case r.trigger <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// loop is the reconcile timer.
|
||||
func (r *Reconciler) loop(ctx context.Context) {
|
||||
first := time.NewTimer(startupDelay)
|
||||
defer first.Stop()
|
||||
|
||||
ticker := time.NewTicker(r.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-r.stop:
|
||||
return
|
||||
case <-first.C:
|
||||
case <-ticker.C:
|
||||
case <-r.trigger:
|
||||
}
|
||||
|
||||
if _, err := r.RunOnce(ctx); err != nil {
|
||||
r.logger.Warn("wanted list reconcile failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Summary reports what a pass did, for logging and for the UI.
|
||||
type Summary struct {
|
||||
// Expanded is how many child wants artist subscriptions produced.
|
||||
Expanded int `json:"expanded"`
|
||||
|
||||
// Satisfied is how many wants the library turned out to own.
|
||||
Satisfied int `json:"satisfied"`
|
||||
|
||||
// Attempted is how many wants were searched for.
|
||||
Attempted int `json:"attempted"`
|
||||
|
||||
// Started is how many of those found a clear enough winner to
|
||||
// download unattended.
|
||||
Started int `json:"started"`
|
||||
|
||||
// Synced is how many wants were pushed to an external list.
|
||||
Synced int `json:"synced"`
|
||||
}
|
||||
|
||||
// changed reports whether the pass altered anything worth refreshing
|
||||
// the UI for.
|
||||
func (s Summary) changed() bool {
|
||||
return s.Expanded > 0 || s.Satisfied > 0 || s.Started > 0
|
||||
}
|
||||
|
||||
// RunOnce works the wanted list once. It is safe to call directly, and
|
||||
// the "search now" button does.
|
||||
func (r *Reconciler) RunOnce(ctx context.Context) (Summary, error) {
|
||||
r.runMu.Lock()
|
||||
defer r.runMu.Unlock()
|
||||
|
||||
var summary Summary
|
||||
|
||||
if r.catalog == nil {
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
expanded, err := r.expandArtists(ctx)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
|
||||
summary.Expanded = expanded
|
||||
|
||||
satisfied, err := r.retireOwned(ctx)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
|
||||
summary.Satisfied = satisfied
|
||||
|
||||
summary.Synced = r.syncExternalLists(ctx)
|
||||
|
||||
attempted, started, err := r.attemptDue(ctx)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
|
||||
summary.Attempted = attempted
|
||||
summary.Started = started
|
||||
|
||||
r.logger.Info(
|
||||
"reconciled wanted list",
|
||||
"expanded", summary.Expanded,
|
||||
"satisfied", summary.Satisfied,
|
||||
"attempted", summary.Attempted,
|
||||
"started", summary.Started,
|
||||
"synced", summary.Synced,
|
||||
)
|
||||
|
||||
if summary.changed() && r.onChange != nil {
|
||||
r.onChange()
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Artist expansion
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// expandArtists turns artist subscriptions into per-album wants.
|
||||
//
|
||||
// The expansion is idempotent: child wants are upserted on (mbid,
|
||||
// library), so re-running adds only what is genuinely new. That is
|
||||
// what makes an artist want a standing subscription rather than a
|
||||
// one-time queue-filling operation — an album released next year gets
|
||||
// picked up by the same code path that ran today.
|
||||
func (r *Reconciler) expandArtists(ctx context.Context) (int, error) {
|
||||
artists, err := r.store.ListArtistWants(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
created := 0
|
||||
|
||||
for _, artist := range artists {
|
||||
n, err := r.expandArtist(ctx, artist)
|
||||
if err != nil {
|
||||
// One artist whose discography will not resolve must not
|
||||
// stop the rest of the list.
|
||||
r.logger.Warn(
|
||||
"could not expand artist want",
|
||||
"artist", artist.Label(),
|
||||
"mbid", artist.MBID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
created += n
|
||||
}
|
||||
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// expandArtist expands one subscription.
|
||||
func (r *Reconciler) expandArtist(ctx context.Context, artist Want) (int, error) {
|
||||
groups, err := r.catalog.ReleaseGroupsForArtist(ctx, artist.MBID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
created := 0
|
||||
|
||||
for _, rg := range groups {
|
||||
if created >= maxExpandPerArtist {
|
||||
break
|
||||
}
|
||||
|
||||
if !wantsReleaseGroup(artist, rg) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Existence is checked before inserting rather than relying on
|
||||
// the upsert, because "how many albums are new this pass" is
|
||||
// the number the UI reports and an upsert cannot tell an insert
|
||||
// from a no-op. It also means a want the user pinned by hand
|
||||
// is never quietly reparented under the artist.
|
||||
if _, exists, err := r.store.FindWant(
|
||||
ctx, rg.MBID, artist.LibraryID,
|
||||
); err != nil || exists {
|
||||
continue
|
||||
}
|
||||
|
||||
credit := rg.Artist
|
||||
if credit == "" {
|
||||
credit = artist.Artist
|
||||
}
|
||||
|
||||
if _, err := r.store.AddWant(ctx, Want{
|
||||
MBID: rg.MBID,
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: artist.LibraryID,
|
||||
Artist: credit,
|
||||
Title: rg.Title,
|
||||
ParentID: artist.ID,
|
||||
}); err != nil {
|
||||
r.logger.Warn(
|
||||
"could not add derived want",
|
||||
"release_group", rg.MBID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
created++
|
||||
}
|
||||
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// wantsReleaseGroup applies an artist subscription's filters to one
|
||||
// release group.
|
||||
func wantsReleaseGroup(artist Want, rg CatalogItem) bool {
|
||||
if rg.MBID == "" || rg.InLibrary {
|
||||
return false
|
||||
}
|
||||
|
||||
if !artist.Secondary && len(rg.SecondaryTypes) > 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
if artist.Scope == ScopeAll {
|
||||
return true
|
||||
}
|
||||
|
||||
// ScopeFuture: only releases the artist put out after the user
|
||||
// subscribed. A partial MusicBrainz date is compared as a string,
|
||||
// which sorts correctly for ISO dates and treats a bare year as the
|
||||
// first of January — the conservative reading, since a release
|
||||
// dated only "2026" against a subscription made in March 2026
|
||||
// should not be assumed to be new.
|
||||
return releaseDateAfter(rg.FirstReleaseDate, artist.CreatedAt)
|
||||
}
|
||||
|
||||
// releaseDateAfter compares a MusicBrainz partial date against a
|
||||
// timestamp. An unknown or unparseable date is treated as not-after,
|
||||
// because a release with no date is almost always an old one.
|
||||
func releaseDateAfter(date string, since time.Time) bool {
|
||||
date = strings.TrimSpace(date)
|
||||
if date == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Pad a partial date to a full one so string comparison works:
|
||||
// "1997" becomes "1997-01-01", "1997-04" becomes "1997-04-01".
|
||||
switch len(date) {
|
||||
case 4:
|
||||
date += "-01-01"
|
||||
case 7: //nolint:mnd // length of "YYYY-MM"
|
||||
date += "-01"
|
||||
}
|
||||
|
||||
return date > since.Format(time.DateOnly)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Retiring what the library already has
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// retireOwned satisfies wants the library turns out to own.
|
||||
//
|
||||
// Ownership is asked of the library rather than inferred from our own
|
||||
// completed downloads on purpose: the user may have bought the album,
|
||||
// ripped their CD, or copied it in from another machine, and a wanted
|
||||
// list that keeps hunting for music already sitting on disk is worse
|
||||
// than no wanted list at all.
|
||||
func (r *Reconciler) retireOwned(ctx context.Context) (int, error) {
|
||||
wants, err := r.store.ListWants(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
satisfied := 0
|
||||
|
||||
for _, w := range wants {
|
||||
if w.State != WantStateWanted || w.Entity.Expands() {
|
||||
continue
|
||||
}
|
||||
|
||||
owned, err := r.catalog.Owns(ctx, w.Entity, w.MBID)
|
||||
if err != nil {
|
||||
r.logger.Debug(
|
||||
"ownership check failed", "want", w.MBID, "error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if !owned {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.store.SatisfyWant(ctx, w.ID); err != nil {
|
||||
r.logger.Warn("could not satisfy want", "want", w.ID, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
satisfied++
|
||||
}
|
||||
|
||||
return satisfied, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Attempting downloads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// attemptDue searches for a bounded batch of due wants and grabs the
|
||||
// ones with a clear winner.
|
||||
func (r *Reconciler) attemptDue(ctx context.Context) (attempted, started int, err error) {
|
||||
due, err := r.store.ListDueWants(ctx, r.batch)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
for _, w := range due {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return attempted, started, nil
|
||||
default:
|
||||
}
|
||||
|
||||
attempted++
|
||||
|
||||
ok, reason := r.attempt(ctx, w)
|
||||
if ok {
|
||||
started++
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.store.RecordAttempt(
|
||||
ctx, w.ID, w.Attempts, reason,
|
||||
); err != nil {
|
||||
r.logger.Warn(
|
||||
"could not record want attempt", "want", w.ID, "error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return attempted, started, nil
|
||||
}
|
||||
|
||||
// tracklistFor resolves what a want should contain, which is the
|
||||
// evidence an unattended download is checked against.
|
||||
//
|
||||
// A track want is its own tracklist: one entry, built from the title
|
||||
// the want already carries. That single expected title is what lets
|
||||
// filename matching score a track download at all — without it a
|
||||
// request for one song would be scored as an album with no tracks and
|
||||
// could never clear the auto-pick bar.
|
||||
func (r *Reconciler) tracklistFor(
|
||||
ctx context.Context,
|
||||
w Want,
|
||||
) ([]ExpectedTrack, error) {
|
||||
if w.Entity != EntityRecording {
|
||||
return r.catalog.Tracklist(ctx, w.Entity, w.MBID)
|
||||
}
|
||||
|
||||
if w.Title == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return []ExpectedTrack{{
|
||||
Position: 1,
|
||||
Title: w.Title,
|
||||
Artist: w.Artist,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// attempt tries one want. It returns false with a human-readable
|
||||
// reason rather than an error, because none of the ways this does not
|
||||
// work out are failures: no providers configured yet, nothing on any
|
||||
// source, or nothing good enough to take without asking are all just
|
||||
// "not today".
|
||||
func (r *Reconciler) attempt(ctx context.Context, w Want) (bool, string) {
|
||||
expected, err := r.tracklistFor(ctx, w)
|
||||
if err != nil || len(expected) == 0 {
|
||||
// Without a tracklist an unattended grab has nothing to verify
|
||||
// itself against, so this want waits rather than guessing. The
|
||||
// tracklist usually arrives on its own once the explore index
|
||||
// fetches the release.
|
||||
return false, "waiting for the tracklist to resolve"
|
||||
}
|
||||
|
||||
req := w.ToRequest(newID())
|
||||
req.Expected = expected
|
||||
|
||||
if req.Artist == "" || req.Album == "" {
|
||||
if item, ok := r.catalog.Describe(ctx, w.Entity, w.MBID); ok {
|
||||
if req.Artist == "" {
|
||||
req.Artist = item.Artist
|
||||
}
|
||||
|
||||
if req.Album == "" {
|
||||
req.Album = item.Title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
started, reason, err := r.manager.Attempt(ctx, req)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNoProviders) {
|
||||
return false, "no download clients are enabled"
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrNoCandidates) {
|
||||
return false, "no source has it yet"
|
||||
}
|
||||
|
||||
return false, err.Error()
|
||||
}
|
||||
|
||||
return started, reason
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// External list sync
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// syncExternalLists pushes wants to providers that keep a persistent
|
||||
// list of their own.
|
||||
//
|
||||
// The sync is one-directional by design. Two systems that both accept
|
||||
// edits to the same list need conflict resolution, and the honest
|
||||
// version of that here is "whichever the user touched last", which is
|
||||
// not something we can observe. So this app's list is the source of
|
||||
// truth and the external one is a projection of it — with the single
|
||||
// exception of ImportExternal below, which the user runs deliberately.
|
||||
func (r *Reconciler) syncExternalLists(ctx context.Context) int {
|
||||
listers := r.manager.listers()
|
||||
if len(listers) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
wants, err := r.store.ListWants(ctx)
|
||||
if err != nil {
|
||||
r.logger.Warn("could not list wants for sync", "error", err)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
synced := 0
|
||||
|
||||
for _, w := range wants {
|
||||
if w.State != WantStateWanted {
|
||||
continue
|
||||
}
|
||||
|
||||
external := w.ExternalIDs
|
||||
if external == nil {
|
||||
external = map[string]string{}
|
||||
}
|
||||
|
||||
changed := false
|
||||
|
||||
for id, l := range listers {
|
||||
key := strconv.FormatInt(id, 10)
|
||||
if _, done := external[key]; done {
|
||||
continue
|
||||
}
|
||||
|
||||
externalID, err := l.PushWant(ctx, w)
|
||||
if err != nil {
|
||||
r.logger.Debug(
|
||||
"could not push want to external list",
|
||||
"want", w.MBID,
|
||||
"provider", id,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if externalID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
external[key] = externalID
|
||||
changed = true
|
||||
synced++
|
||||
}
|
||||
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.store.SetWantExternalIDs(ctx, w.ID, external); err != nil {
|
||||
r.logger.Warn(
|
||||
"could not record external want ids", "want", w.ID, "error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return synced
|
||||
}
|
||||
|
||||
// ImportExternal pulls an external manager's own list into the wanted
|
||||
// list. This is the one place data flows the other way, and it is a
|
||||
// deliberate user action ("import my monitored Lidarr artists") rather
|
||||
// than part of the loop, because silently adopting whatever another
|
||||
// system is monitoring is not something to do behind the user's back.
|
||||
func (r *Reconciler) ImportExternal(
|
||||
ctx context.Context,
|
||||
providerID int64,
|
||||
libraryID int64,
|
||||
) (int, error) {
|
||||
listers := r.manager.listers()
|
||||
|
||||
l, ok := listers[providerID]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf(
|
||||
"%w: provider %d keeps no list", ErrUnsupported, providerID,
|
||||
)
|
||||
}
|
||||
|
||||
external, err := l.ListWants(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list external wants: %w", err)
|
||||
}
|
||||
|
||||
imported := 0
|
||||
|
||||
for _, w := range external {
|
||||
w.LibraryID = libraryID
|
||||
|
||||
if _, err := r.store.AddWant(ctx, w); err != nil {
|
||||
r.logger.Warn(
|
||||
"could not import external want", "mbid", w.MBID, "error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
imported++
|
||||
}
|
||||
|
||||
if imported > 0 && r.onChange != nil {
|
||||
r.onChange()
|
||||
}
|
||||
|
||||
r.Trigger()
|
||||
|
||||
return imported, nil
|
||||
}
|
||||
@@ -0,0 +1,497 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeCatalog answers the reconciler's four questions from maps, so the
|
||||
// wanted list can be tested without an explore index.
|
||||
type fakeCatalog struct {
|
||||
mu sync.Mutex
|
||||
|
||||
// discographies maps artist MBID to release groups.
|
||||
discographies map[string][]CatalogItem
|
||||
|
||||
// tracklists maps an MBID to what it should contain.
|
||||
tracklists map[string][]ExpectedTrack
|
||||
|
||||
// owned is the set of MBIDs the library has.
|
||||
owned map[string]bool
|
||||
|
||||
// discographyErr is returned by ReleaseGroupsForArtist when set.
|
||||
discographyErr error
|
||||
}
|
||||
|
||||
func newFakeCatalog() *fakeCatalog {
|
||||
return &fakeCatalog{
|
||||
discographies: map[string][]CatalogItem{},
|
||||
tracklists: map[string][]ExpectedTrack{},
|
||||
owned: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *fakeCatalog) ReleaseGroupsForArtist(
|
||||
_ context.Context,
|
||||
artistMBID string,
|
||||
) ([]CatalogItem, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.discographyErr != nil {
|
||||
return nil, c.discographyErr
|
||||
}
|
||||
|
||||
return c.discographies[artistMBID], nil
|
||||
}
|
||||
|
||||
func (c *fakeCatalog) Tracklist(
|
||||
_ context.Context,
|
||||
_ Entity,
|
||||
mbid string,
|
||||
) ([]ExpectedTrack, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.tracklists[mbid], nil
|
||||
}
|
||||
|
||||
func (c *fakeCatalog) Owns(
|
||||
_ context.Context,
|
||||
_ Entity,
|
||||
mbid string,
|
||||
) (bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.owned[mbid], nil
|
||||
}
|
||||
|
||||
func (c *fakeCatalog) Describe(
|
||||
_ context.Context,
|
||||
_ Entity,
|
||||
_ string,
|
||||
) (CatalogItem, bool) {
|
||||
return CatalogItem{}, false
|
||||
}
|
||||
|
||||
// reconcileFixture is a manager fixture plus a wanted list over it.
|
||||
type reconcileFixture struct {
|
||||
managerFixture
|
||||
|
||||
catalog *fakeCatalog
|
||||
reconciler *Reconciler
|
||||
}
|
||||
|
||||
func newReconcileFixture(t *testing.T) reconcileFixture {
|
||||
t.Helper()
|
||||
|
||||
mf := newManagerFixture(t)
|
||||
cat := newFakeCatalog()
|
||||
|
||||
r := NewReconciler(slogDiscard(), mf.store, mf.manager, cat)
|
||||
|
||||
return reconcileFixture{managerFixture: mf, catalog: cat, reconciler: r}
|
||||
}
|
||||
|
||||
// An artist subscription becomes one want per album, and re-running
|
||||
// adds nothing — which is what makes it a subscription rather than a
|
||||
// one-time queue fill.
|
||||
func TestExpandArtistIsIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
f.catalog.discographies["artist-1"] = []CatalogItem{
|
||||
{MBID: "rg-1", Title: "First", FirstReleaseDate: "2030-01-01"},
|
||||
{MBID: "rg-2", Title: "Second", FirstReleaseDate: "2030-06-01"},
|
||||
}
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "artist-1",
|
||||
Entity: EntityArtist,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
Scope: ScopeAll,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
first, err := f.reconciler.expandArtists(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("expandArtists: %v", err)
|
||||
}
|
||||
|
||||
if first != 2 {
|
||||
t.Fatalf("first pass created %d wants, want 2", first)
|
||||
}
|
||||
|
||||
second, err := f.reconciler.expandArtists(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("expandArtists again: %v", err)
|
||||
}
|
||||
|
||||
if second != 0 {
|
||||
t.Errorf("second pass created %d wants, want 0", second)
|
||||
}
|
||||
|
||||
// A new album appearing later is picked up by the same pass.
|
||||
f.catalog.mu.Lock()
|
||||
f.catalog.discographies["artist-1"] = append(
|
||||
f.catalog.discographies["artist-1"],
|
||||
CatalogItem{MBID: "rg-3", Title: "Third", FirstReleaseDate: "2031-01-01"},
|
||||
)
|
||||
f.catalog.mu.Unlock()
|
||||
|
||||
third, err := f.reconciler.expandArtists(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("expandArtists third: %v", err)
|
||||
}
|
||||
|
||||
if third != 1 {
|
||||
t.Errorf("third pass created %d wants, want 1", third)
|
||||
}
|
||||
}
|
||||
|
||||
// A default artist subscription takes new releases only, so subscribing
|
||||
// does not silently queue a back catalogue.
|
||||
func TestExpandArtistFutureScopeSkipsBackCatalogue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
f.catalog.discographies["artist-1"] = []CatalogItem{
|
||||
{MBID: "rg-old", Title: "Old", FirstReleaseDate: "1997-04-22"},
|
||||
{MBID: "rg-new", Title: "New", FirstReleaseDate: "2099-01-01"},
|
||||
}
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "artist-1",
|
||||
Entity: EntityArtist,
|
||||
LibraryID: 1,
|
||||
Scope: ScopeFuture,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
if _, err := f.reconciler.expandArtists(ctx); err != nil {
|
||||
t.Fatalf("expandArtists: %v", err)
|
||||
}
|
||||
|
||||
wants, err := f.store.ListWants(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListWants: %v", err)
|
||||
}
|
||||
|
||||
for _, w := range wants {
|
||||
if w.MBID == "rg-old" {
|
||||
t.Error("future scope queued a back-catalogue album")
|
||||
}
|
||||
}
|
||||
|
||||
if len(wants) != 2 {
|
||||
t.Errorf("got %d wants (artist + new album), want 2", len(wants))
|
||||
}
|
||||
}
|
||||
|
||||
// One artist whose discography will not resolve must not stop the rest
|
||||
// of the list being expanded.
|
||||
func TestExpandArtistToleratesCatalogFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
f.catalog.discographyErr = errors.New("index not ready") //nolint:err113 // test
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "artist-1",
|
||||
Entity: EntityArtist,
|
||||
LibraryID: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
created, err := f.reconciler.expandArtists(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("expandArtists returned an error for one bad artist: %v", err)
|
||||
}
|
||||
|
||||
if created != 0 {
|
||||
t.Errorf("created %d wants from a failing catalog, want 0", created)
|
||||
}
|
||||
}
|
||||
|
||||
// Something the library already owns is retired, however it got there.
|
||||
func TestRetireOwnedSatisfiesWants(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
id, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
f.catalog.owned["rg-1"] = true
|
||||
|
||||
n, err := f.reconciler.retireOwned(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("retireOwned: %v", err)
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
t.Fatalf("retired %d wants, want 1", n)
|
||||
}
|
||||
|
||||
w, err := f.store.GetWant(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetWant: %v", err)
|
||||
}
|
||||
|
||||
if w.State != WantStateSatisfied {
|
||||
t.Errorf("state = %q, want satisfied", w.State)
|
||||
}
|
||||
}
|
||||
|
||||
// The end-to-end case: a want with a clear winner downloads without
|
||||
// anyone watching, and is satisfied when the files land.
|
||||
func TestReconcileDownloadsAndSatisfies(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
provider := fakeWithAlbum(1, "source", ".flac")
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
id, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
Title: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
f.catalog.tracklists["rg-1"] = fourTrackRequest().Expected
|
||||
|
||||
summary, err := f.reconciler.RunOnce(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if summary.Attempted != 1 || summary.Started != 1 {
|
||||
t.Fatalf(
|
||||
"attempted=%d started=%d, want 1 and 1 (last error: see want)",
|
||||
summary.Attempted, summary.Started,
|
||||
)
|
||||
}
|
||||
|
||||
waitFor(t, func() bool {
|
||||
w, err := f.store.GetWant(ctx, id)
|
||||
|
||||
return err == nil && w.State == WantStateSatisfied
|
||||
}, "want was never satisfied after its download completed")
|
||||
|
||||
if provider.GrabCalls != 1 {
|
||||
t.Errorf("grab calls = %d, want 1", provider.GrabCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing good enough is not a failure. The want stays wanted, gains
|
||||
// an attempt and a reason, and leaves no request row behind.
|
||||
func TestReconcileKeepsWantingWhenNothingIsGoodEnough(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A provider that finds only an unrelated album: enough to return
|
||||
// candidates, nowhere near enough to auto-pick.
|
||||
provider := NewFakeProvider(1, "weak", Caps{CanSearch: true, CanTransport: true})
|
||||
provider.Candidates = []Candidate{candidateFor(
|
||||
"weak-1", []string{"Something Else Entirely"}, ".mp3", 3_000_000,
|
||||
)}
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
id, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
Title: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
f.catalog.tracklists["rg-1"] = fourTrackRequest().Expected
|
||||
|
||||
summary, err := f.reconciler.RunOnce(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if summary.Started != 0 {
|
||||
t.Fatalf("started %d downloads, want 0", summary.Started)
|
||||
}
|
||||
|
||||
w, err := f.store.GetWant(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetWant: %v", err)
|
||||
}
|
||||
|
||||
if w.State != WantStateWanted {
|
||||
t.Errorf("state = %q, want it still wanted", w.State)
|
||||
}
|
||||
|
||||
if w.Attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1", w.Attempts)
|
||||
}
|
||||
|
||||
if w.LastError == "" {
|
||||
t.Error("no reason was recorded for the user")
|
||||
}
|
||||
|
||||
if !w.NextTryAt.After(time.Now()) {
|
||||
t.Errorf("next try at %v, want it in the future", w.NextTryAt)
|
||||
}
|
||||
|
||||
// The whole point of Attempt over Start: an unsuccessful pass
|
||||
// leaves no request row to clutter the downloads list.
|
||||
requests, err := f.store.ListRequests(ctx, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("ListRequests: %v", err)
|
||||
}
|
||||
|
||||
if len(requests) != 0 {
|
||||
t.Errorf("got %d request rows from a fruitless pass, want 0", len(requests))
|
||||
}
|
||||
}
|
||||
|
||||
// A want with no resolvable tracklist waits rather than guessing.
|
||||
func TestReconcileWaitsWithoutTracklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
provider := fakeWithAlbum(1, "source", ".flac")
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
Title: "OK Computer",
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
summary, err := f.reconciler.RunOnce(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if summary.Started != 0 {
|
||||
t.Errorf("started %d downloads with no tracklist, want 0", summary.Started)
|
||||
}
|
||||
|
||||
if provider.SearchCalls != 0 {
|
||||
t.Errorf(
|
||||
"searched %d times with no tracklist to verify against, want 0",
|
||||
provider.SearchCalls,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Artist subscriptions are never attempted as downloads: they expand.
|
||||
func TestArtistWantsAreNeverDue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "artist-1",
|
||||
Entity: EntityArtist,
|
||||
LibraryID: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
due, err := f.store.ListDueWants(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDueWants: %v", err)
|
||||
}
|
||||
|
||||
if len(due) != 0 {
|
||||
t.Errorf("got %d due wants, want 0 — artists expand, not download", len(due))
|
||||
}
|
||||
}
|
||||
|
||||
// A pass attempts at most its batch size, so a large list is worked
|
||||
// through steadily rather than in one flood.
|
||||
func TestReconcileRespectsBatchSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
f.reconciler.SetBatch(2)
|
||||
|
||||
for _, mbid := range []string{"rg-1", "rg-2", "rg-3", "rg-4"} {
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: mbid,
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Title: "Album " + mbid,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
f.catalog.tracklists[mbid] = fourTrackRequest().Expected
|
||||
}
|
||||
|
||||
summary, err := f.reconciler.RunOnce(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if summary.Attempted != 2 {
|
||||
t.Errorf("attempted %d wants, want 2 (the batch size)", summary.Attempted)
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor polls a condition, failing the test if it never holds. Used
|
||||
// where the pipeline hands work to a goroutine.
|
||||
func waitFor(t *testing.T, cond func() bool, msg string) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
t.Fatal(msg)
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"yellowjacket/backend/system"
|
||||
)
|
||||
|
||||
// Provider credentials — slskd API keys, Lidarr tokens, qBittorrent
|
||||
// passwords — must not sit in the TOML config, which is world-readable
|
||||
// by default and gets pasted into bug reports.
|
||||
//
|
||||
// They live in a separate 0600 JSON file in the user data directory.
|
||||
// That is deliberately not encryption: a key stored beside the data it
|
||||
// unlocks protects nothing, and pretending otherwise is worse than
|
||||
// being clear about it. What the file mode does buy is protection from
|
||||
// other local users and from the config file being shared casually.
|
||||
//
|
||||
// An OS keyring backend (libsecret / Keychain / DPAPI) is the right
|
||||
// long-term answer and the Store interface exists so it can be added
|
||||
// without touching any provider.
|
||||
|
||||
// ErrSecretNotFound is returned when a named secret has never been set.
|
||||
var ErrSecretNotFound = errors.New("secret not found")
|
||||
|
||||
// secretsFileName is the store's file inside the user data directory.
|
||||
const secretsFileName = "download-secrets.json"
|
||||
|
||||
// secretsFileMode is owner read/write only.
|
||||
const secretsFileMode = 0o600
|
||||
|
||||
// SecretStore holds provider credentials.
|
||||
type SecretStore interface {
|
||||
// Get returns the secret for a provider's named field.
|
||||
Get(providerID int64, name string) (string, error)
|
||||
|
||||
// Set stores a secret. An empty value deletes it.
|
||||
Set(providerID int64, name, value string) error
|
||||
|
||||
// DeleteProvider removes every secret belonging to a provider.
|
||||
DeleteProvider(providerID int64) error
|
||||
}
|
||||
|
||||
// fileSecretStore is the default SecretStore: a 0600 JSON file.
|
||||
type fileSecretStore struct {
|
||||
path string
|
||||
|
||||
mu sync.RWMutex
|
||||
loaded bool
|
||||
data map[string]string
|
||||
}
|
||||
|
||||
// NewFileSecretStore returns a SecretStore backed by a 0600 file in the
|
||||
// user data directory.
|
||||
func NewFileSecretStore() (SecretStore, error) {
|
||||
dir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve user data dir: %w", err)
|
||||
}
|
||||
|
||||
return &fileSecretStore{
|
||||
path: filepath.Join(dir, secretsFileName),
|
||||
data: map[string]string{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewFileSecretStoreAt returns a store backed by an explicit path.
|
||||
// Tests use this; production goes through NewFileSecretStore.
|
||||
func NewFileSecretStoreAt(path string) SecretStore {
|
||||
return &fileSecretStore{path: path, data: map[string]string{}}
|
||||
}
|
||||
|
||||
// secretKey namespaces a secret by provider so two slskd instances do
|
||||
// not share credentials.
|
||||
func secretKey(providerID int64, name string) string {
|
||||
return strconv.FormatInt(providerID, 10) + ":" + name
|
||||
}
|
||||
|
||||
// load reads the file once. A missing file is an empty store, not an
|
||||
// error — nothing has been configured yet.
|
||||
func (s *fileSecretStore) load() error {
|
||||
if s.loaded {
|
||||
return nil
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(s.path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
s.data = map[string]string{}
|
||||
s.loaded = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("read secrets file: %w", err)
|
||||
}
|
||||
|
||||
data := map[string]string{}
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
return fmt.Errorf("parse secrets file: %w", err)
|
||||
}
|
||||
|
||||
s.data = data
|
||||
s.loaded = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// save writes the file atomically with restrictive permissions.
|
||||
func (s *fileSecretStore) save() error {
|
||||
raw, err := json.Marshal(s.data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode secrets: %w", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
|
||||
return fmt.Errorf("create secrets dir: %w", err)
|
||||
}
|
||||
|
||||
// Write to a temp file in the same directory, chmod before the
|
||||
// rename, so the secret is never briefly world-readable.
|
||||
tmp := s.path + ".tmp"
|
||||
|
||||
if err := os.WriteFile(tmp, raw, secretsFileMode); err != nil {
|
||||
return fmt.Errorf("write secrets file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Chmod(tmp, secretsFileMode); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
|
||||
return fmt.Errorf("chmod secrets file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
|
||||
return fmt.Errorf("replace secrets file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns a stored secret.
|
||||
func (s *fileSecretStore) Get(providerID int64, name string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if err := s.load(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
v, ok := s.data[secretKey(providerID, name)]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// Set stores or clears a secret.
|
||||
func (s *fileSecretStore) Set(providerID int64, name, value string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if err := s.load(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key := secretKey(providerID, name)
|
||||
|
||||
if value == "" {
|
||||
delete(s.data, key)
|
||||
} else {
|
||||
s.data[key] = value
|
||||
}
|
||||
|
||||
return s.save()
|
||||
}
|
||||
|
||||
// DeleteProvider drops every secret for a provider.
|
||||
func (s *fileSecretStore) DeleteProvider(providerID int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if err := s.load(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prefix := strconv.FormatInt(providerID, 10) + ":"
|
||||
|
||||
for k := range s.data {
|
||||
if len(k) > len(prefix) && k[:len(prefix)] == prefix {
|
||||
delete(s.data, k)
|
||||
}
|
||||
}
|
||||
|
||||
return s.save()
|
||||
}
|
||||
|
||||
// lookupFor binds a store to one provider, producing the SecretLookup
|
||||
// handed to constructors.
|
||||
func lookupFor(store SecretStore, providerID int64) SecretLookup {
|
||||
return func(name string) (string, error) {
|
||||
if store == nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name)
|
||||
}
|
||||
|
||||
return store.Get(providerID, name)
|
||||
}
|
||||
}
|
||||
|
||||
// memSecretStore is an in-memory SecretStore for tests.
|
||||
type memSecretStore struct {
|
||||
mu sync.RWMutex
|
||||
data map[string]string
|
||||
}
|
||||
|
||||
// NewMemSecretStore returns an in-memory SecretStore.
|
||||
func NewMemSecretStore() SecretStore {
|
||||
return &memSecretStore{data: map[string]string{}}
|
||||
}
|
||||
|
||||
func (s *memSecretStore) Get(providerID int64, name string) (string, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
v, ok := s.data[secretKey(providerID, name)]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (s *memSecretStore) Set(providerID int64, name, value string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if value == "" {
|
||||
delete(s.data, secretKey(providerID, name))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
s.data[secretKey(providerID, name)] = value
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memSecretStore) DeleteProvider(providerID int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
prefix := strconv.FormatInt(providerID, 10) + ":"
|
||||
|
||||
for k := range s.data {
|
||||
if len(k) > len(prefix) && k[:len(prefix)] == prefix {
|
||||
delete(s.data, k)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileSecretStoreRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "secrets.json")
|
||||
store := NewFileSecretStoreAt(path)
|
||||
|
||||
if err := store.Set(1, "apiKey", "hunter2"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Get(1, "apiKey")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
|
||||
if got != "hunter2" {
|
||||
t.Errorf("Get = %q, want hunter2", got)
|
||||
}
|
||||
|
||||
// A fresh store over the same file must see the value.
|
||||
if got, err := NewFileSecretStoreAt(path).Get(1, "apiKey"); err != nil ||
|
||||
got != "hunter2" {
|
||||
t.Errorf("reload got %q, err %v; want hunter2", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Credentials must not be readable by other local users.
|
||||
func TestFileSecretStorePermissions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "secrets.json")
|
||||
store := NewFileSecretStoreAt(path)
|
||||
|
||||
if err := store.Set(1, "apiKey", "hunter2"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat: %v", err)
|
||||
}
|
||||
|
||||
if perm := info.Mode().Perm(); perm != secretsFileMode {
|
||||
t.Errorf("mode = %o, want %o", perm, secretsFileMode)
|
||||
}
|
||||
}
|
||||
|
||||
// Two configured instances of the same provider kind must not share
|
||||
// credentials.
|
||||
func TestFileSecretStoreNamespacesByProvider(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json"))
|
||||
|
||||
if err := store.Set(1, "apiKey", "first"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Set(2, "apiKey", "second"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
for id, want := range map[int64]string{1: "first", 2: "second"} {
|
||||
got, err := store.Get(id, "apiKey")
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%d): %v", id, err)
|
||||
}
|
||||
|
||||
if got != want {
|
||||
t.Errorf("Get(%d) = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSecretStoreDeleteProvider(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json"))
|
||||
|
||||
if err := store.Set(1, "apiKey", "a"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Set(1, "password", "b"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Set(2, "apiKey", "keep"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
if err := store.DeleteProvider(1); err != nil {
|
||||
t.Fatalf("DeleteProvider: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"apiKey", "password"} {
|
||||
if _, err := store.Get(1, name); !errors.Is(err, ErrSecretNotFound) {
|
||||
t.Errorf("Get(1, %q) error = %v, want ErrSecretNotFound", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if got, err := store.Get(2, "apiKey"); err != nil || got != "keep" {
|
||||
t.Errorf("other provider's secret was removed: %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSecretStoreMissingFileIsEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "nope.json"))
|
||||
|
||||
if _, err := store.Get(1, "apiKey"); !errors.Is(err, ErrSecretNotFound) {
|
||||
t.Errorf("error = %v, want ErrSecretNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetEmptyValueDeletes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json"))
|
||||
|
||||
if err := store.Set(1, "apiKey", "x"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Set(1, "apiKey", ""); err != nil {
|
||||
t.Fatalf("Set empty: %v", err)
|
||||
}
|
||||
|
||||
if _, err := store.Get(1, "apiKey"); !errors.Is(err, ErrSecretNotFound) {
|
||||
t.Errorf("error = %v, want ErrSecretNotFound", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,598 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
|
||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
|
||||
"yellowjacket/backend/events"
|
||||
)
|
||||
|
||||
// Service is the frontend-facing surface of the download subsystem.
|
||||
// Its methods are bound into Wails and called from TypeScript, so
|
||||
// signatures use plain types and return errors the UI can render.
|
||||
type Service struct {
|
||||
logger *slog.Logger
|
||||
manager *Manager
|
||||
store *Store
|
||||
secrets SecretStore
|
||||
|
||||
// reconciler works the wanted list. Optional; nil means wants are
|
||||
// stored but never acted on.
|
||||
reconciler *Reconciler
|
||||
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// NewService builds the bound service.
|
||||
func NewService(
|
||||
logger *slog.Logger,
|
||||
manager *Manager,
|
||||
store *Store,
|
||||
secrets SecretStore,
|
||||
) *Service {
|
||||
return &Service{
|
||||
logger: logger,
|
||||
manager: manager,
|
||||
store: store,
|
||||
secrets: secrets,
|
||||
}
|
||||
}
|
||||
|
||||
// SetContext injects the Wails runtime context for event emission.
|
||||
func (s *Service) SetContext(ctx context.Context) {
|
||||
s.ctx = ctx
|
||||
}
|
||||
|
||||
// emit publishes an event, tolerating a service that has no runtime
|
||||
// context yet. Emitting on a non-runtime context is fatal in Wails, so
|
||||
// the nil check is load-bearing rather than defensive.
|
||||
func (s *Service) emit(name string, data ...any) {
|
||||
if s.ctx == nil {
|
||||
return
|
||||
}
|
||||
|
||||
runtime.EventsEmit(s.ctx, name, data...)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provider configuration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ProviderKinds returns every provider type that can be added, with the
|
||||
// settings each one needs. The settings page renders its forms from
|
||||
// this, so a new adapter needs no frontend change.
|
||||
func (s *Service) ProviderKinds() []Descriptor {
|
||||
return Descriptors()
|
||||
}
|
||||
|
||||
// ListProviders returns the user's configured download clients.
|
||||
func (s *Service) ListProviders() ([]Config, error) {
|
||||
return s.store.ListProviders(context.Background())
|
||||
}
|
||||
|
||||
// AddProvider creates a provider and stores any secret settings
|
||||
// separately. Secrets arrive in the same map as ordinary settings
|
||||
// because that is what the form submits; they are split out here and
|
||||
// never written to the provider row.
|
||||
func (s *Service) AddProvider(
|
||||
kind string,
|
||||
name string,
|
||||
settings map[string]string,
|
||||
) (int64, error) {
|
||||
desc, ok := DescriptorFor(Kind(kind))
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%w: %s", ErrUnknownKind, kind)
|
||||
}
|
||||
|
||||
plain, secret := splitSecrets(desc, settings)
|
||||
|
||||
id, err := s.store.CreateProvider(context.Background(), Config{
|
||||
Kind: Kind(kind),
|
||||
Name: name,
|
||||
Enabled: true,
|
||||
Priority: 50,
|
||||
Settings: plain,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for k, v := range secret {
|
||||
if err := s.secrets.Set(id, k, v); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.manager.Reload(context.Background()); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadProvidersChanged)
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateProvider saves changes to a provider. A secret field left
|
||||
// blank keeps its stored value rather than clearing it — the form does
|
||||
// not echo secrets back, so an empty box means "unchanged", not
|
||||
// "delete".
|
||||
func (s *Service) UpdateProvider(
|
||||
id int64,
|
||||
name string,
|
||||
enabled bool,
|
||||
priority int,
|
||||
settings map[string]string,
|
||||
) error {
|
||||
ctx := context.Background()
|
||||
|
||||
existing, err := s.store.GetProvider(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
desc, ok := DescriptorFor(existing.Kind)
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrUnknownKind, existing.Kind)
|
||||
}
|
||||
|
||||
plain, secret := splitSecrets(desc, settings)
|
||||
|
||||
if err := s.store.UpdateProvider(ctx, Config{
|
||||
ID: id,
|
||||
Kind: existing.Kind,
|
||||
Name: name,
|
||||
Enabled: enabled,
|
||||
Priority: priority,
|
||||
Settings: plain,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for k, v := range secret {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.secrets.Set(id, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.manager.Reload(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadProvidersChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteProvider removes a provider and its credentials.
|
||||
func (s *Service) DeleteProvider(id int64) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if err := s.store.DeleteProvider(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.secrets.DeleteProvider(id); err != nil {
|
||||
s.logger.Warn(
|
||||
"could not delete provider secrets", "provider", id, "error", err,
|
||||
)
|
||||
}
|
||||
|
||||
if err := s.manager.Reload(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadProvidersChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestProvider backs the "test connection" button. It builds the
|
||||
// provider from its stored config and asks it to check itself, so the
|
||||
// result reflects exactly what a real search would use.
|
||||
func (s *Service) TestProvider(id int64) error {
|
||||
ctx := context.Background()
|
||||
|
||||
cfg, err := s.store.GetProvider(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p, err := New(cfg, lookupFor(s.secrets, id), s.logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = p.Close() }()
|
||||
|
||||
if err := p.Check(ctx); err != nil {
|
||||
return fmt.Errorf("%s: %w", cfg.Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Requests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// SearchRequest is what the frontend submits to start a download.
|
||||
type SearchRequest struct {
|
||||
LibraryID int64 `json:"libraryId"`
|
||||
ReleaseMBID string `json:"releaseMbid"`
|
||||
ReleaseGroupMBID string `json:"releaseGroupMbid"`
|
||||
Artist string `json:"artist"`
|
||||
Album string `json:"album"`
|
||||
Query string `json:"query"`
|
||||
Expected []ExpectedTrack `json:"expected"`
|
||||
}
|
||||
|
||||
// StartResult is what the picker needs after a search.
|
||||
type StartResult struct {
|
||||
RequestID string `json:"requestId"`
|
||||
Candidates []Candidate `json:"candidates"`
|
||||
|
||||
// AutoPicked reports that the pipeline already chose and is
|
||||
// downloading, so the picker should show progress rather than a
|
||||
// list of choices.
|
||||
AutoPicked bool `json:"autoPicked"`
|
||||
}
|
||||
|
||||
// Start searches for a release and either auto-picks a clear winner or
|
||||
// returns ranked candidates for the user to choose from.
|
||||
func (s *Service) Start(req SearchRequest) (StartResult, error) {
|
||||
r := Request{
|
||||
ID: newID(),
|
||||
LibraryID: req.LibraryID,
|
||||
ReleaseMBID: req.ReleaseMBID,
|
||||
ReleaseGroupMBID: req.ReleaseGroupMBID,
|
||||
Artist: req.Artist,
|
||||
Album: req.Album,
|
||||
Query: req.Query,
|
||||
Expected: req.Expected,
|
||||
}
|
||||
|
||||
candidates, err := s.manager.Start(context.Background(), r)
|
||||
if err != nil {
|
||||
return StartResult{}, err
|
||||
}
|
||||
|
||||
result := StartResult{
|
||||
RequestID: r.ID,
|
||||
Candidates: candidates,
|
||||
AutoPicked: AutoPickable(r, candidates),
|
||||
}
|
||||
|
||||
s.emit(events.DownloadsChanged)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Pick starts the transfer for the candidate the user chose.
|
||||
func (s *Service) Pick(requestID, candidateID string) error {
|
||||
if err := s.manager.Pick(
|
||||
context.Background(), requestID, candidateID,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadsChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cancel aborts a live request.
|
||||
func (s *Service) Cancel(requestID string) error {
|
||||
if err := s.manager.Cancel(context.Background(), requestID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadsChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Candidates returns the ranked candidates of a live request, so the
|
||||
// picker can be reopened without searching again.
|
||||
func (s *Service) Candidates(requestID string) []Candidate {
|
||||
return s.manager.Candidates(requestID)
|
||||
}
|
||||
|
||||
// RequestView is one row of the downloads list.
|
||||
type RequestView struct {
|
||||
Request
|
||||
|
||||
State State `json:"state"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Items []Item `json:"items"`
|
||||
}
|
||||
|
||||
// ListRequests returns recent download requests, newest first.
|
||||
func (s *Service) ListRequests(limit int) ([]RequestView, error) {
|
||||
const defaultLimit = 50
|
||||
|
||||
if limit <= 0 {
|
||||
limit = defaultLimit
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
requests, err := s.store.ListRequests(ctx, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]RequestView, 0, len(requests))
|
||||
|
||||
for _, r := range requests {
|
||||
state, errText, err := s.store.GetRequestState(ctx, r.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := s.store.ListItemsForRequest(ctx, r.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = append(out, RequestView{
|
||||
Request: r,
|
||||
State: state,
|
||||
Error: errText,
|
||||
Items: items,
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ClearFinished removes terminal requests from the list.
|
||||
func (s *Service) ClearFinished() error {
|
||||
if err := s.store.ClearFinished(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.DownloadsChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wanted list
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// SetReconciler wires the wanted-list loop. Optional: without it the
|
||||
// wanted list still stores and lists wants, it just never acts on them.
|
||||
func (s *Service) SetReconciler(r *Reconciler) {
|
||||
s.reconciler = r
|
||||
}
|
||||
|
||||
// WantRequest is what the frontend submits to want something. It is
|
||||
// one MBID and the type of thing it names, because that is genuinely
|
||||
// all a want is.
|
||||
type WantRequest struct {
|
||||
MBID string `json:"mbid"`
|
||||
Entity string `json:"entity"`
|
||||
LibraryID int64 `json:"libraryId"`
|
||||
|
||||
// Artist and Title are display text only, and optional: the
|
||||
// reconciler fills them in from the catalog when the caller has
|
||||
// nothing but an MBID.
|
||||
Artist string `json:"artist"`
|
||||
Title string `json:"title"`
|
||||
|
||||
// Scope and Secondary apply to artist wants.
|
||||
Scope string `json:"scope"`
|
||||
Secondary bool `json:"secondary"`
|
||||
}
|
||||
|
||||
// AddWant puts something on the wanted list and asks for a reconcile
|
||||
// pass, so the user sees something happen rather than waiting six hours
|
||||
// for the next scheduled one.
|
||||
func (s *Service) AddWant(req WantRequest) (int64, error) {
|
||||
entity := Entity(req.Entity)
|
||||
if !entity.Valid() {
|
||||
return 0, fmt.Errorf("%w: entity %q", ErrUnsupported, req.Entity)
|
||||
}
|
||||
|
||||
scope := WantScope(req.Scope)
|
||||
if scope != ScopeAll {
|
||||
scope = ScopeFuture
|
||||
}
|
||||
|
||||
id, err := s.store.AddWant(context.Background(), Want{
|
||||
MBID: req.MBID,
|
||||
Entity: entity,
|
||||
LibraryID: req.LibraryID,
|
||||
Artist: req.Artist,
|
||||
Title: req.Title,
|
||||
Scope: scope,
|
||||
Secondary: req.Secondary,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
if s.reconciler != nil {
|
||||
s.reconciler.Trigger()
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// ListWants returns the whole wanted list.
|
||||
func (s *Service) ListWants() ([]Want, error) {
|
||||
return s.store.ListWants(context.Background())
|
||||
}
|
||||
|
||||
// IsWanted answers the Explore pages' question — should this album show
|
||||
// "want" or "wanted?" — without making them load the whole list.
|
||||
func (s *Service) IsWanted(mbid string, libraryID int64) (bool, error) {
|
||||
_, found, err := s.store.FindWant(context.Background(), mbid, libraryID)
|
||||
|
||||
return found, err
|
||||
}
|
||||
|
||||
// RemoveWant takes something off the list. Removing an artist takes
|
||||
// its derived albums with it, by cascade; an album the user pinned
|
||||
// themselves has no parent and survives.
|
||||
func (s *Service) RemoveWant(id int64) error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Tell any external list first, while the row is still readable.
|
||||
s.withdrawExternal(ctx, id)
|
||||
|
||||
if err := s.store.DeleteWant(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PauseWant stops attempts without forgetting the want.
|
||||
func (s *Service) PauseWant(id int64, paused bool) error {
|
||||
state := WantStateWanted
|
||||
if paused {
|
||||
state = WantStatePaused
|
||||
}
|
||||
|
||||
if err := s.store.SetWantState(
|
||||
context.Background(), id, state, "",
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearSatisfiedWants drops everything already owned.
|
||||
func (s *Service) ClearSatisfiedWants() error {
|
||||
if err := s.store.ClearSatisfiedWants(context.Background()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReconcileWanted runs a pass now and reports what it did. This backs
|
||||
// the "check now" button, so it runs synchronously: the user pressed it
|
||||
// and is waiting for an answer.
|
||||
func (s *Service) ReconcileWanted() (Summary, error) {
|
||||
if s.reconciler == nil {
|
||||
return Summary{}, fmt.Errorf(
|
||||
"%w: the wanted list is not running", ErrUnsupported,
|
||||
)
|
||||
}
|
||||
|
||||
summary, err := s.reconciler.RunOnce(context.Background())
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// ImportExternalWants adopts a provider's own list — "import the
|
||||
// artists Lidarr is already monitoring".
|
||||
func (s *Service) ImportExternalWants(
|
||||
providerID int64,
|
||||
libraryID int64,
|
||||
) (int, error) {
|
||||
if s.reconciler == nil {
|
||||
return 0, fmt.Errorf(
|
||||
"%w: the wanted list is not running", ErrUnsupported,
|
||||
)
|
||||
}
|
||||
|
||||
n, err := s.reconciler.ImportExternal(
|
||||
context.Background(), providerID, libraryID,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
s.emit(events.WantedListChanged)
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// withdrawExternal best-effort unmonitors a want in the external lists
|
||||
// it was pushed to. Failures are logged and ignored: the user asked to
|
||||
// remove it from *this* list, and an unreachable Lidarr is not a reason
|
||||
// to refuse.
|
||||
func (s *Service) withdrawExternal(ctx context.Context, id int64) {
|
||||
w, err := s.store.GetWant(ctx, id)
|
||||
if err != nil || len(w.ExternalIDs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for key, externalID := range w.ExternalIDs {
|
||||
providerID, err := strconv.ParseInt(key, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
l, ok := s.manager.listers()[providerID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := l.RemoveWant(ctx, externalID); err != nil {
|
||||
s.logger.Debug(
|
||||
"could not withdraw want from external list",
|
||||
"want", id,
|
||||
"provider", providerID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// splitSecrets partitions a submitted settings map into plain values,
|
||||
// which go in the provider row, and secret values, which go in the
|
||||
// secret store. The split is driven by the descriptor so a provider
|
||||
// declaring a field secret is enough to keep it out of the database.
|
||||
func splitSecrets(
|
||||
desc Descriptor,
|
||||
settings map[string]string,
|
||||
) (plain, secret map[string]string) {
|
||||
plain = make(map[string]string, len(settings))
|
||||
secret = make(map[string]string)
|
||||
|
||||
secretKeys := make(map[string]bool, len(desc.Fields))
|
||||
|
||||
for _, f := range desc.Fields {
|
||||
if f.Secret {
|
||||
secretKeys[f.Key] = true
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range settings {
|
||||
if secretKeys[k] {
|
||||
secret[k] = v
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
plain[k] = v
|
||||
}
|
||||
|
||||
return plain, secret
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/system"
|
||||
)
|
||||
|
||||
// Downloads land here first and only move into the library once they
|
||||
// have been verified and tagged. The reason is not tidiness: the
|
||||
// library scanner watches library paths, and a half-written file or a
|
||||
// mislabelled Soulseek folder that lands there gets ingested, indexed
|
||||
// and surfaced to the user before anyone can check it. Staging makes
|
||||
// the import step the single writer into library paths.
|
||||
|
||||
// stagingDirName is the staging root inside the user data directory.
|
||||
const stagingDirName = "downloads"
|
||||
|
||||
// staleAge is how long an abandoned staging directory survives before
|
||||
// the startup sweep removes it. Long enough that a download
|
||||
// interrupted by a crash can still be inspected; short enough that a
|
||||
// failed grab does not sit on disk forever.
|
||||
const staleAge = 48 * time.Hour
|
||||
|
||||
// ErrEscapesStaging is returned when a provider reports a file path
|
||||
// outside the directory it was given.
|
||||
var ErrEscapesStaging = errors.New("path escapes the staging directory")
|
||||
|
||||
// Staging owns the download staging area.
|
||||
type Staging struct {
|
||||
root string
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewStaging creates the staging area under the user data directory.
|
||||
func NewStaging(logger *slog.Logger) (*Staging, error) {
|
||||
dir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve user data dir: %w", err)
|
||||
}
|
||||
|
||||
return NewStagingAt(filepath.Join(dir, stagingDirName), logger)
|
||||
}
|
||||
|
||||
// NewStagingAt creates a staging area at an explicit root.
|
||||
func NewStagingAt(root string, logger *slog.Logger) (*Staging, error) {
|
||||
if err := os.MkdirAll(root, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("create staging root: %w", err)
|
||||
}
|
||||
|
||||
return &Staging{root: root, logger: logger}, nil
|
||||
}
|
||||
|
||||
// Root returns the staging root directory.
|
||||
func (s *Staging) Root() string {
|
||||
return s.root
|
||||
}
|
||||
|
||||
// Reserve creates and returns a directory for one download item.
|
||||
func (s *Staging) Reserve(itemID string) (string, error) {
|
||||
dir := filepath.Join(s.root, sanitizeSegment(itemID))
|
||||
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return "", fmt.Errorf("create staging dir: %w", err)
|
||||
}
|
||||
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// Release removes a download item's staging directory and everything
|
||||
// in it. Called after a successful import and after a failed grab.
|
||||
func (s *Staging) Release(dir string) error {
|
||||
if !s.contains(dir) {
|
||||
return fmt.Errorf("%w: %s", ErrEscapesStaging, dir)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return fmt.Errorf("remove staging dir: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// contains reports whether dir is inside the staging root. Guards
|
||||
// every destructive operation, because dir ultimately comes from a
|
||||
// database row a provider wrote.
|
||||
func (s *Staging) contains(dir string) bool {
|
||||
absRoot, err := filepath.Abs(s.root)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
absDir, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(absRoot, absDir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// Verify checks that every path a provider reported is a real file
|
||||
// inside dir, and returns them cleaned. A transport that reports a
|
||||
// path outside its directory is either buggy or hostile; either way the
|
||||
// import must not follow it.
|
||||
func (s *Staging) Verify(dir string, files []string) ([]string, error) {
|
||||
if !s.contains(dir) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrEscapesStaging, dir)
|
||||
}
|
||||
|
||||
absDir, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve staging dir: %w", err)
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
abs := f
|
||||
if !filepath.IsAbs(abs) {
|
||||
abs = filepath.Join(absDir, f)
|
||||
}
|
||||
|
||||
abs = filepath.Clean(abs)
|
||||
|
||||
rel, err := filepath.Rel(absDir, abs)
|
||||
if err != nil ||
|
||||
rel == ".." ||
|
||||
strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrEscapesStaging, f)
|
||||
}
|
||||
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat downloaded file %s: %w", rel, err)
|
||||
}
|
||||
|
||||
if info.IsDir() || info.Size() == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, abs)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Sweep removes staging directories left behind by a previous run.
|
||||
// Anything still present at startup belongs to a download that did not
|
||||
// finish, since a completed import releases its directory.
|
||||
//
|
||||
// Directories younger than staleAge are kept: a grab may legitimately
|
||||
// be resumed, and deleting a partial transfer the user is waiting on
|
||||
// would be worse than leaving a few megabytes on disk.
|
||||
func (s *Staging) Sweep() (removed int, err error) {
|
||||
entries, err := os.ReadDir(s.root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("read staging root: %w", err)
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(-staleAge)
|
||||
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if info.ModTime().After(cutoff) {
|
||||
continue
|
||||
}
|
||||
|
||||
dir := filepath.Join(s.root, e.Name())
|
||||
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
s.logger.Warn(
|
||||
"could not remove stale staging directory",
|
||||
"dir", dir,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
removed++
|
||||
}
|
||||
|
||||
if removed > 0 {
|
||||
s.logger.Info("removed stale staging directories", "count", removed)
|
||||
}
|
||||
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// SweepOrphans removes staging directories whose item IDs are not in
|
||||
// the live set. Called after the item store is loaded, so a directory
|
||||
// belonging to a download the database no longer knows about goes away
|
||||
// even if it is recent.
|
||||
func (s *Staging) SweepOrphans(live map[string]bool) (removed int, err error) {
|
||||
entries, err := os.ReadDir(s.root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("read staging root: %w", err)
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || live[e.Name()] {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(filepath.Join(s.root, e.Name())); err != nil {
|
||||
s.logger.Warn(
|
||||
"could not remove orphaned staging directory",
|
||||
"dir", e.Name(),
|
||||
"error", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
removed++
|
||||
}
|
||||
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// sanitizeSegment reduces a string to a safe single path segment.
|
||||
func sanitizeSegment(s string) string {
|
||||
var b strings.Builder
|
||||
|
||||
b.Grow(len(s))
|
||||
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z',
|
||||
r >= 'A' && r <= 'Z',
|
||||
r >= '0' && r <= '9',
|
||||
r == '-', r == '_':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
|
||||
out := b.String()
|
||||
if out == "" {
|
||||
return "item"
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newTestStaging(t *testing.T) *Staging {
|
||||
t.Helper()
|
||||
|
||||
s, err := NewStagingAt(t.TempDir(), slogDiscard())
|
||||
if err != nil {
|
||||
t.Fatalf("NewStagingAt: %v", err)
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func TestStagingReserveAndRelease(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
dir, err := s.Reserve("item-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("staging dir not created: %v", err)
|
||||
}
|
||||
|
||||
if err := s.Release(dir); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Error("staging dir still exists after Release")
|
||||
}
|
||||
}
|
||||
|
||||
// A provider reporting a path outside its staging directory is either
|
||||
// buggy or hostile. Either way the import must refuse to follow it,
|
||||
// because the next step moves those paths into the library.
|
||||
func TestStagingVerifyRejectsEscape(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
dir, err := s.Reserve("item-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
outside := filepath.Join(s.Root(), "elsewhere.flac")
|
||||
if err := os.WriteFile(outside, []byte("x"), 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
tests := []string{
|
||||
"../elsewhere.flac",
|
||||
outside,
|
||||
filepath.Join(dir, "..", "elsewhere.flac"),
|
||||
}
|
||||
|
||||
for _, path := range tests {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if _, err := s.Verify(dir, []string{path}); !errors.Is(
|
||||
err, ErrEscapesStaging,
|
||||
) {
|
||||
t.Errorf("Verify(%q) error = %v, want ErrEscapesStaging", path, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagingReleaseRejectsOutsideRoot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
other := t.TempDir()
|
||||
|
||||
if err := s.Release(other); !errors.Is(err, ErrEscapesStaging) {
|
||||
t.Errorf("Release outside root error = %v, want ErrEscapesStaging", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(other); err != nil {
|
||||
t.Error("Release removed a directory outside the staging root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagingVerifySkipsEmptyFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
dir, err := s.Reserve("item-1")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
good := filepath.Join(dir, "good.flac")
|
||||
empty := filepath.Join(dir, "empty.flac")
|
||||
|
||||
if err := os.WriteFile(good, []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(empty, nil, 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
files, err := s.Verify(dir, []string{good, empty})
|
||||
if err != nil {
|
||||
t.Fatalf("Verify: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 1 || files[0] != good {
|
||||
t.Errorf("Verify = %v, want just %s", files, good)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagingSweepKeepsRecentDirs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
recent, err := s.Reserve("recent")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
stale, err := s.Reserve("stale")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
old := time.Now().Add(-staleAge - time.Hour)
|
||||
if err := os.Chtimes(stale, old, old); err != nil {
|
||||
t.Fatalf("Chtimes: %v", err)
|
||||
}
|
||||
|
||||
removed, err := s.Sweep()
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep: %v", err)
|
||||
}
|
||||
|
||||
if removed != 1 {
|
||||
t.Errorf("removed = %d, want 1", removed)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(recent); err != nil {
|
||||
t.Error("sweep removed a recent staging dir")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(stale); !os.IsNotExist(err) {
|
||||
t.Error("sweep kept a stale staging dir")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagingSweepOrphans(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newTestStaging(t)
|
||||
|
||||
live, err := s.Reserve("live")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
orphan, err := s.Reserve("orphan")
|
||||
if err != nil {
|
||||
t.Fatalf("Reserve: %v", err)
|
||||
}
|
||||
|
||||
removed, err := s.SweepOrphans(map[string]bool{"live": true})
|
||||
if err != nil {
|
||||
t.Fatalf("SweepOrphans: %v", err)
|
||||
}
|
||||
|
||||
if removed != 1 {
|
||||
t.Errorf("removed = %d, want 1", removed)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(live); err != nil {
|
||||
t.Error("sweep removed a live staging dir")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(orphan); !os.IsNotExist(err) {
|
||||
t.Error("sweep kept an orphaned staging dir")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/database/sql/sqlcgen"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned when a request or item ID is unknown.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
// Store is the download subsystem's persistence layer. It owns the
|
||||
// JSON encoding of the blob columns so nothing above it has to know
|
||||
// that candidates are stored as text.
|
||||
type Store struct {
|
||||
db *database.DB
|
||||
}
|
||||
|
||||
// NewStore returns a Store over the application database.
|
||||
func NewStore(db *database.DB) *Store {
|
||||
return &Store{db: db}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Providers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ListProviders returns every configured provider, best priority first.
|
||||
func (s *Store) ListProviders(ctx context.Context) ([]Config, error) {
|
||||
rows, err := s.db.ReadQueries.ListDownloadProviders(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list download providers: %w", err)
|
||||
}
|
||||
|
||||
out := make([]Config, 0, len(rows))
|
||||
|
||||
for _, r := range rows {
|
||||
out = append(out, providerRowToConfig(r))
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetProvider returns one provider's config.
|
||||
func (s *Store) GetProvider(ctx context.Context, id int64) (Config, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadProvider(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Config{}, fmt.Errorf("%w: provider %d", ErrNotFound, id)
|
||||
}
|
||||
|
||||
return Config{}, fmt.Errorf("get download provider: %w", err)
|
||||
}
|
||||
|
||||
return providerRowToConfig(row), nil
|
||||
}
|
||||
|
||||
// CreateProvider inserts a provider and returns its new ID.
|
||||
func (s *Store) CreateProvider(ctx context.Context, cfg Config) (int64, error) {
|
||||
settings, err := json.Marshal(cfg.Settings)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("encode provider settings: %w", err)
|
||||
}
|
||||
|
||||
id, err := s.db.Queries.CreateDownloadProvider(
|
||||
ctx,
|
||||
sqlcgen.CreateDownloadProviderParams{
|
||||
Kind: string(cfg.Kind),
|
||||
Name: cfg.Name,
|
||||
Enabled: boolToInt(cfg.Enabled),
|
||||
Priority: int64(cfg.Priority),
|
||||
Settings: string(settings),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("create download provider: %w", err)
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// UpdateProvider saves changes to an existing provider.
|
||||
func (s *Store) UpdateProvider(ctx context.Context, cfg Config) error {
|
||||
settings, err := json.Marshal(cfg.Settings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode provider settings: %w", err)
|
||||
}
|
||||
|
||||
if err := s.db.Queries.UpdateDownloadProvider(
|
||||
ctx,
|
||||
sqlcgen.UpdateDownloadProviderParams{
|
||||
Name: cfg.Name,
|
||||
Enabled: boolToInt(cfg.Enabled),
|
||||
Priority: int64(cfg.Priority),
|
||||
Settings: string(settings),
|
||||
ID: cfg.ID,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("update download provider: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteProvider removes a provider row. Its secrets are removed
|
||||
// separately by the manager, which owns the secret store.
|
||||
func (s *Store) DeleteProvider(ctx context.Context, id int64) error {
|
||||
if err := s.db.Queries.DeleteDownloadProvider(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete download provider: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// providerRowToConfig decodes a stored provider row. A settings blob
|
||||
// that fails to parse yields an empty map rather than an error: the
|
||||
// provider will fail its own Check with a useful message, which beats
|
||||
// making the whole settings page unloadable.
|
||||
func providerRowToConfig(r sqlcgen.DownloadProvider) Config {
|
||||
settings := map[string]string{}
|
||||
_ = json.Unmarshal([]byte(r.Settings), &settings)
|
||||
|
||||
return Config{
|
||||
ID: r.ID,
|
||||
Kind: Kind(r.Kind),
|
||||
Name: r.Name,
|
||||
Enabled: r.Enabled != 0,
|
||||
Priority: int(r.Priority),
|
||||
Settings: settings,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Requests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// CreateRequest persists a new request.
|
||||
func (s *Store) CreateRequest(ctx context.Context, req Request) error {
|
||||
expected, err := json.Marshal(req.Expected)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode expected tracks: %w", err)
|
||||
}
|
||||
|
||||
source := req.Source
|
||||
if source == "" {
|
||||
source = "manual"
|
||||
}
|
||||
|
||||
wantID := sql.NullInt64{}
|
||||
if req.WantID != 0 {
|
||||
wantID = sql.NullInt64{Int64: req.WantID, Valid: true}
|
||||
}
|
||||
|
||||
if err := s.db.Queries.CreateDownloadRequest(
|
||||
ctx,
|
||||
sqlcgen.CreateDownloadRequestParams{
|
||||
ID: req.ID,
|
||||
LibraryID: req.LibraryID,
|
||||
Source: source,
|
||||
WantID: wantID,
|
||||
ReleaseMbid: toNullString(req.ReleaseMBID),
|
||||
ReleaseGroupMbid: toNullString(req.ReleaseGroupMBID),
|
||||
RecordingMbid: toNullString(req.RecordingMBID),
|
||||
Artist: req.Artist,
|
||||
Album: req.Album,
|
||||
Query: req.Query,
|
||||
Expected: string(expected),
|
||||
State: string(StateSearching),
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRequest loads a request by ID.
|
||||
func (s *Store) GetRequest(ctx context.Context, id string) (Request, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadRequest(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Request{}, fmt.Errorf("%w: request %s", ErrNotFound, id)
|
||||
}
|
||||
|
||||
return Request{}, fmt.Errorf("get download request: %w", err)
|
||||
}
|
||||
|
||||
return requestRowToRequest(row), nil
|
||||
}
|
||||
|
||||
// GetRequestState returns a request's current state and error text.
|
||||
// Kept separate from GetRequest because state is the one field that
|
||||
// changes constantly while the rest of the row is immutable.
|
||||
func (s *Store) GetRequestState(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
) (State, string, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadRequest(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", "", fmt.Errorf("%w: request %s", ErrNotFound, id)
|
||||
}
|
||||
|
||||
return "", "", fmt.Errorf("get download request state: %w", err)
|
||||
}
|
||||
|
||||
return State(row.State), row.Error, nil
|
||||
}
|
||||
|
||||
// ListRequests returns the most recent requests, newest first.
|
||||
func (s *Store) ListRequests(ctx context.Context, limit int) ([]Request, error) {
|
||||
rows, err := s.db.ReadQueries.ListDownloadRequests(ctx, int64(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list download requests: %w", err)
|
||||
}
|
||||
|
||||
out := make([]Request, 0, len(rows))
|
||||
|
||||
for _, r := range rows {
|
||||
out = append(out, requestRowToRequest(r))
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetRequestState updates a request's state and error text.
|
||||
func (s *Store) SetRequestState(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
state State,
|
||||
errText string,
|
||||
) error {
|
||||
if err := s.db.Queries.SetDownloadRequestState(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadRequestStateParams{
|
||||
State: string(state),
|
||||
Error: errText,
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download request state: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteRequest removes a request and, by cascade, its items.
|
||||
func (s *Store) DeleteRequest(ctx context.Context, id string) error {
|
||||
if err := s.db.Queries.DeleteDownloadRequest(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete download request: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearFinished removes every terminal request.
|
||||
func (s *Store) ClearFinished(ctx context.Context) error {
|
||||
if err := s.db.Queries.DeleteFinishedDownloadRequests(ctx); err != nil {
|
||||
return fmt.Errorf("clear finished download requests: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// requestRowToRequest decodes a stored request row.
|
||||
func requestRowToRequest(r sqlcgen.DownloadRequest) Request {
|
||||
var expected []ExpectedTrack
|
||||
|
||||
_ = json.Unmarshal([]byte(r.Expected), &expected)
|
||||
|
||||
return Request{
|
||||
ID: r.ID,
|
||||
LibraryID: r.LibraryID,
|
||||
Source: r.Source,
|
||||
WantID: r.WantID.Int64,
|
||||
ReleaseMBID: r.ReleaseMbid.String,
|
||||
ReleaseGroupMBID: r.ReleaseGroupMbid.String,
|
||||
RecordingMBID: r.RecordingMbid.String,
|
||||
Artist: r.Artist,
|
||||
Album: r.Album,
|
||||
Query: r.Query,
|
||||
Expected: expected,
|
||||
CreatedAt: r.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Items
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Item is one grab attempt, as stored.
|
||||
type Item struct {
|
||||
ID string `json:"id"`
|
||||
RequestID string `json:"requestId"`
|
||||
ProviderID int64 `json:"providerId"`
|
||||
Transport int64 `json:"transportId,omitempty"`
|
||||
ExternalID string `json:"externalId,omitempty"`
|
||||
Candidate Candidate `json:"candidate"`
|
||||
State State `json:"state"`
|
||||
StagingDir string `json:"-"`
|
||||
BytesDone int64 `json:"bytesDone"`
|
||||
BytesTotal int64 `json:"bytesTotal"`
|
||||
Imported []string `json:"-"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// CreateItem persists a grab attempt.
|
||||
func (s *Store) CreateItem(ctx context.Context, item Item) error {
|
||||
candidate, err := json.Marshal(item.Candidate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode candidate: %w", err)
|
||||
}
|
||||
|
||||
transport := sql.NullInt64{}
|
||||
if item.Transport != 0 {
|
||||
transport = sql.NullInt64{Int64: item.Transport, Valid: true}
|
||||
}
|
||||
|
||||
if err := s.db.Queries.CreateDownloadItem(
|
||||
ctx,
|
||||
sqlcgen.CreateDownloadItemParams{
|
||||
ID: item.ID,
|
||||
RequestID: item.RequestID,
|
||||
ProviderID: item.ProviderID,
|
||||
TransportID: transport,
|
||||
ExternalID: item.ExternalID,
|
||||
Candidate: string(candidate),
|
||||
State: string(item.State),
|
||||
StagingDir: item.StagingDir,
|
||||
BytesTotal: item.BytesTotal,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("create download item: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetItem loads one item.
|
||||
func (s *Store) GetItem(ctx context.Context, id string) (Item, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadItem(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Item{}, fmt.Errorf("%w: item %s", ErrNotFound, id)
|
||||
}
|
||||
|
||||
return Item{}, fmt.Errorf("get download item: %w", err)
|
||||
}
|
||||
|
||||
return itemRowToItem(row), nil
|
||||
}
|
||||
|
||||
// ListItemsForRequest returns a request's grab attempts, oldest first.
|
||||
func (s *Store) ListItemsForRequest(
|
||||
ctx context.Context,
|
||||
requestID string,
|
||||
) ([]Item, error) {
|
||||
rows, err := s.db.ReadQueries.ListDownloadItemsForRequest(ctx, requestID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list download items: %w", err)
|
||||
}
|
||||
|
||||
out := make([]Item, 0, len(rows))
|
||||
|
||||
for _, r := range rows {
|
||||
out = append(out, itemRowToItem(r))
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListLiveItems returns every non-terminal item. Called at startup to
|
||||
// decide what to resume, reconcile or abandon.
|
||||
func (s *Store) ListLiveItems(ctx context.Context) ([]Item, error) {
|
||||
rows, err := s.db.ReadQueries.ListLiveDownloadItems(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list live download items: %w", err)
|
||||
}
|
||||
|
||||
out := make([]Item, 0, len(rows))
|
||||
|
||||
for _, r := range rows {
|
||||
out = append(out, itemRowToItem(r))
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetItemState updates an item's state and error text.
|
||||
func (s *Store) SetItemState(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
state State,
|
||||
errText string,
|
||||
) error {
|
||||
if err := s.db.Queries.SetDownloadItemState(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadItemStateParams{
|
||||
State: string(state),
|
||||
Error: errText,
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download item state: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetItemProgress records transfer progress.
|
||||
func (s *Store) SetItemProgress(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
done, total int64,
|
||||
) error {
|
||||
if err := s.db.Queries.SetDownloadItemProgress(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadItemProgressParams{
|
||||
BytesDone: done,
|
||||
BytesTotal: total,
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download item progress: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetItemExternalID records a delegating manager's own identifier.
|
||||
func (s *Store) SetItemExternalID(
|
||||
ctx context.Context,
|
||||
id, externalID string,
|
||||
) error {
|
||||
if err := s.db.Queries.SetDownloadItemExternalID(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadItemExternalIDParams{
|
||||
ExternalID: externalID,
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download item external id: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetItemImported records the library paths files landed at and marks
|
||||
// the item complete.
|
||||
func (s *Store) SetItemImported(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
paths []string,
|
||||
) error {
|
||||
encoded, err := json.Marshal(paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode imported paths: %w", err)
|
||||
}
|
||||
|
||||
if err := s.db.Queries.SetDownloadItemImported(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadItemImportedParams{
|
||||
ImportedPaths: string(encoded),
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download item imported: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// itemRowToItem decodes a stored item row.
|
||||
func itemRowToItem(r sqlcgen.DownloadItem) Item {
|
||||
var (
|
||||
candidate Candidate
|
||||
imported []string
|
||||
)
|
||||
|
||||
_ = json.Unmarshal([]byte(r.Candidate), &candidate)
|
||||
_ = json.Unmarshal([]byte(r.ImportedPaths), &imported)
|
||||
|
||||
return Item{
|
||||
ID: r.ID,
|
||||
RequestID: r.RequestID,
|
||||
ProviderID: r.ProviderID,
|
||||
Transport: r.TransportID.Int64,
|
||||
ExternalID: r.ExternalID,
|
||||
Candidate: candidate,
|
||||
State: State(r.State),
|
||||
StagingDir: r.StagingDir,
|
||||
BytesDone: r.BytesDone,
|
||||
BytesTotal: r.BytesTotal,
|
||||
Imported: imported,
|
||||
Error: r.Error,
|
||||
CreatedAt: r.CreatedAt,
|
||||
UpdatedAt: r.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// boolToInt converts a bool to SQLite's integer boolean.
|
||||
func boolToInt(b bool) int64 {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// toNullString wraps a possibly-empty string for a nullable column.
|
||||
func toNullString(s string) sql.NullString {
|
||||
return sql.NullString{String: s, Valid: s != ""}
|
||||
}
|
||||
@@ -0,0 +1,667 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// qBittorrent
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// qbitStub is a fake qBittorrent Web API.
|
||||
type qbitStub struct {
|
||||
server *httptest.Server
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
// loginOK controls whether auth succeeds.
|
||||
loginOK bool
|
||||
|
||||
// torrentStates is what the info endpoint returns, in order; the
|
||||
// last entry repeats.
|
||||
torrentStates [][]qbitTorrent
|
||||
pollCount int
|
||||
|
||||
// addedForm records the add-torrent parameters.
|
||||
addedForm url.Values
|
||||
}
|
||||
|
||||
func newQbitStub(t *testing.T) *qbitStub {
|
||||
t.Helper()
|
||||
|
||||
s := &qbitStub{loginOK: true}
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/api/v2/auth/login", func(w http.ResponseWriter, _ *http.Request) {
|
||||
s.mu.Lock()
|
||||
ok := s.loginOK
|
||||
s.mu.Unlock()
|
||||
|
||||
if ok {
|
||||
_, _ = w.Write([]byte("Ok."))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// qBittorrent answers a bad login with 200 and "Fails.".
|
||||
_, _ = w.Write([]byte("Fails."))
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v2/app/version", func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("v4.6.0"))
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v2/torrents/add", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Errorf("parse add form: %v", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.addedForm = r.PostForm
|
||||
s.mu.Unlock()
|
||||
|
||||
_, _ = w.Write([]byte("Ok."))
|
||||
})
|
||||
|
||||
mux.HandleFunc("/api/v2/torrents/info", func(w http.ResponseWriter, _ *http.Request) {
|
||||
s.mu.Lock()
|
||||
|
||||
idx := s.pollCount
|
||||
if idx >= len(s.torrentStates) {
|
||||
idx = len(s.torrentStates) - 1
|
||||
} else {
|
||||
s.pollCount++
|
||||
}
|
||||
|
||||
var batch []qbitTorrent
|
||||
if idx >= 0 && len(s.torrentStates) > 0 {
|
||||
batch = s.torrentStates[idx]
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, batch)
|
||||
})
|
||||
|
||||
s.server = httptest.NewServer(mux)
|
||||
t.Cleanup(s.server.Close)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func newStubQbit(t *testing.T, stub *qbitStub) *qbittorrent {
|
||||
t.Helper()
|
||||
|
||||
p, err := newQBittorrent(
|
||||
Config{
|
||||
ID: 1,
|
||||
Kind: KindQBittorrent,
|
||||
Name: "qbittorrent",
|
||||
Settings: map[string]string{
|
||||
"url": stub.server.URL,
|
||||
"username": "admin",
|
||||
},
|
||||
},
|
||||
func(string) (string, error) { return "secret", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newQBittorrent: %v", err)
|
||||
}
|
||||
|
||||
q, ok := p.(*qbittorrent)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *qbittorrent", p)
|
||||
}
|
||||
|
||||
q.pollInterval = time.Millisecond
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
func TestQbitCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newQbitStub(t)
|
||||
q := newStubQbit(t, stub)
|
||||
|
||||
if err := q.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// qBittorrent returns HTTP 200 with "Fails." for a bad password, so a
|
||||
// status-code-only check would report success.
|
||||
func TestQbitRejectsBadPassword(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newQbitStub(t)
|
||||
stub.loginOK = false
|
||||
|
||||
q := newStubQbit(t, stub)
|
||||
|
||||
if err := q.Check(context.Background()); !errors.Is(err, ErrQbitAuth) {
|
||||
t.Errorf("error = %v, want ErrQbitAuth", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The transport must be a pure transport: no search role.
|
||||
func TestQbitDeclaresTransportOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newQbitStub(t)
|
||||
q := newStubQbit(t, stub)
|
||||
|
||||
caps := q.Info().Caps
|
||||
|
||||
if caps.CanSearch || caps.CanDelegate {
|
||||
t.Errorf("qBittorrent should transport only, got %+v", caps)
|
||||
}
|
||||
|
||||
if !caps.Handles(ProtocolTorrent) {
|
||||
t.Error("qBittorrent should handle the torrent protocol")
|
||||
}
|
||||
|
||||
if caps.Handles(ProtocolUsenet) {
|
||||
t.Error("qBittorrent should not claim usenet")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQbitGrabCollectsCompletedTorrent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newQbitStub(t)
|
||||
|
||||
// The torrent's content lands in a directory qBittorrent owns.
|
||||
content := t.TempDir()
|
||||
|
||||
for _, name := range []string{"01 Airbag.flac", "02 Paranoid Android.flac"} {
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(content, name), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
stub.torrentStates = [][]qbitTorrent{
|
||||
{{
|
||||
Hash: "abc123", State: "downloading",
|
||||
Progress: 0.4, Size: 1000, Completed: 400,
|
||||
}},
|
||||
{{
|
||||
Hash: "abc123", State: "uploading",
|
||||
Progress: 1.0, Size: 1000, Completed: 1000,
|
||||
ContentPath: content,
|
||||
}},
|
||||
}
|
||||
|
||||
q := newStubQbit(t, stub)
|
||||
dst := t.TempDir()
|
||||
|
||||
c := Candidate{
|
||||
ID: "prowlarr:x",
|
||||
Protocol: ProtocolTorrent,
|
||||
Title: "Radiohead - OK Computer",
|
||||
Payload: map[string]string{
|
||||
"link": "magnet:?xt=urn:btih:abc123",
|
||||
"infoHash": "abc123",
|
||||
},
|
||||
}
|
||||
|
||||
got, err := q.Grab(context.Background(), c, dst, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
for _, f := range got.Files {
|
||||
if !strings.HasPrefix(f, dst) {
|
||||
t.Errorf("file %s is outside the staging dir", f)
|
||||
}
|
||||
}
|
||||
|
||||
// The torrent was saved into our staging directory and kept out of
|
||||
// qBittorrent's own move-on-completion rules.
|
||||
stub.mu.Lock()
|
||||
form := stub.addedForm
|
||||
stub.mu.Unlock()
|
||||
|
||||
if form.Get("savepath") != dst {
|
||||
t.Errorf("savepath = %q, want the staging dir %q", form.Get("savepath"), dst)
|
||||
}
|
||||
|
||||
if form.Get("autoTMM") != "false" {
|
||||
t.Errorf("autoTMM = %q, want false", form.Get("autoTMM"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestQbitGrabFailsOnErrorState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newQbitStub(t)
|
||||
stub.torrentStates = [][]qbitTorrent{
|
||||
{{Hash: "abc123", State: "error"}},
|
||||
}
|
||||
|
||||
q := newStubQbit(t, stub)
|
||||
|
||||
c := Candidate{
|
||||
Protocol: ProtocolTorrent,
|
||||
Payload: map[string]string{
|
||||
"link": "magnet:?xt=urn:btih:abc123", "infoHash": "abc123",
|
||||
},
|
||||
}
|
||||
|
||||
_, err := q.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrQbitTransferFailed) {
|
||||
t.Errorf("error = %v, want ErrQbitTransferFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfoHashFromMagnet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"magnet:?xt=urn:btih:ABC123&dn=x", "abc123"},
|
||||
{"magnet:?dn=x&xt=urn:btih:def456", "def456"},
|
||||
{"magnet:?dn=no-hash", ""},
|
||||
{"https://example.com/x.torrent", ""},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := infoHashFromMagnet(tt.in); got != tt.want {
|
||||
t.Errorf("infoHashFromMagnet(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SABnzbd
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// sabStub is a fake SABnzbd API.
|
||||
type sabStub struct {
|
||||
server *httptest.Server
|
||||
|
||||
mu sync.Mutex
|
||||
|
||||
// queueSlots and historySlots are returned in order per mode; the
|
||||
// last entry repeats.
|
||||
queueSlots [][]sabQueueSlot
|
||||
queuePolls int
|
||||
historySlots []sabHistorySlot
|
||||
|
||||
addStatus bool
|
||||
addError string
|
||||
|
||||
badKey bool
|
||||
}
|
||||
|
||||
func newSabStub(t *testing.T) *sabStub {
|
||||
t.Helper()
|
||||
|
||||
s := &sabStub{addStatus: true}
|
||||
|
||||
s.server = httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
badKey := s.badKey
|
||||
s.mu.Unlock()
|
||||
|
||||
if badKey {
|
||||
// SABnzbd reports a bad key with HTTP 200 and a body.
|
||||
writeJSON(t, w, map[string]any{
|
||||
"status": false, "error": "API Key Incorrect",
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
switch r.URL.Query().Get("mode") {
|
||||
case "version":
|
||||
writeJSON(t, w, map[string]any{"version": "4.1.0"})
|
||||
case "addurl":
|
||||
s.mu.Lock()
|
||||
status, errMsg := s.addStatus, s.addError
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{
|
||||
"status": status, "error": errMsg,
|
||||
"nzo_ids": []string{"SABnzbd_nzo_1"},
|
||||
})
|
||||
case "queue":
|
||||
s.mu.Lock()
|
||||
|
||||
idx := s.queuePolls
|
||||
if idx >= len(s.queueSlots) {
|
||||
idx = len(s.queueSlots) - 1
|
||||
} else {
|
||||
s.queuePolls++
|
||||
}
|
||||
|
||||
var slots []sabQueueSlot
|
||||
if idx >= 0 && len(s.queueSlots) > 0 {
|
||||
slots = s.queueSlots[idx]
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{
|
||||
"queue": map[string]any{"slots": slots},
|
||||
})
|
||||
case "history":
|
||||
s.mu.Lock()
|
||||
slots := s.historySlots
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{
|
||||
"history": map[string]any{"slots": slots},
|
||||
})
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
},
|
||||
))
|
||||
|
||||
t.Cleanup(s.server.Close)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func newStubSab(t *testing.T, stub *sabStub) *sabnzbd {
|
||||
t.Helper()
|
||||
|
||||
p, err := newSABnzbd(
|
||||
Config{
|
||||
ID: 1,
|
||||
Kind: KindSABnzbd,
|
||||
Name: "sabnzbd",
|
||||
Settings: map[string]string{"url": stub.server.URL},
|
||||
},
|
||||
func(string) (string, error) { return "test-key", nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newSABnzbd: %v", err)
|
||||
}
|
||||
|
||||
s, ok := p.(*sabnzbd)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T, want *sabnzbd", p)
|
||||
}
|
||||
|
||||
s.pollInterval = time.Millisecond
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func TestSabCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
s := newStubSab(t, stub)
|
||||
|
||||
if err := s.Check(context.Background()); err != nil {
|
||||
t.Errorf("Check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSabDeclaresUsenetOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
s := newStubSab(t, stub)
|
||||
|
||||
caps := s.Info().Caps
|
||||
|
||||
if caps.CanSearch || caps.CanDelegate {
|
||||
t.Errorf("SABnzbd should transport only, got %+v", caps)
|
||||
}
|
||||
|
||||
if !caps.Handles(ProtocolUsenet) {
|
||||
t.Error("SABnzbd should handle the usenet protocol")
|
||||
}
|
||||
|
||||
if caps.Handles(ProtocolTorrent) {
|
||||
t.Error("SABnzbd should not claim torrents")
|
||||
}
|
||||
}
|
||||
|
||||
// Completion is read from history, not from the queue emptying: a job
|
||||
// leaves the queue before post-processing finishes.
|
||||
func TestSabGrabWaitsForHistory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
|
||||
storage := t.TempDir()
|
||||
|
||||
for _, name := range []string{"01 Airbag.flac", "02 Paranoid Android.flac"} {
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(storage, name), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
stub.queueSlots = [][]sabQueueSlot{
|
||||
{{
|
||||
NzoID: "SABnzbd_nzo_1", Status: "Downloading",
|
||||
MB: "100.0", MBLeft: "60.0",
|
||||
}},
|
||||
// Second poll: gone from the queue.
|
||||
{},
|
||||
}
|
||||
stub.historySlots = []sabHistorySlot{{
|
||||
NzoID: "SABnzbd_nzo_1", Status: "Completed", Storage: storage,
|
||||
}}
|
||||
|
||||
s := newStubSab(t, stub)
|
||||
dst := t.TempDir()
|
||||
|
||||
c := Candidate{
|
||||
Protocol: ProtocolUsenet,
|
||||
Title: "Radiohead - OK Computer",
|
||||
Payload: map[string]string{"link": "https://example.com/x.nzb"},
|
||||
}
|
||||
|
||||
got, err := s.Grab(context.Background(), c, dst, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
for _, f := range got.Files {
|
||||
if !strings.HasPrefix(f, dst) {
|
||||
t.Errorf("file %s is outside the staging dir", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSabGrabFailsOnFailedJob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
stub.queueSlots = [][]sabQueueSlot{{}}
|
||||
stub.historySlots = []sabHistorySlot{{
|
||||
NzoID: "SABnzbd_nzo_1",
|
||||
Status: "Failed",
|
||||
FailMsg: "Unpacking failed",
|
||||
}}
|
||||
|
||||
s := newStubSab(t, stub)
|
||||
|
||||
c := Candidate{
|
||||
Payload: map[string]string{"link": "https://example.com/x.nzb"},
|
||||
}
|
||||
|
||||
_, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSabTransferFailed) {
|
||||
t.Errorf("error = %v, want ErrSabTransferFailed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A job that vanishes from both queue and history was removed out from
|
||||
// under us, which must not look like success.
|
||||
func TestSabGrabDetectsVanishedJob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
stub.queueSlots = [][]sabQueueSlot{{}}
|
||||
stub.historySlots = nil
|
||||
|
||||
s := newStubSab(t, stub)
|
||||
|
||||
c := Candidate{
|
||||
Payload: map[string]string{"link": "https://example.com/x.nzb"},
|
||||
}
|
||||
|
||||
_, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSabNoJob) {
|
||||
t.Errorf("error = %v, want ErrSabNoJob", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An NZB URL is untrusted input from an indexer.
|
||||
func TestSabGrabRejectsNonHTTPURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSabStub(t)
|
||||
s := newStubSab(t, stub)
|
||||
|
||||
c := Candidate{Payload: map[string]string{"link": "file:///etc/passwd"}}
|
||||
|
||||
_, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrUnsafeURL) {
|
||||
t.Errorf("error = %v, want ErrUnsafeURL", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMB(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
in string
|
||||
want int64
|
||||
}{
|
||||
{"1.0", 1024 * 1024},
|
||||
{"0", 0},
|
||||
{" 2.5 ", int64(2.5 * 1024 * 1024)},
|
||||
{"garbage", 0},
|
||||
{"", 0},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.in, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := parseMB(tt.in); got != tt.want {
|
||||
t.Errorf("parseMB(%q) = %d, want %d", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// collectTree flattens whatever shape the transport produced, because
|
||||
// the importer wants a flat set of paths inside staging.
|
||||
func TestCollectTree(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("directory tree is flattened", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
nested := filepath.Join(root, "CD1")
|
||||
|
||||
if err := os.MkdirAll(nested, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(root, "a.flac"), []byte("x"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(nested, "b.flac"), []byte("y"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := collectTree(root, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("collectTree: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single file", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
file := filepath.Join(root, "single.flac")
|
||||
|
||||
if err := os.WriteFile(file, []byte("x"), 0o600); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := collectTree(file, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("collectTree: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Fatalf("collected %d files, want 1", len(got.Files))
|
||||
}
|
||||
|
||||
if filepath.Dir(got.Files[0]) != dst {
|
||||
t.Errorf("file landed at %s, want inside %s", got.Files[0], dst)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing path errors", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if _, err := collectTree(
|
||||
filepath.Join(t.TempDir(), "nope"), t.TempDir(),
|
||||
); err == nil {
|
||||
t.Error("want an error for a missing content path")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
// Package download acquires music from user-configured external
|
||||
// services and imports it into the library.
|
||||
//
|
||||
// The services users connect are not the same kind of thing: some
|
||||
// search, some move bytes, some are whole automation systems we hand a
|
||||
// request to. Rather than one interface every adapter half-implements,
|
||||
// a provider fills one or more of three roles — Searcher, Transporter,
|
||||
// Delegator — and declares which in its Caps. The pipeline composes
|
||||
// them: a search-only provider (Prowlarr) is paired with a transport
|
||||
// (qBittorrent, SABnzbd) by protocol at grab time, while providers that
|
||||
// do both (slskd, yt-dlp) pair with themselves.
|
||||
//
|
||||
// Nothing here downloads into the library. Grabs land in a staging
|
||||
// directory, are verified and tagged against the release the user
|
||||
// actually asked for, and only then move into library paths.
|
||||
package download
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Kind identifies a provider implementation. It is stored in the
|
||||
// database and used to look up the constructor in the registry, so
|
||||
// values are stable strings and never renamed.
|
||||
type Kind string
|
||||
|
||||
// Provider kinds.
|
||||
const (
|
||||
KindSlskd Kind = "slskd"
|
||||
KindYtDlp Kind = "yt-dlp"
|
||||
KindLidarr Kind = "lidarr"
|
||||
KindProwlarr Kind = "prowlarr"
|
||||
KindQBittorrent Kind = "qbittorrent"
|
||||
KindSABnzbd Kind = "sabnzbd"
|
||||
|
||||
// KindFake is an in-memory provider used by tests. It is never
|
||||
// offered in the UI.
|
||||
KindFake Kind = "fake"
|
||||
)
|
||||
|
||||
// Protocol is how a candidate's bytes are moved. Search-only providers
|
||||
// report it so the pipeline can pick a compatible transport; providers
|
||||
// that transport their own results use ProtocolDirect.
|
||||
type Protocol string
|
||||
|
||||
// Transport protocols.
|
||||
const (
|
||||
// ProtocolDirect means the finding provider also does the fetch.
|
||||
ProtocolDirect Protocol = "direct"
|
||||
ProtocolTorrent Protocol = "torrent"
|
||||
ProtocolUsenet Protocol = "usenet"
|
||||
)
|
||||
|
||||
// Caps declares which roles a provider fills and which optional
|
||||
// behaviours it supports. The frontend renders controls from this
|
||||
// rather than switching on Kind, so a provider that gains resume
|
||||
// support later needs no frontend change.
|
||||
type Caps struct {
|
||||
// Roles.
|
||||
CanSearch bool `json:"canSearch"`
|
||||
CanTransport bool `json:"canTransport"`
|
||||
CanDelegate bool `json:"canDelegate"`
|
||||
|
||||
// CanList marks a provider that keeps a persistent wanted list of
|
||||
// its own, which the reconciler mirrors this app's list into.
|
||||
CanList bool `json:"canList"`
|
||||
|
||||
// Optional behaviours.
|
||||
CanResume bool `json:"canResume"`
|
||||
CanCancel bool `json:"canCancel"`
|
||||
ReportsSize bool `json:"reportsSize"`
|
||||
|
||||
// Protocols this provider can transport. Empty for providers that
|
||||
// only fetch their own search results.
|
||||
Transports []Protocol `json:"transports"`
|
||||
}
|
||||
|
||||
// Handles reports whether the provider can transport the given protocol.
|
||||
func (c Caps) Handles(p Protocol) bool {
|
||||
return slices.Contains(c.Transports, p)
|
||||
}
|
||||
|
||||
// Request is what the user asked for. Requests that carry a MusicBrainz
|
||||
// anchor are far more reliable than free-text ones, because the anchor
|
||||
// gives the import step an expected tracklist to match against — so the
|
||||
// pipeline records which it got and refuses to auto-pick without one.
|
||||
type Request struct {
|
||||
ID string `json:"id"`
|
||||
|
||||
// Anchors. Any may be empty; all empty means free-text.
|
||||
ReleaseMBID string `json:"releaseMbid,omitempty"`
|
||||
ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"`
|
||||
|
||||
// RecordingMBID anchors a single-track request. Its Expected holds
|
||||
// exactly that one track, which is what lets a track request be
|
||||
// scored — and therefore auto-picked — on the same footing as an
|
||||
// album.
|
||||
RecordingMBID string `json:"recordingMbid,omitempty"`
|
||||
|
||||
// WantID links back to the wanted-list row this request was raised
|
||||
// for, or 0 for a request the user started by hand. The reconciler
|
||||
// writes the outcome back through it.
|
||||
WantID int64 `json:"wantId,omitempty"`
|
||||
|
||||
// Source records where the request came from, for the downloads
|
||||
// list. Empty means "manual".
|
||||
Source string `json:"source,omitempty"`
|
||||
|
||||
// Display and query text. Artist/Album are what searches are built
|
||||
// from; Query overrides them when the user typed something raw.
|
||||
Artist string `json:"artist"`
|
||||
Album string `json:"album"`
|
||||
Query string `json:"query,omitempty"`
|
||||
|
||||
// Expected is the tracklist the anchor resolves to, used for
|
||||
// completeness scoring and for the autotag match at import. Empty
|
||||
// for free-text requests.
|
||||
Expected []ExpectedTrack `json:"expected,omitempty"`
|
||||
|
||||
// LibraryID is the library imported files belong to.
|
||||
LibraryID int64 `json:"libraryId"`
|
||||
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
// Anchored reports whether the request carries a MusicBrainz ID. Only
|
||||
// anchored requests are eligible for auto-pick.
|
||||
func (r Request) Anchored() bool {
|
||||
return r.ReleaseMBID != "" ||
|
||||
r.ReleaseGroupMBID != "" ||
|
||||
r.RecordingMBID != ""
|
||||
}
|
||||
|
||||
// SearchText returns the string to hand a provider's search endpoint.
|
||||
func (r Request) SearchText() string {
|
||||
if r.Query != "" {
|
||||
return r.Query
|
||||
}
|
||||
|
||||
return strings.TrimSpace(r.Artist + " " + r.Album)
|
||||
}
|
||||
|
||||
// ExpectedTrack is one track of the release the user asked for.
|
||||
type ExpectedTrack struct {
|
||||
Position int `json:"position"`
|
||||
DiscNumber int `json:"discNumber"`
|
||||
Title string `json:"title"`
|
||||
Artist string `json:"artist"`
|
||||
LengthMillis int64 `json:"lengthMillis"`
|
||||
}
|
||||
|
||||
// Candidate is one acquirable thing a provider found: a Soulseek user's
|
||||
// folder, a torrent, a YouTube playlist. Providers fill the descriptive
|
||||
// fields; the ranker fills Match, Quality and Score.
|
||||
type Candidate struct {
|
||||
// ID is unique within the provider that produced it, and is what
|
||||
// gets handed back to Grab.
|
||||
ID string `json:"id"`
|
||||
ProviderID int64 `json:"providerId"`
|
||||
Kind Kind `json:"kind"`
|
||||
|
||||
// Protocol determines which transport can fetch this.
|
||||
Protocol Protocol `json:"protocol"`
|
||||
|
||||
// Descriptive.
|
||||
Title string `json:"title"`
|
||||
Artist string `json:"artist,omitempty"`
|
||||
Origin string `json:"origin,omitempty"` // peer username, indexer name, channel
|
||||
|
||||
Files []CandidateFile `json:"files"`
|
||||
TotalSize int64 `json:"totalSize"`
|
||||
|
||||
// Health is the provider's own availability signal, normalized to
|
||||
// 0..1: seeder count for torrents, free upload slots and queue
|
||||
// length for Soulseek. 0.5 when the provider has no signal.
|
||||
Health float64 `json:"health"`
|
||||
|
||||
// Scores, filled by the ranker.
|
||||
Match MatchScore `json:"match"`
|
||||
Quality QualityScore `json:"quality"`
|
||||
Score float64 `json:"score"`
|
||||
|
||||
// Payload is provider-private data needed to fetch this candidate
|
||||
// (magnet URI, NZB URL, slskd file list). Never shown to the user.
|
||||
Payload map[string]string `json:"-"`
|
||||
}
|
||||
|
||||
// CandidateFile is one file inside a candidate. Soulseek and torrent
|
||||
// results give paths and sizes but no tags, so Format and duration are
|
||||
// inferred from the path and size where possible.
|
||||
type CandidateFile struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
Format Format `json:"format"`
|
||||
Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown
|
||||
IsAudio bool `json:"isAudio"`
|
||||
MatchedTo int `json:"matchedTo,omitempty"` // expected track position
|
||||
}
|
||||
|
||||
// Format is a normalized audio container/codec name.
|
||||
type Format string
|
||||
|
||||
// Audio formats, ordered by the quality ranking in formatRank.
|
||||
const (
|
||||
FormatUnknown Format = ""
|
||||
FormatFLAC Format = "flac"
|
||||
FormatALAC Format = "alac"
|
||||
FormatWAV Format = "wav"
|
||||
FormatMP3 Format = "mp3"
|
||||
FormatAAC Format = "aac"
|
||||
FormatOGG Format = "ogg"
|
||||
FormatOpus Format = "opus"
|
||||
FormatWMA Format = "wma"
|
||||
)
|
||||
|
||||
// Lossless reports whether the format preserves the source exactly.
|
||||
func (f Format) Lossless() bool {
|
||||
return f == FormatFLAC || f == FormatALAC || f == FormatWAV
|
||||
}
|
||||
|
||||
// Supported reports whether the player can decode this format. Grabs
|
||||
// of unsupported formats are still allowed — the user may want them —
|
||||
// but they rank below playable ones.
|
||||
func (f Format) Supported() bool {
|
||||
switch f {
|
||||
case FormatMP3, FormatFLAC, FormatOGG, FormatWAV:
|
||||
return true
|
||||
case FormatUnknown, FormatALAC, FormatAAC, FormatOpus, FormatWMA:
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// MatchScore answers "is this the release the user asked for?" It is
|
||||
// deliberately separate from QualityScore: a perfect match at 128kbps
|
||||
// and a mediocre match in FLAC are different failures, and collapsing
|
||||
// them into one number makes the ranking impossible to explain.
|
||||
type MatchScore struct {
|
||||
// Overall is 0..1.
|
||||
Overall float64 `json:"overall"`
|
||||
|
||||
TitleFit float64 `json:"titleFit"` // filenames vs expected titles
|
||||
ArtistFit float64 `json:"artistFit"` // path/origin vs expected artist
|
||||
AlbumFit float64 `json:"albumFit"` // folder name vs album title
|
||||
Completeness float64 `json:"completeness"` // audio files vs expected count
|
||||
|
||||
// Anchored records whether an MBID drove this score. Unanchored
|
||||
// matches are capped, because there is nothing to be right about.
|
||||
Anchored bool `json:"anchored"`
|
||||
}
|
||||
|
||||
// QualityScore answers "is this a good copy?".
|
||||
type QualityScore struct {
|
||||
// Overall is 0..1.
|
||||
Overall float64 `json:"overall"`
|
||||
|
||||
FormatRank float64 `json:"formatRank"` // FLAC > V0 > 320 > lower
|
||||
Bitrate float64 `json:"bitrate"`
|
||||
Health float64 `json:"health"` // seeders, free slots
|
||||
Priority float64 `json:"priority"` // user's per-provider preference
|
||||
|
||||
// Mixed marks a candidate whose files are not all the same format,
|
||||
// which usually means a hand-assembled folder rather than a rip.
|
||||
Mixed bool `json:"mixed"`
|
||||
}
|
||||
|
||||
// AudioFiles returns only the audio entries of a candidate.
|
||||
func (c Candidate) AudioFiles() []CandidateFile {
|
||||
out := make([]CandidateFile, 0, len(c.Files))
|
||||
|
||||
for _, f := range c.Files {
|
||||
if f.IsAudio {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// State is the lifecycle position of a download item.
|
||||
type State string
|
||||
|
||||
// Download item states. Searching through Importing are live;
|
||||
// Complete, Cancelled and Failed are terminal.
|
||||
const (
|
||||
StateSearching State = "searching"
|
||||
StateFound State = "found"
|
||||
StateQueued State = "queued"
|
||||
StateGrabbing State = "grabbing"
|
||||
StateVerifying State = "verifying"
|
||||
StateTagging State = "tagging"
|
||||
StateImporting State = "importing"
|
||||
StateComplete State = "complete"
|
||||
StateCancelled State = "cancelled"
|
||||
StateFailed State = "failed"
|
||||
)
|
||||
|
||||
// IsTerminal reports whether the state means no further progress will
|
||||
// happen without a new attempt.
|
||||
func (s State) IsTerminal() bool {
|
||||
return s == StateComplete || s == StateCancelled || s == StateFailed
|
||||
}
|
||||
|
||||
// Progress is a transport's periodic report. Total is 0 when the
|
||||
// provider cannot say how large the transfer is.
|
||||
type Progress struct {
|
||||
Current int64
|
||||
Total int64
|
||||
Phase string
|
||||
}
|
||||
|
||||
// ProgressFunc receives transport progress. Implementations must
|
||||
// tolerate being called from any goroutine and at high frequency.
|
||||
type ProgressFunc func(Progress)
|
||||
|
||||
// Result is what a transport produced.
|
||||
type Result struct {
|
||||
// Dir is the staging directory the files landed in.
|
||||
Dir string
|
||||
|
||||
// Files are absolute paths, all under Dir.
|
||||
Files []string
|
||||
|
||||
// BytesTransferred is what actually moved, for reporting.
|
||||
BytesTransferred int64
|
||||
|
||||
// Delegated marks a result produced by an external manager that has
|
||||
// already imported the files into its own library. Files are then
|
||||
// absolute paths outside staging, and the pipeline records them
|
||||
// where they are instead of tagging and moving them.
|
||||
Delegated bool
|
||||
}
|
||||
|
||||
// DelegateStatus is a delegating manager's answer to "are we there
|
||||
// yet?".
|
||||
type DelegateStatus struct {
|
||||
State State
|
||||
|
||||
// Progress is 0..1 when the manager reports it, -1 when it does not.
|
||||
Progress float64
|
||||
|
||||
// ImportedPaths are files the manager has already placed on disk.
|
||||
// A delegate that imports into its own library reports them here so
|
||||
// the pipeline can reconcile rather than re-import.
|
||||
ImportedPaths []string
|
||||
|
||||
Message string
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Want is a persistent "I want this", stored as a MusicBrainz ID and
|
||||
// almost nothing else.
|
||||
//
|
||||
// The distinction from Request is the whole point of this file. A
|
||||
// Request is one attempt: it searches, it grabs, it succeeds or fails,
|
||||
// and then it is history. A Want outlives every attempt made on its
|
||||
// behalf. Nothing being findable today is the normal case for obscure
|
||||
// music, and the correct response is to try again next week, not to
|
||||
// show the user a failed row they have to remember to retry.
|
||||
//
|
||||
// Because a Want is only an MBID, it stays true when everything around
|
||||
// it changes: the explore index is rebuilt, a provider is swapped out,
|
||||
// the release the user originally saw is superseded by a remaster. The
|
||||
// display fields are a cache for the list view and are never consulted
|
||||
// for matching.
|
||||
|
||||
// Entity says what a want's MBID names, and is the only type
|
||||
// distinction the wanted list makes.
|
||||
type Entity string
|
||||
|
||||
// Want entity types.
|
||||
const (
|
||||
// EntityArtist is a subscription rather than a thing to fetch: it
|
||||
// is never satisfied, and each reconcile expands the artist's
|
||||
// discography into child wants.
|
||||
EntityArtist Entity = "artist"
|
||||
|
||||
// EntityReleaseGroup is an album in the abstract — any release of
|
||||
// it satisfies the want, which is what a user means by "I want this
|
||||
// album".
|
||||
EntityReleaseGroup Entity = "release-group"
|
||||
|
||||
// EntityRelease is one specific edition, used when the user picked
|
||||
// a particular pressing.
|
||||
EntityRelease Entity = "release"
|
||||
|
||||
// EntityRecording is a single track.
|
||||
EntityRecording Entity = "recording"
|
||||
)
|
||||
|
||||
// Valid reports whether e is a known entity type.
|
||||
func (e Entity) Valid() bool {
|
||||
switch e {
|
||||
case EntityArtist, EntityReleaseGroup, EntityRelease, EntityRecording:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Expands reports whether this entity produces child wants rather than
|
||||
// being downloaded directly.
|
||||
func (e Entity) Expands() bool {
|
||||
return e == EntityArtist
|
||||
}
|
||||
|
||||
// WantState is where a want sits. There is deliberately no "failed":
|
||||
// an attempt can fail, a want cannot. A want that has tried and not
|
||||
// found anything is still wanted, with attempts and last_error
|
||||
// recording why it is taking a while.
|
||||
type WantState string
|
||||
|
||||
// Want states.
|
||||
const (
|
||||
// WantStateWanted is the active state: due for another attempt when
|
||||
// its backoff elapses.
|
||||
WantStateWanted WantState = "wanted"
|
||||
|
||||
// WantStateSatisfied means the library owns it. How it got there —
|
||||
// downloaded here, ripped, bought elsewhere — does not matter.
|
||||
WantStateSatisfied WantState = "satisfied"
|
||||
|
||||
// WantStatePaused is the user saying "keep this on the list but
|
||||
// stop trying".
|
||||
WantStatePaused WantState = "paused"
|
||||
)
|
||||
|
||||
// WantScope applies to artist wants only.
|
||||
type WantScope string
|
||||
|
||||
// Artist want scopes.
|
||||
const (
|
||||
// ScopeFuture takes only releases first published after the artist
|
||||
// was added. Default, because subscribing to an artist should not
|
||||
// silently queue their entire back catalogue.
|
||||
ScopeFuture WantScope = "future"
|
||||
|
||||
// ScopeAll backfills the whole discography as well.
|
||||
ScopeAll WantScope = "all"
|
||||
)
|
||||
|
||||
// Want is one row of the wanted list.
|
||||
type Want struct {
|
||||
ID int64 `json:"id"`
|
||||
MBID string `json:"mbid"`
|
||||
Entity Entity `json:"entity"`
|
||||
LibraryID int64 `json:"libraryId"`
|
||||
|
||||
// Artist and Title are display cache only. Matching always uses
|
||||
// the MBID.
|
||||
Artist string `json:"artist"`
|
||||
Title string `json:"title"`
|
||||
|
||||
Scope WantScope `json:"scope"`
|
||||
|
||||
// Secondary includes compilations, live albums and remixes in an
|
||||
// artist want's expansion.
|
||||
Secondary bool `json:"secondary"`
|
||||
|
||||
State WantState `json:"state"`
|
||||
|
||||
// ParentID is set on wants the reconciler derived from an artist
|
||||
// subscription. A want the user pinned directly has none, so
|
||||
// removing the artist leaves it alone.
|
||||
ParentID int64 `json:"parentId,omitempty"`
|
||||
|
||||
Attempts int `json:"attempts"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
LastTriedAt time.Time `json:"lastTriedAt,omitempty"`
|
||||
NextTryAt time.Time `json:"nextTryAt,omitempty"`
|
||||
|
||||
// ExternalIDs maps provider row ID (as a string, because JSON
|
||||
// object keys are strings) to that provider's own identifier for
|
||||
// this want. Only set for providers that keep a persistent list of
|
||||
// their own.
|
||||
ExternalIDs map[string]string `json:"externalIds,omitempty"`
|
||||
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// Anchored is always true for a want: it is an MBID by construction.
|
||||
// The method exists so wants and requests read the same at call sites.
|
||||
func (w Want) Anchored() bool { return w.MBID != "" }
|
||||
|
||||
// Label is the wanted list's one-line description of a want.
|
||||
func (w Want) Label() string {
|
||||
switch {
|
||||
case w.Artist != "" && w.Title != "":
|
||||
return w.Artist + " — " + w.Title
|
||||
case w.Title != "":
|
||||
return w.Title
|
||||
case w.Artist != "":
|
||||
return w.Artist
|
||||
default:
|
||||
return string(w.Entity) + " " + w.MBID
|
||||
}
|
||||
}
|
||||
|
||||
// Retry backoff. A want that cannot be found is usually one that will
|
||||
// not be findable for a while — a pre-release, something only ever on
|
||||
// physical media, an artist no source indexes — so the schedule climbs
|
||||
// fast and then sits at a weekly poll rather than hammering providers
|
||||
// with the same fruitless search.
|
||||
const (
|
||||
// wantRetryBase is the delay after the first unsuccessful attempt.
|
||||
wantRetryBase = 6 * time.Hour
|
||||
|
||||
// wantRetryMax caps the backoff. A weekly retry on a list of a few
|
||||
// hundred wants is a handful of searches a day, which every
|
||||
// provider tolerates.
|
||||
wantRetryMax = 7 * 24 * time.Hour
|
||||
|
||||
// wantRetryJitter spreads retries so a list added in one sitting
|
||||
// does not come due in one burst.
|
||||
wantRetryJitter = 0.2
|
||||
)
|
||||
|
||||
// nextRetry returns when a want with the given attempt count should be
|
||||
// tried again: exponential from wantRetryBase, capped at wantRetryMax,
|
||||
// jittered so a batch added together does not stay in lockstep forever.
|
||||
func nextRetry(now time.Time, attempts int) time.Time {
|
||||
if attempts < 1 {
|
||||
attempts = 1
|
||||
}
|
||||
|
||||
// Cap the exponent before shifting so a long-lived want cannot
|
||||
// overflow the duration into something negative.
|
||||
const maxExp = 16
|
||||
|
||||
exp := min(attempts-1, maxExp)
|
||||
|
||||
delay := float64(wantRetryBase) * math.Pow(2, float64(exp))
|
||||
if delay > float64(wantRetryMax) {
|
||||
delay = float64(wantRetryMax)
|
||||
}
|
||||
|
||||
jitter := delay * wantRetryJitter * (rand.Float64()*2 - 1) //nolint:gosec // spreading retries, not a secret
|
||||
|
||||
return now.Add(time.Duration(delay + jitter))
|
||||
}
|
||||
|
||||
// wantSource is the request source recorded for reconciler-raised
|
||||
// requests, so the downloads list can tell them apart from the ones a
|
||||
// user started by hand.
|
||||
const wantSource = "wanted"
|
||||
|
||||
// ToRequest builds the download request that would satisfy this want.
|
||||
// Expected is filled by the caller from the catalog, since resolving a
|
||||
// tracklist is I/O and this is not.
|
||||
func (w Want) ToRequest(id string) Request {
|
||||
req := Request{
|
||||
ID: id,
|
||||
LibraryID: w.LibraryID,
|
||||
Artist: w.Artist,
|
||||
Album: w.Title,
|
||||
WantID: w.ID,
|
||||
Source: wantSource,
|
||||
}
|
||||
|
||||
switch w.Entity {
|
||||
case EntityRelease:
|
||||
req.ReleaseMBID = w.MBID
|
||||
case EntityReleaseGroup:
|
||||
req.ReleaseGroupMBID = w.MBID
|
||||
case EntityRecording:
|
||||
// A recording has no release anchor, so ranking has only the
|
||||
// title to go on and auto-pick stays off. The MBID is still
|
||||
// carried in RecordingMBID so a provider that can use it does.
|
||||
req.RecordingMBID = w.MBID
|
||||
case EntityArtist:
|
||||
// Artist wants expand into children and are never turned into
|
||||
// a request directly; this case exists so the switch is
|
||||
// exhaustive rather than because it can happen.
|
||||
}
|
||||
|
||||
return req
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNextRetryClimbsAndCaps(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// The jitter makes exact equality wrong to assert, so each step is
|
||||
// checked as a band around its nominal delay.
|
||||
tests := []struct {
|
||||
attempts int
|
||||
nominal time.Duration
|
||||
}{
|
||||
{attempts: 1, nominal: wantRetryBase},
|
||||
{attempts: 2, nominal: 2 * wantRetryBase},
|
||||
{attempts: 3, nominal: 4 * wantRetryBase},
|
||||
{attempts: 20, nominal: wantRetryMax},
|
||||
{attempts: 500, nominal: wantRetryMax},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := nextRetry(now, tt.attempts).Sub(now)
|
||||
|
||||
lo := time.Duration(float64(tt.nominal) * (1 - wantRetryJitter))
|
||||
hi := time.Duration(float64(tt.nominal) * (1 + wantRetryJitter))
|
||||
|
||||
if got < lo || got > hi {
|
||||
t.Errorf(
|
||||
"attempts=%d delay=%v, want within [%v, %v]",
|
||||
tt.attempts, got, lo, hi,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A want that has been retried for years must not overflow into a
|
||||
// negative delay, which would make it due forever.
|
||||
func TestNextRetryNeverGoesBackwards(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
for _, attempts := range []int{0, 1, 64, 1000, 1 << 20} {
|
||||
if next := nextRetry(now, attempts); !next.After(now) {
|
||||
t.Errorf("attempts=%d scheduled %v, not after now", attempts, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseDateAfter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
since := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
date string
|
||||
want bool
|
||||
}{
|
||||
{name: "later full date", date: "2026-06-01", want: true},
|
||||
{name: "earlier full date", date: "2026-01-01", want: false},
|
||||
{name: "same day is not after", date: "2026-03-15", want: false},
|
||||
{name: "later year", date: "2027", want: true},
|
||||
// A bare year is read as its 1 January, so the year of the
|
||||
// subscription itself does not count as new.
|
||||
{name: "same year, bare", date: "2026", want: false},
|
||||
{name: "later month, bare", date: "2026-08", want: true},
|
||||
{name: "earlier month, bare", date: "2026-02", want: false},
|
||||
{name: "unknown date is old", date: "", want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := releaseDateAfter(tt.date, since); got != tt.want {
|
||||
t.Errorf("%s: releaseDateAfter(%q) = %v, want %v",
|
||||
tt.name, tt.date, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWantsReleaseGroupFilters(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
subscribed := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
future := Want{Scope: ScopeFuture, CreatedAt: subscribed}
|
||||
all := Want{Scope: ScopeAll, CreatedAt: subscribed}
|
||||
allSecondary := Want{
|
||||
Scope: ScopeAll, Secondary: true, CreatedAt: subscribed,
|
||||
}
|
||||
|
||||
newAlbum := CatalogItem{MBID: "a", FirstReleaseDate: "2026-09-01"}
|
||||
oldAlbum := CatalogItem{MBID: "b", FirstReleaseDate: "1997-04-22"}
|
||||
ownedAlbum := CatalogItem{
|
||||
MBID: "c", FirstReleaseDate: "2026-09-01", InLibrary: true,
|
||||
}
|
||||
liveAlbum := CatalogItem{
|
||||
MBID: "d",
|
||||
FirstReleaseDate: "2026-09-01",
|
||||
SecondaryTypes: []string{"Live"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
artist Want
|
||||
rg CatalogItem
|
||||
want bool
|
||||
}{
|
||||
{name: "future takes new", artist: future, rg: newAlbum, want: true},
|
||||
{name: "future skips old", artist: future, rg: oldAlbum, want: false},
|
||||
{name: "all takes old", artist: all, rg: oldAlbum, want: true},
|
||||
{name: "owned is never wanted", artist: all, rg: ownedAlbum, want: false},
|
||||
{name: "secondary off skips live", artist: all, rg: liveAlbum, want: false},
|
||||
{
|
||||
name: "secondary on takes live",
|
||||
artist: allSecondary,
|
||||
rg: liveAlbum,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no mbid is unusable",
|
||||
artist: all,
|
||||
rg: CatalogItem{FirstReleaseDate: "2026-09-01"},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := wantsReleaseGroup(tt.artist, tt.rg); got != tt.want {
|
||||
t.Errorf("%s: got %v, want %v", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWantToRequestAnchors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
entity Entity
|
||||
check func(Request) string
|
||||
}{
|
||||
{
|
||||
entity: EntityReleaseGroup,
|
||||
check: func(r Request) string {
|
||||
return r.ReleaseGroupMBID
|
||||
},
|
||||
},
|
||||
{
|
||||
entity: EntityRelease,
|
||||
check: func(r Request) string { return r.ReleaseMBID },
|
||||
},
|
||||
{
|
||||
entity: EntityRecording,
|
||||
check: func(r Request) string { return r.RecordingMBID },
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
w := Want{ID: 7, MBID: "mbid-x", Entity: tt.entity, LibraryID: 1}
|
||||
|
||||
req := w.ToRequest("req-1")
|
||||
|
||||
if got := tt.check(req); got != "mbid-x" {
|
||||
t.Errorf("%s: anchor = %q, want mbid-x", tt.entity, got)
|
||||
}
|
||||
|
||||
if !req.Anchored() {
|
||||
t.Errorf("%s: request is not anchored", tt.entity)
|
||||
}
|
||||
|
||||
if req.WantID != 7 {
|
||||
t.Errorf("%s: WantID = %d, want 7", tt.entity, req.WantID)
|
||||
}
|
||||
|
||||
if req.Source != wantSource {
|
||||
t.Errorf("%s: Source = %q, want %q", tt.entity, req.Source, wantSource)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An anchored request with no tracklist has nothing to verify itself
|
||||
// against, so it must not be auto-picked no matter how good the
|
||||
// candidate looks.
|
||||
func TestAutoPickableRequiresTracklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := Request{ReleaseGroupMBID: "rg-1", Artist: "A", Album: "B"}
|
||||
|
||||
ranked := []Candidate{{
|
||||
Match: MatchScore{Overall: 0.99, Anchored: true},
|
||||
Quality: QualityScore{Overall: 0.9},
|
||||
Score: 0.95,
|
||||
}}
|
||||
|
||||
if AutoPickable(req, ranked) {
|
||||
t.Error("auto-picked a request with no expected tracklist")
|
||||
}
|
||||
|
||||
req.Expected = []ExpectedTrack{{Position: 1, Title: "T"}}
|
||||
|
||||
if !AutoPickable(req, ranked) {
|
||||
t.Error("did not auto-pick a well-anchored, well-matched request")
|
||||
}
|
||||
}
|
||||
|
||||
// The wanted list's identity is the MBID, so the same one arriving
|
||||
// twice — in a different case, with whitespace — is one row.
|
||||
func TestAddWantNormalizesAndDeduplicates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
first, err := f.store.AddWant(ctx, Want{
|
||||
MBID: " ABC-123 ",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Title: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
second, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "abc-123",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant again: %v", err)
|
||||
}
|
||||
|
||||
if first != second {
|
||||
t.Errorf("ids %d and %d, want the same row", first, second)
|
||||
}
|
||||
|
||||
// Re-adding with no title must not wipe the one we have.
|
||||
w, err := f.store.GetWant(ctx, first)
|
||||
if err != nil {
|
||||
t.Fatalf("GetWant: %v", err)
|
||||
}
|
||||
|
||||
if w.Title != "OK Computer" {
|
||||
t.Errorf("title = %q, want it preserved", w.Title)
|
||||
}
|
||||
|
||||
if w.MBID != "abc-123" {
|
||||
t.Errorf("mbid = %q, want normalized", w.MBID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWantStoreLifecycle(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
id, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
Title: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant: %v", err)
|
||||
}
|
||||
|
||||
// A brand new want is due immediately.
|
||||
due, err := f.store.ListDueWants(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDueWants: %v", err)
|
||||
}
|
||||
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("got %d due wants, want 1", len(due))
|
||||
}
|
||||
|
||||
// Recording an attempt pushes it out of the due set without
|
||||
// changing its state: a want that was not found is still wanted.
|
||||
if err := f.store.RecordAttempt(ctx, id, 0, "no source has it yet"); err != nil {
|
||||
t.Fatalf("RecordAttempt: %v", err)
|
||||
}
|
||||
|
||||
due, err = f.store.ListDueWants(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDueWants after attempt: %v", err)
|
||||
}
|
||||
|
||||
if len(due) != 0 {
|
||||
t.Errorf("got %d due wants after an attempt, want 0", len(due))
|
||||
}
|
||||
|
||||
w, err := f.store.GetWant(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetWant: %v", err)
|
||||
}
|
||||
|
||||
if w.State != WantStateWanted {
|
||||
t.Errorf("state = %q, want it still wanted", w.State)
|
||||
}
|
||||
|
||||
if w.Attempts != 1 {
|
||||
t.Errorf("attempts = %d, want 1", w.Attempts)
|
||||
}
|
||||
|
||||
if w.LastError == "" {
|
||||
t.Error("last error was not recorded")
|
||||
}
|
||||
|
||||
if err := f.store.SatisfyWant(ctx, id); err != nil {
|
||||
t.Fatalf("SatisfyWant: %v", err)
|
||||
}
|
||||
|
||||
w, err = f.store.GetWant(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetWant after satisfy: %v", err)
|
||||
}
|
||||
|
||||
if w.State != WantStateSatisfied {
|
||||
t.Errorf("state = %q, want satisfied", w.State)
|
||||
}
|
||||
}
|
||||
|
||||
// Removing an artist subscription takes the albums it derived with it,
|
||||
// so a user who unsubscribes does not keep downloading that artist.
|
||||
func TestDeleteArtistWantCascadesToChildren(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
ctx := context.Background()
|
||||
|
||||
artist, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "artist-1",
|
||||
Entity: EntityArtist,
|
||||
LibraryID: 1,
|
||||
Artist: "Radiohead",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("AddWant artist: %v", err)
|
||||
}
|
||||
|
||||
if _, err := f.store.AddWant(ctx, Want{
|
||||
MBID: "rg-1",
|
||||
Entity: EntityReleaseGroup,
|
||||
LibraryID: 1,
|
||||
ParentID: artist,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddWant child: %v", err)
|
||||
}
|
||||
|
||||
children, err := f.store.ListChildWants(ctx, artist)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChildWants: %v", err)
|
||||
}
|
||||
|
||||
if len(children) != 1 {
|
||||
t.Fatalf("got %d children, want 1", len(children))
|
||||
}
|
||||
|
||||
if err := f.store.DeleteWant(ctx, artist); err != nil {
|
||||
t.Fatalf("DeleteWant: %v", err)
|
||||
}
|
||||
|
||||
all, err := f.store.ListWants(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListWants: %v", err)
|
||||
}
|
||||
|
||||
if len(all) != 0 {
|
||||
t.Errorf("got %d wants after deleting the artist, want 0", len(all))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/database/sql/sqlcgen"
|
||||
)
|
||||
|
||||
// The wanted list's persistence. Kept apart from the request/item
|
||||
// storage in store.go because the two have opposite lifetimes: items
|
||||
// are written constantly and swept, wants are written rarely and kept.
|
||||
|
||||
// defaultDueBatch bounds how many wants one reconcile pass picks up.
|
||||
// The list can be thousands of rows after a discography backfill, and a
|
||||
// pass that tried to search all of them would take a day and annoy
|
||||
// every provider on the way.
|
||||
const defaultDueBatch = 25
|
||||
|
||||
// AddWant inserts a want, or returns the existing row's ID if the same
|
||||
// MBID is already wanted in this library. Asking twice is not two
|
||||
// wants, and re-asking must not reset a backoff that is deliberately
|
||||
// long.
|
||||
func (s *Store) AddWant(ctx context.Context, w Want) (int64, error) {
|
||||
if !w.Entity.Valid() {
|
||||
return 0, fmt.Errorf("%w: entity %q", ErrUnsupported, w.Entity)
|
||||
}
|
||||
|
||||
if w.Scope == "" {
|
||||
w.Scope = ScopeFuture
|
||||
}
|
||||
|
||||
// The MBID is the identity of a want, so it is normalized here
|
||||
// rather than at each call site: the same identifier arriving from
|
||||
// an Explore page and from a pasted URL must be one row, or the
|
||||
// uniqueness constraint that makes artist expansion idempotent
|
||||
// stops holding.
|
||||
w.MBID = strings.ToLower(strings.TrimSpace(w.MBID))
|
||||
|
||||
if w.MBID == "" {
|
||||
return 0, fmt.Errorf("%w: a want needs an MBID", ErrUnsupported)
|
||||
}
|
||||
|
||||
parent := sql.NullInt64{}
|
||||
if w.ParentID != 0 {
|
||||
parent = sql.NullInt64{Int64: w.ParentID, Valid: true}
|
||||
}
|
||||
|
||||
id, err := s.db.Queries.UpsertDownloadWant(
|
||||
ctx,
|
||||
sqlcgen.UpsertDownloadWantParams{
|
||||
Mbid: w.MBID,
|
||||
Entity: string(w.Entity),
|
||||
LibraryID: w.LibraryID,
|
||||
Artist: w.Artist,
|
||||
Title: w.Title,
|
||||
Scope: string(w.Scope),
|
||||
Secondary: boolToInt(w.Secondary),
|
||||
ParentID: parent,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("add download want: %w", err)
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetWant loads one want.
|
||||
func (s *Store) GetWant(ctx context.Context, id int64) (Want, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadWant(ctx, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Want{}, fmt.Errorf("%w: want %d", ErrNotFound, id)
|
||||
}
|
||||
|
||||
return Want{}, fmt.Errorf("get download want: %w", err)
|
||||
}
|
||||
|
||||
return wantRowToWant(row), nil
|
||||
}
|
||||
|
||||
// FindWant looks a want up by what it names rather than by row ID,
|
||||
// which is how callers holding an MBID (the Explore pages, a provider
|
||||
// sync) ask "is this already wanted?".
|
||||
func (s *Store) FindWant(
|
||||
ctx context.Context,
|
||||
mbid string,
|
||||
libraryID int64,
|
||||
) (Want, bool, error) {
|
||||
row, err := s.db.ReadQueries.GetDownloadWantByMBID(
|
||||
ctx,
|
||||
sqlcgen.GetDownloadWantByMBIDParams{Mbid: mbid, LibraryID: libraryID},
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Want{}, false, nil
|
||||
}
|
||||
|
||||
return Want{}, false, fmt.Errorf("find download want: %w", err)
|
||||
}
|
||||
|
||||
return wantRowToWant(row), true, nil
|
||||
}
|
||||
|
||||
// ListWants returns the whole wanted list, active first.
|
||||
func (s *Store) ListWants(ctx context.Context) ([]Want, error) {
|
||||
rows, err := s.db.ReadQueries.ListDownloadWants(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list download wants: %w", err)
|
||||
}
|
||||
|
||||
return wantRowsToWants(rows), nil
|
||||
}
|
||||
|
||||
// ListArtistWants returns active artist subscriptions, which are what
|
||||
// the reconciler expands.
|
||||
func (s *Store) ListArtistWants(ctx context.Context) ([]Want, error) {
|
||||
rows, err := s.db.ReadQueries.ListDownloadWantsByEntity(
|
||||
ctx,
|
||||
sqlcgen.ListDownloadWantsByEntityParams{
|
||||
Entity: string(EntityArtist),
|
||||
State: string(WantStateWanted),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list artist wants: %w", err)
|
||||
}
|
||||
|
||||
return wantRowsToWants(rows), nil
|
||||
}
|
||||
|
||||
// ListDueWants returns downloadable wants whose backoff has elapsed,
|
||||
// least-attempted first so a new addition is not stuck behind a
|
||||
// hundred long-shot retries.
|
||||
func (s *Store) ListDueWants(ctx context.Context, limit int) ([]Want, error) {
|
||||
if limit <= 0 {
|
||||
limit = defaultDueBatch
|
||||
}
|
||||
|
||||
rows, err := s.db.ReadQueries.ListDueDownloadWants(ctx, int64(limit))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list due download wants: %w", err)
|
||||
}
|
||||
|
||||
return wantRowsToWants(rows), nil
|
||||
}
|
||||
|
||||
// ListChildWants returns the wants an artist subscription produced.
|
||||
func (s *Store) ListChildWants(
|
||||
ctx context.Context,
|
||||
parentID int64,
|
||||
) ([]Want, error) {
|
||||
rows, err := s.db.ReadQueries.ListChildDownloadWants(
|
||||
ctx,
|
||||
sql.NullInt64{Int64: parentID, Valid: true},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list child download wants: %w", err)
|
||||
}
|
||||
|
||||
return wantRowsToWants(rows), nil
|
||||
}
|
||||
|
||||
// SetWantState moves a want between wanted, paused and satisfied.
|
||||
func (s *Store) SetWantState(
|
||||
ctx context.Context,
|
||||
id int64,
|
||||
state WantState,
|
||||
errText string,
|
||||
) error {
|
||||
if err := s.db.Queries.SetDownloadWantState(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadWantStateParams{
|
||||
State: string(state),
|
||||
LastError: errText,
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set download want state: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordAttempt notes an unsuccessful pass over a want and schedules
|
||||
// the next one. The want stays wanted: not finding something is a fact
|
||||
// about today's providers, not a verdict on the request.
|
||||
func (s *Store) RecordAttempt(
|
||||
ctx context.Context,
|
||||
id int64,
|
||||
attempts int,
|
||||
reason string,
|
||||
) error {
|
||||
next := nextRetry(time.Now(), attempts+1)
|
||||
|
||||
if err := s.db.Queries.RecordDownloadWantAttempt(
|
||||
ctx,
|
||||
sqlcgen.RecordDownloadWantAttemptParams{
|
||||
LastError: reason,
|
||||
NextTryAt: sql.NullTime{Time: next, Valid: true},
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("record download want attempt: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SatisfyWant marks a want as owned.
|
||||
func (s *Store) SatisfyWant(ctx context.Context, id int64) error {
|
||||
if err := s.db.Queries.SatisfyDownloadWant(ctx, id); err != nil {
|
||||
return fmt.Errorf("satisfy download want: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetWantExternalIDs records the identifiers external managers gave
|
||||
// this want in their own persistent lists.
|
||||
func (s *Store) SetWantExternalIDs(
|
||||
ctx context.Context,
|
||||
id int64,
|
||||
ids map[string]string,
|
||||
) error {
|
||||
encoded, err := json.Marshal(ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode want external ids: %w", err)
|
||||
}
|
||||
|
||||
if err := s.db.Queries.SetDownloadWantExternalIDs(
|
||||
ctx,
|
||||
sqlcgen.SetDownloadWantExternalIDsParams{
|
||||
ExternalIds: string(encoded),
|
||||
ID: id,
|
||||
},
|
||||
); err != nil {
|
||||
return fmt.Errorf("set want external ids: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteWant removes a want and, by cascade, anything an artist want
|
||||
// derived.
|
||||
func (s *Store) DeleteWant(ctx context.Context, id int64) error {
|
||||
if err := s.db.Queries.DeleteDownloadWant(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete download want: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearSatisfiedWants drops everything already owned.
|
||||
func (s *Store) ClearSatisfiedWants(ctx context.Context) error {
|
||||
if err := s.db.Queries.DeleteSatisfiedDownloadWants(ctx); err != nil {
|
||||
return fmt.Errorf("clear satisfied download wants: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// wantRowsToWants decodes a slice of stored rows.
|
||||
func wantRowsToWants(rows []sqlcgen.DownloadWant) []Want {
|
||||
out := make([]Want, 0, len(rows))
|
||||
|
||||
for _, r := range rows {
|
||||
out = append(out, wantRowToWant(r))
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// wantRowToWant decodes a stored want row. A malformed external-ID
|
||||
// blob yields an empty map rather than an error: losing the link to a
|
||||
// Lidarr row is recoverable on the next sync, making the wanted list
|
||||
// unreadable is not.
|
||||
func wantRowToWant(r sqlcgen.DownloadWant) Want {
|
||||
external := map[string]string{}
|
||||
_ = json.Unmarshal([]byte(r.ExternalIds), &external)
|
||||
|
||||
return Want{
|
||||
ID: r.ID,
|
||||
MBID: r.Mbid,
|
||||
Entity: Entity(r.Entity),
|
||||
LibraryID: r.LibraryID,
|
||||
Artist: r.Artist,
|
||||
Title: r.Title,
|
||||
Scope: WantScope(r.Scope),
|
||||
Secondary: r.Secondary != 0,
|
||||
State: WantState(r.State),
|
||||
ParentID: r.ParentID.Int64,
|
||||
Attempts: int(r.Attempts),
|
||||
LastError: r.LastError,
|
||||
LastTriedAt: r.LastTriedAt.Time,
|
||||
NextTryAt: r.NextTryAt.Time,
|
||||
ExternalIDs: external,
|
||||
CreatedAt: r.CreatedAt,
|
||||
UpdatedAt: r.UpdatedAt,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user