Merge branch 'main' into fix/small-issue-batch
CI / check (push) Skipped
CI / e2e (push) Skipped
CI / check (pull_request) Successful in 2m31s
CI / e2e (pull_request) Canceled after 0s

This commit is contained in:
2026-08-18 15:22:56 +00:00
37 changed files with 1774 additions and 382 deletions
+3 -2
View File
@@ -411,9 +411,10 @@ func (c *Config) SetDownloadPreferences(prefs download.AutoDownloadPrefs) error
formats = append(formats, string(f))
}
c.Downloads.MinFileSizeMB = prefs.MinSizeMB
c.Downloads.MinKbps = prefs.MinKbps
c.Downloads.MaxKbps = prefs.MaxKbps
c.Downloads.PreferredKbps = prefs.PreferredKbps
c.Downloads.MaxFileSizeMB = prefs.MaxSizeMB
c.Downloads.PreferredFileSizeMB = prefs.PreferredSizeMB
c.Downloads.AllowedFormats = formats
if err := c.Save(); err != nil {
+28 -11
View File
@@ -34,13 +34,29 @@ type UserConfig struct {
// in one burst that every provider sees as a flood.
WantedBatch int `toml:"WantedBatch"`
// MinFileSizeMB, MaxFileSizeMB and PreferredFileSizeMB bound and
// nudge what auto-pick (interactive or via the request list) may
// grab without asking. Zero on any of them is permissive: see
// AutoDownloadPrefs.
MinFileSizeMB int `toml:"MinFileSizeMB"`
MaxFileSizeMB int `toml:"MaxFileSizeMB"`
PreferredFileSizeMB int `toml:"PreferredFileSizeMB"`
// MinKbps, MaxKbps and PreferredKbps bound and nudge what auto-pick
// (interactive or via the request list) may grab without asking.
// Zero on any of them is permissive: see AutoDownloadPrefs.
//
// They replaced MinFileSizeMB / MaxFileSizeMB /
// PreferredFileSizeMB, which were megabytes and so said nothing
// without knowing how long the release was. The old keys are
// deliberately *not* read back: a number that meant "300 MB" cannot
// be reinterpreted as a bitrate without knowing the album it was
// aimed at, so migrating it would be inventing an intent the user
// never expressed. An existing config falls back to no window,
// which is the permissive default and matches a fresh install —
// and MaxFileSizeMB is the one that does carry over, because a
// ceiling on total bytes still means exactly what it did.
MinKbps int `toml:"MinKbps"`
MaxKbps int `toml:"MaxKbps"`
PreferredKbps int `toml:"PreferredKbps"`
// MaxFileSizeMB is a hard ceiling on a candidate's total size, kept
// in megabytes on purpose — it is a question about disk space, not
// about quality, and it has to apply to a candidate whose bitrate
// cannot be worked out at all.
MaxFileSizeMB int `toml:"MaxFileSizeMB"`
// AllowedFormats restricts auto-pick to these formats. Empty means
// no restriction. Values are Format strings ("flac", "mp3", ...).
@@ -56,10 +72,11 @@ func (c *UserConfig) AutoDownloadPrefs() AutoDownloadPrefs {
}
return AutoDownloadPrefs{
MinSizeMB: c.MinFileSizeMB,
MaxSizeMB: c.MaxFileSizeMB,
PreferredSizeMB: c.PreferredFileSizeMB,
AllowedFormats: formats,
MinKbps: c.MinKbps,
MaxKbps: c.MaxKbps,
PreferredKbps: c.PreferredKbps,
MaxSizeMB: c.MaxFileSizeMB,
AllowedFormats: formats,
}
}
+8 -10
View File
@@ -236,6 +236,12 @@ func (m *Manager) AutoPickable(dl Download, ranked []Candidate) bool {
return AutoPickable(dl, ranked, m.preferences())
}
// AutoPickVeto wraps the package function the same way, and is what the
// request list quotes back to the user.
func (m *Manager) AutoPickVeto(dl Download, ranked []Candidate) string {
return AutoPickVeto(dl, ranked, m.preferences())
}
// Reload rebuilds every provider from stored config. Called at startup
// and after any provider settings change.
//
@@ -612,16 +618,8 @@ func (m *Manager) Attempt(
return false, "", err
}
if !m.AutoPickable(dl, ranked) {
best := ranked[0]
return false, fmt.Sprintf(
"best of %d found is not a confident enough match "+
"(match %.0f%%, quality %.0f%%)",
len(ranked),
best.Match.Overall*100, //nolint:mnd // percent
best.Quality.Overall*100,
), nil
if veto := m.AutoPickVeto(dl, ranked); veto != "" {
return false, veto, nil
}
if err := m.store.CreateDownload(ctx, dl); err != nil {
+44 -6
View File
@@ -218,8 +218,17 @@ func TestManagerEndToEndAutoPick(t *testing.T) {
}, "staging was never released, or the library was never rescanned")
}
// An ambiguous result set must park for the user rather than guess.
func TestManagerWaitsWhenAmbiguous(t *testing.T) {
// Two equally good copies are not an ambiguity — they are a spare.
//
// This asserted the opposite for as long as auto-pick required 0.08 of
// daylight over the runner-up, and that rule was wrong in exactly the
// case it fired hardest: a popular album turns up several *correct*
// copies, all matching the tracklist, differing only in format and
// seeders. There is no question there about what to fetch, only about
// which copy, and the ranking already answers that — closest to the
// preferred bitrate first. A candidate does not have to be better than
// the field, only good enough on its own terms.
func TestManagerAutoPicksAmongEquallyGoodCopies(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
@@ -237,11 +246,41 @@ func TestManagerWaitsWhenAmbiguous(t *testing.T) {
t.Fatalf("Start: %v", err)
}
if f.manager.AutoPickable(dl, ranked) {
t.Fatal("two equivalent candidates must not auto-pick")
if veto := f.manager.AutoPickVeto(dl, ranked); veto != "" {
t.Fatalf("two equally good copies must auto-pick, got veto: %s", veto)
}
waitForDownloadState(t, f.store, dl.ID, StateComplete)
// Exactly one of them was fetched, not both.
if grabs := a.GrabCalls + b.GrabCalls; grabs != 1 {
t.Errorf("grabs = %d, want exactly 1", grabs)
}
}
// The user can still pick explicitly when auto-pick is not what
// happened — a candidate the ranking did not choose is still grabbable.
func TestManagerPickIsExplicit(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
a := fakeWithAlbum(1, "source-a", ".flac")
b := fakeWithAlbum(2, "source-b", ".flac")
f.manager.installProvider(Config{ID: 1, Priority: 50}, a)
f.manager.installProvider(Config{ID: 2, Priority: 50}, b)
// No tracklist: never auto-picks, so the result set parks for the
// user and Pick is the only way anything is fetched.
dl := fourTrackDownload()
dl.Expected = nil
ranked, err := f.manager.Start(context.Background(), dl)
if err != nil {
t.Fatalf("Start: %v", err)
}
// Nothing was grabbed while waiting for the user.
if a.GrabCalls != 0 || b.GrabCalls != 0 {
t.Errorf(
"grabs happened without a pick: a=%d b=%d",
@@ -258,7 +297,6 @@ func TestManagerWaitsWhenAmbiguous(t *testing.T) {
t.Errorf("stored request id = %s, want %s", stored.ID, dl.ID)
}
// The user picks the second one explicitly.
if err := f.manager.Pick(
context.Background(), dl.ID, ranked[1].ID,
); err != nil {
+313 -77
View File
@@ -1,6 +1,7 @@
package download
import (
"fmt"
"math"
"sort"
"strings"
@@ -34,38 +35,102 @@ const (
weightArtistFit = 0.12
)
// Quality sub-weights. They sum to 1.0 along with weightSizeFit below.
// Quality sub-weights. Each set sums to 1.0.
//
// There are two of them because a stated preference changes what the
// other numbers are *for*. `formatRank` and `bitrateScore` are the
// app guessing at how good a copy is — FLAC over MP3, 320 over 128 —
// and that guess exists precisely because the user has not said. Once
// they have, the guess should not outvote them: with the old single set
// a preference of 320 kbps moved a candidate's score by at most 0.05
// against the 0.42 riding on format, so asking for 320 and being handed
// a FLAC every time was the *designed* behaviour. That is the same
// fault the megabyte window had — a preference the user can express and
// the ranking can ignore.
const (
weightFormat = 0.42
weightBitrate = 0.23
weightHealth = 0.20
weightPriority = 0.10
weightSizeFit = 0.05
weightFormat = 0.42
weightBitrate = 0.23
weightHealth = 0.20
weightPriority = 0.10
weightBitrateFit = 0.05
)
// Quality sub-weights when the user has named a preferred bitrate.
// The weight comes off format and bitrate — the two proxies the
// preference replaces — and health and priority are untouched, since
// neither is a stand-in for anything the user just said.
const (
statedWeightFormat = 0.20
statedWeightBitrate = 0.10
statedWeightHealth = 0.20
statedWeightPriority = 0.10
statedWeightBitrateFit = 0.40
)
// qualityWeights picks the set, in the order scoreQuality applies them.
func qualityWeights(p AutoDownloadPrefs) (
format, bitrate, health, priority, fit float64,
) {
if p.PreferredKbps > 0 {
return statedWeightFormat,
statedWeightBitrate,
statedWeightHealth,
statedWeightPriority,
statedWeightBitrateFit
}
return weightFormat,
weightBitrate,
weightHealth,
weightPriority,
weightBitrateFit
}
// unanchoredCap bounds the match score of a free-text request. Without
// an MBID there is no tracklist to be right about, so a confident-
// looking score would be a lie — and auto-pick keys off this.
const unanchoredCap = 0.65
// AutoDownloadPrefs gates and scores what AutoPickable may choose
// without asking. Zero values are permissive: no size window and no
// format restriction.
// without asking. Zero values are permissive: no bitrate window, no
// size ceiling and no format restriction.
//
// **The window is a rate, not a size.** It used to be three numbers in
// megabytes, which cannot mean anything on their own: 300 MB is a
// generous FLAC single and a suspiciously small boxset, and the user
// setting the number has no idea which release the pipeline will
// eventually apply it to. A bitrate is the same statement normalised
// by how long the music is, so one number holds across a 9-minute EP
// and a 3-hour opera — and it is the unit the thing being described is
// actually measured in. The runtime is known for every request
// auto-pick can act on (`Download.Expected` carries per-track lengths,
// and an anchored request is the only kind that reaches here), so this
// costs no extra lookup.
type AutoDownloadPrefs struct {
// MinSizeMB and MaxSizeMB bound what auto-pick will grab. Zero
// means no bound on that side. A candidate outside the window is
// filtered out of auto-pick entirely, not merely scored down — a
// tiny "sampler" torrent or a boxset ten times the expected size is
// usually the wrong thing entirely, not a worse copy of the right
// thing.
MinSizeMB int `json:"minSizeMb"`
MaxSizeMB int `json:"maxSizeMb"`
// MinKbps and MaxKbps bound the average bitrate auto-pick will
// grab. Zero means no bound on that side. A candidate outside the
// window is filtered out of auto-pick entirely, not merely scored
// down — a 96 kbps rip of the right album is not a worse copy the
// user might accept, it is one they said not to take unattended.
//
// For reference: 320 is the top of MP3, ~5001000 is FLAC depending
// on the material, and anything under ~128 is a transcode.
MinKbps int `json:"minKbps"`
MaxKbps int `json:"maxKbps"`
// PreferredSizeMB nudges the score toward a target size within the
// min/max window (a lossless rip and a heavily-padded lossless rip
// can both pass the window). Zero disables the nudge; sizeFit then
// returns a neutral value that does not affect ranking.
PreferredSizeMB int `json:"preferredSizeMb"`
// PreferredKbps nudges the score toward a target rate within the
// window, and breaks the tie when several candidates are equally
// good matches. Zero disables the nudge; bitrateFit then returns a
// neutral value that does not affect ranking.
PreferredKbps int `json:"preferredKbps"`
// MaxSizeMB is a hard ceiling on the whole candidate, and it is
// deliberately still a size. It answers a different question from
// the window above — not "is this the quality I want" but "is this
// going to fill the disk" — and it has to hold even for a candidate
// whose bitrate cannot be worked out, which is exactly the shape a
// mislabelled boxset arrives in. Zero means no ceiling.
MaxSizeMB int `json:"maxSizeMb"`
// AllowedFormats restricts auto-pick to candidates whose audio
// files are all in one of these formats. Empty means no
@@ -74,19 +139,33 @@ type AutoDownloadPrefs struct {
}
// eligible reports whether a candidate may be auto-picked under these
// preferences: within the size window (when set) and, when a format
// list is given, every audio file in an allowed format.
func (p AutoDownloadPrefs) eligible(c Candidate) bool {
// preferences: inside the bitrate window and the size ceiling (when
// set) and, when a format list is given, every audio file in an
// allowed format.
//
// `runtimeMillis` is how long the requested release is, and 0 means
// nobody knows. An unknown runtime **passes** the bitrate window
// rather than failing it: the window is a statement about quality, and
// refusing everything the moment a tracklist is missing a length would
// turn a gap in MusicBrainz into a silent embargo. The size ceiling
// still applies, which is why it exists separately.
func (p AutoDownloadPrefs) eligible(c Candidate, runtimeMillis int64) bool {
const bytesPerMB = 1 << 20
if p.MinSizeMB > 0 && c.TotalSize < int64(p.MinSizeMB)*bytesPerMB {
return false
}
if p.MaxSizeMB > 0 && c.TotalSize > int64(p.MaxSizeMB)*bytesPerMB {
return false
}
if kbps := candidateKbps(c, runtimeMillis); kbps > 0 {
if p.MinKbps > 0 && kbps < float64(p.MinKbps) {
return false
}
if p.MaxKbps > 0 && kbps > float64(p.MaxKbps) {
return false
}
}
if len(p.AllowedFormats) == 0 {
return true
}
@@ -107,11 +186,14 @@ func (p AutoDownloadPrefs) eligible(c Candidate) bool {
// filter returns only the candidates these preferences allow to be
// auto-picked, in the same (already ranked) order.
func (p AutoDownloadPrefs) filter(ranked []Candidate) []Candidate {
func (p AutoDownloadPrefs) filter(
ranked []Candidate,
runtimeMillis int64,
) []Candidate {
out := make([]Candidate, 0, len(ranked))
for _, c := range ranked {
if p.eligible(c) {
if p.eligible(c, runtimeMillis) {
out = append(out, c)
}
}
@@ -119,32 +201,116 @@ func (p AutoDownloadPrefs) filter(ranked []Candidate) []Candidate {
return out
}
// sizeFit scores how close totalSize is to PreferredSizeMB, 0..1,
// falling off linearly as the size doubles or halves away from it.
// Returns a neutral 0.5 when no preference is set, so the absence of a
// preference does not bias ranking.
func (p AutoDownloadPrefs) sizeFit(totalSize int64) float64 {
// bitrateFit scores how close a candidate's average bitrate is to
// PreferredKbps, falling off linearly as it doubles or halves away
// from it.
//
// The range is **0.5 to 1.0, not 0 to 1**, and the floor is the point.
// This carries 0.40 of the quality score once a preference is set, so a
// span down to zero would let a preference of 320 kbps push a perfectly
// good FLAC under `minQuality` and out of auto-pick altogether —
// turning "I like 320" into "never take anything else", silently. A
// preference may promote the copy that matches it; it may not
// disqualify the others. That is what `MinKbps`/`MaxKbps` are for, and
// they say so out loud.
//
// Returns the neutral floor when no preference is set or the rate
// cannot be worked out, so neither an absent preference nor an absent
// runtime biases ranking.
func (p AutoDownloadPrefs) bitrateFit(
c Candidate,
runtimeMillis int64,
) float64 {
const (
bytesPerMB = 1 << 20
neutral = 0.5
neutral = 0.5
span = 0.5
)
if p.PreferredSizeMB <= 0 || totalSize <= 0 {
if p.PreferredKbps <= 0 {
return neutral
}
preferred := float64(p.PreferredSizeMB) * bytesPerMB
ratio := float64(totalSize) / preferred
kbps := candidateKbps(c, runtimeMillis)
if kbps <= 0 {
return neutral
}
ratio := kbps / float64(p.PreferredKbps)
if ratio < 1 {
ratio = 1 / ratio
}
// ratio is now >= 1: 1.0 is an exact match, 2.0 is double or half
// the preferred size. Falls to 0 at 2x away and beyond.
fit := 1 - (ratio - 1)
// the preferred rate, where the closeness term reaches 0.
return neutral + span*clamp01(1-(ratio-1))
}
return clamp01(fit)
// candidateKbps is a candidate's average audio bitrate, or 0 when it
// cannot be worked out.
//
// Two sources, in this order, and the order matters:
//
// - **Derived from bytes over runtime**, which is the honest one. It
// covers lossless (where a stated bitrate rarely exists), it cannot
// be lied to by a filename, and it is what the user's window means.
// Only the *audio* files count: cover scans and a log file are not
// part of the bitrate, and a folder with 30 MB of artwork would
// otherwise read as a better rip than the same music without it.
// - **The mean stated bitrate**, when the runtime is unknown. Weaker
// — a provider that parses it from an MP3 header states it and one
// that guesses from the filename also "states" it — but a number
// from the file itself beats no number at all.
func candidateKbps(c Candidate, runtimeMillis int64) float64 {
const bitsPerByte = 8
audio := c.AudioFiles()
if len(audio) == 0 {
return 0
}
if runtimeMillis > 0 {
var bytes int64
for _, f := range audio {
bytes += f.Size
}
if bytes > 0 {
// bytes×8 bits over seconds, expressed in kbps: the two
// factors of 1000 (millis→seconds, bits→kilobits) cancel.
return float64(bytes) * bitsPerByte /
float64(runtimeMillis)
}
}
var (
sum int
count int
)
for _, f := range audio {
if f.Bitrate > 0 {
sum += f.Bitrate
count++
}
}
if count == 0 {
return 0
}
return float64(sum) / float64(count)
}
// runtimeMillis is how long the requested release is, summed over its
// expected tracklist. Zero when the tracklist is absent or carries no
// lengths, which is what every caller here treats as "unknown".
func (d Download) runtimeMillis() int64 {
var total int64
for _, t := range d.Expected {
total += t.LengthMillis
}
return total
}
// Score fills a candidate's Match, Quality and Score fields.
@@ -160,7 +326,9 @@ func Score(dl Download, c Candidate, priority int, prefs AutoDownloadPrefs) Cand
c.Files = mergeMatched(c.Files, matched)
c.Match = scoreMatch(dl, c, audio, titleFit)
c.Quality = scoreQuality(c, audio, priority, prefs)
c.Quality = scoreQuality(
c, audio, priority, prefs, dl.runtimeMillis(),
)
c.Score = weightMatch*c.Match.Overall + weightQuality*c.Quality.Overall
@@ -279,11 +447,12 @@ func scoreQuality(
audio []CandidateFile,
priority int,
prefs AutoDownloadPrefs,
runtimeMillis int64,
) QualityScore {
q := QualityScore{
Health: clamp01(c.Health),
Priority: clamp01(float64(priority) / 100.0),
SizeFit: prefs.sizeFit(c.TotalSize),
Health: clamp01(c.Health),
Priority: clamp01(float64(priority) / 100.0),
BitrateFit: prefs.bitrateFit(c, runtimeMillis),
}
if len(audio) == 0 {
@@ -310,11 +479,13 @@ func scoreQuality(
q.FormatRank = worst
q.Bitrate = bitrateScore(audio)
q.Overall = weightFormat*q.FormatRank +
weightBitrate*q.Bitrate +
weightHealth*q.Health +
weightPriority*q.Priority +
weightSizeFit*q.SizeFit
wFormat, wBitrate, wHealth, wPriority, wFit := qualityWeights(prefs)
q.Overall = wFormat*q.FormatRank +
wBitrate*q.Bitrate +
wHealth*q.Health +
wPriority*q.Priority +
wFit*q.BitrateFit
if q.Mixed {
q.Overall *= 0.9
@@ -444,6 +615,19 @@ func Rank(
return out[i].Match.Overall > out[j].Match.Overall
}
// Closest to the preferred bitrate wins the tie.
//
// This is what decides which copy is taken now that auto-pick
// no longer requires the winner to be clear of the field: when
// several candidates are equally good matches of equal overall
// quality, the one the user said they wanted the shape of is
// the answer, ahead of provider priority. With no preference
// set every BitrateFit is the same neutral value and this
// falls through, exactly as before.
if out[i].Quality.BitrateFit != out[j].Quality.BitrateFit {
return out[i].Quality.BitrateFit > out[j].Quality.BitrateFit
}
if out[i].Quality.Priority != out[j].Quality.Priority {
return out[i].Quality.Priority > out[j].Quality.Priority
}
@@ -454,19 +638,58 @@ func Rank(
return out
}
// AutoPickable reports whether a ranked list has a clear enough winner
// to grab without asking. It demands an anchored request, a high match,
// decent quality, and daylight between first and second place — if two
// candidates are close, the choice is the user's.
func AutoPickable(dl Download, ranked []Candidate, prefs AutoDownloadPrefs) bool {
const (
minMatch = 0.85
minQuality = 0.5
minLead = 0.08
)
// Auto-pick gates. Named rather than inlined because AutoPickVeto
// reports which of them refused, and a number in a sentence the user
// reads should be the same number the decision used.
const (
minMatch = 0.85
minQuality = 0.5
)
if !dl.Anchored() || len(ranked) == 0 {
return false
// AutoPickable reports whether a ranked list has a candidate worth
// grabbing without asking: an anchored request with a tracklist behind
// it, and a candidate that clears the match and quality bars inside the
// user's guardrails.
//
// **It does not require the winner to be better than the runner-up.**
// It used to demand 0.08 of daylight on the combined score, which meant
// the check fired hardest in the case it was never written for: a
// popular album turns up five *correct* copies, all matching the
// tracklist at 95%+ and differing only in format and seeders, their
// scores land within a point of each other, and auto-pick refused
// forever on the grounds that the choice was the user's. It was not.
// There was no question about *what* to fetch, only about which copy —
// and abundance is the one condition under which that question matters
// least. A candidate does not need to be the best one, only one that
// meets the criteria; where several do, `Rank` puts the one closest to
// the preferred bitrate first.
func AutoPickable(dl Download, ranked []Candidate, prefs AutoDownloadPrefs) bool {
return AutoPickVeto(dl, ranked, prefs) == ""
}
// AutoPickVeto returns the reason auto-pick declined, or "" when it
// would go ahead.
//
// It exists because "it rejected all of them" was indistinguishable
// from "it found nothing good". The request list's message was built
// from `ranked[0]` — the best candidate *before* the size and format
// guardrails, and before the lead check — so a request refused because
// the user's maximum size excluded every copy, or because three equally
// good copies were found, reported "best of 12 found is not a confident
// enough match (match 96%, quality 88%)". Numbers that clear both
// thresholds, beside a refusal, is a message that teaches the user the
// matcher is broken. Each gate names itself now.
func AutoPickVeto(
dl Download,
ranked []Candidate,
prefs AutoDownloadPrefs,
) string {
if len(ranked) == 0 {
return "nothing found"
}
if !dl.Anchored() {
return "the request is free text, so there is no release to be right about"
}
// An anchor with no tracklist behind it is an anchor in name only:
@@ -474,29 +697,42 @@ func AutoPickable(dl Download, ranked []Candidate, prefs AutoDownloadPrefs) bool
// is exactly the evidence a wrong-album candidate also has. This
// matters most for the request list, where nobody is watching.
if len(dl.Expected) == 0 {
return false
return "no tracklist for this release is known yet, so a candidate cannot be checked against it"
}
// The guardrails apply before the match/quality/lead checks: a
// candidate outside the allowed size or format is not a worse
// choice, it is not a choice auto-pick may make at all, so it must
// not count as "the winner" nor as "second place" for the lead
// check below.
eligible := prefs.filter(ranked)
// The guardrails apply before the match and quality checks: a
// candidate outside the allowed bitrate, size or format is not a
// worse choice, it is not a choice auto-pick may make at all, so it
// must not count as "the winner" either.
eligible := prefs.filter(ranked, dl.runtimeMillis())
if len(eligible) == 0 {
return false
return fmt.Sprintf(
"all %d found are outside the auto-download bitrate, size or format limits",
len(ranked),
)
}
best := eligible[0]
if best.Match.Overall < minMatch || best.Quality.Overall < minQuality {
return false
if best.Match.Overall < minMatch {
return fmt.Sprintf(
"best of %d found matches this release only %.0f%% (needs %.0f%%)",
len(ranked),
best.Match.Overall*100, //nolint:mnd // percent
minMatch*100, //nolint:mnd // percent
)
}
if len(eligible) > 1 && best.Score-eligible[1].Score < minLead {
return false
if best.Quality.Overall < minQuality {
return fmt.Sprintf(
"best of %d found is the right release but scores %.0f%% on quality (needs %.0f%%)",
len(ranked),
best.Quality.Overall*100, //nolint:mnd // percent
minQuality*100, //nolint:mnd // percent
)
}
return true
return ""
}
// mergeMatched copies MatchedTo assignments from the audio-only slice
+360 -57
View File
@@ -1,6 +1,34 @@
package download
import "testing"
import (
"strings"
"testing"
)
// trackMillis is five minutes; okComputer's four of them make a
// twenty-minute release, which is what turns a candidate's byte count
// into a bitrate the assertions below can name.
const trackMillis = 5 * 60 * 1000
// okComputerRuntime is that release's runtime, for the helpers that
// need it directly.
const okComputerRuntime = 4 * trackMillis
// kbpsCandidate builds an annotated candidate whose audio adds up to
// the given average bitrate over okComputer's runtime.
func kbpsCandidate(id, ext string, kbps int) Candidate {
// bits = kbps × 1000 × (runtimeMillis / 1000), so the thousands
// cancel and the byte count is kbps × runtimeMillis / 8.
const bitsPerByte = 8
total := int64(kbps) * okComputerRuntime / bitsPerByte
c := candidateFor(id, allTitles(), ext, total/int64(len(allTitles())))
c.Files = AnnotateFiles(c.Files)
c.TotalSize = total
return c
}
// okComputer is the reference request used across ranking tests.
func okComputer() Download {
@@ -8,11 +36,15 @@ func okComputer() Download {
ReleaseMBID: "mbid-ok-computer",
Artist: "Radiohead",
Album: "OK Computer",
// Four five-minute tracks: twenty minutes, so a candidate's
// bitrate is a number these tests can state exactly. Without
// lengths there is no runtime and the bitrate window has
// nothing to divide by.
Expected: []ExpectedTrack{
{Position: 1, Title: "Airbag"},
{Position: 2, Title: "Paranoid Android"},
{Position: 3, Title: "Subterranean Homesick Alien"},
{Position: 4, Title: "Exit Music (For a Film)"},
{Position: 1, Title: "Airbag", LengthMillis: trackMillis},
{Position: 2, Title: "Paranoid Android", LengthMillis: trackMillis},
{Position: 3, Title: "Subterranean Homesick Alien", LengthMillis: trackMillis},
{Position: 4, Title: "Exit Music (For a Film)", LengthMillis: trackMillis},
},
}
}
@@ -187,7 +219,7 @@ func TestUnanchoredMatchIsCapped(t *testing.T) {
}
}
func TestAutoPickableRequiresAnchorAndLead(t *testing.T) {
func TestAutoPickableRequiresAnchorAndTracklist(t *testing.T) {
t.Parallel()
dl := okComputer()
@@ -211,14 +243,18 @@ func TestAutoPickableRequiresAnchorAndLead(t *testing.T) {
}
})
t.Run("two close candidates are not", func(t *testing.T) {
// Two identical copies are a spare, not an ambiguity. This
// asserted the opposite while auto-pick required daylight over the
// runner-up — a rule that made abundance the thing that stopped a
// request being satisfied, which is backwards.
t.Run("two equally good candidates still are", func(t *testing.T) {
t.Parallel()
twin := best
twin.ID = "twin"
if AutoPickable(dl, []Candidate{best, twin}, AutoDownloadPrefs{}) {
t.Error("identical candidates must not auto-pick")
if !AutoPickable(dl, []Candidate{best, twin}, AutoDownloadPrefs{}) {
t.Error("identical good candidates must auto-pick")
}
})
@@ -300,18 +336,11 @@ func TestProviderPriorityBreaksTies(t *testing.T) {
}
}
const mb = 1 << 20
func TestAutoDownloadPrefsEligible(t *testing.T) {
t.Parallel()
flacCandidate := candidateFor("c", allTitles(), ".flac", 30_000_000)
flacCandidate.Files = AnnotateFiles(flacCandidate.Files)
flacCandidate.TotalSize = 300 * mb
mp3Candidate := candidateFor("c", allTitles(), ".mp3", 3_000_000)
mp3Candidate.Files = AnnotateFiles(mp3Candidate.Files)
mp3Candidate.TotalSize = 30 * mb
flacCandidate := kbpsCandidate("c", ".flac", 900)
mp3Candidate := kbpsCandidate("c", ".mp3", 128)
tests := []struct {
name string
@@ -321,18 +350,25 @@ func TestAutoDownloadPrefsEligible(t *testing.T) {
}{
{"zero value is permissive", AutoDownloadPrefs{}, flacCandidate, true},
{
"within min/max window",
AutoDownloadPrefs{MinSizeMB: 100, MaxSizeMB: 500},
"within the bitrate window",
AutoDownloadPrefs{MinKbps: 320, MaxKbps: 1200},
flacCandidate, true,
},
{
"below minimum",
AutoDownloadPrefs{MinSizeMB: 400},
"below the minimum bitrate",
AutoDownloadPrefs{MinKbps: 500},
mp3Candidate, false,
},
{
"above the maximum bitrate",
AutoDownloadPrefs{MaxKbps: 500},
flacCandidate, false,
},
{
"above maximum",
AutoDownloadPrefs{MaxSizeMB: 200},
// The ceiling is bytes, not a rate, and it is the guard
// that still works when the bitrate cannot be worked out.
"above the hard size ceiling",
AutoDownloadPrefs{MaxSizeMB: 50},
flacCandidate, false,
},
{
@@ -351,57 +387,131 @@ func TestAutoDownloadPrefsEligible(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := tt.prefs.eligible(tt.c); got != tt.want {
got := tt.prefs.eligible(tt.c, okComputerRuntime)
if got != tt.want {
t.Errorf("eligible() = %v, want %v", got, tt.want)
}
})
}
}
// A release nobody knows the length of cannot be judged on bitrate, and
// the window must not become a silent embargo because MusicBrainz is
// missing a track length. The size ceiling still applies — that is why
// it is a separate field.
func TestBitrateWindowPassesAnUnknownRuntime(t *testing.T) {
t.Parallel()
c := kbpsCandidate("c", ".mp3", 128)
prefs := AutoDownloadPrefs{MinKbps: 900}
if !prefs.eligible(c, 0) {
t.Error("an unknown runtime must pass the bitrate window")
}
if prefs.eligible(c, okComputerRuntime) {
t.Error("a known runtime must still be judged")
}
ceiling := AutoDownloadPrefs{MaxSizeMB: 1}
if ceiling.eligible(c, 0) {
t.Error("the size ceiling must apply even with no runtime")
}
}
// Artwork is not part of the bitrate. A folder carrying 30 MB of
// scans would otherwise read as a better rip than the same music
// without them, which is backwards.
func TestBitrateIgnoresNonAudioFiles(t *testing.T) {
t.Parallel()
c := kbpsCandidate("c", ".mp3", 320)
bare := candidateKbps(c, okComputerRuntime)
c.Files = append(c.Files, CandidateFile{
Path: "Radiohead - OK Computer/cover.jpg",
Size: 30 << 20,
})
c.Files = AnnotateFiles(c.Files)
if got := candidateKbps(c, okComputerRuntime); got != bare {
t.Errorf("bitrate with artwork = %f, want %f", got, bare)
}
}
// Where no runtime is known, a stated per-file bitrate is better than
// no answer at all.
func TestBitrateFallsBackToTheStatedRate(t *testing.T) {
t.Parallel()
c := candidateFor("c", allTitles(), ".mp3", 3_000_000)
for i := range c.Files {
c.Files[i].Bitrate = 192
}
c.Files = AnnotateFiles(c.Files)
if got := candidateKbps(c, 0); got != 192 {
t.Errorf("stated bitrate = %f, want 192", got)
}
}
func TestAutoDownloadPrefsFilter(t *testing.T) {
t.Parallel()
small := candidateFor("small", allTitles(), ".flac", 10_000_000)
small.TotalSize = 50 * mb
lossy := kbpsCandidate("lossy", ".mp3", 128)
lossless := kbpsCandidate("lossless", ".flac", 900)
big := candidateFor("big", allTitles(), ".flac", 30_000_000)
big.TotalSize = 500 * mb
prefs := AutoDownloadPrefs{MinKbps: 500}
prefs := AutoDownloadPrefs{MinSizeMB: 100, MaxSizeMB: 600}
filtered := prefs.filter(
[]Candidate{lossy, lossless}, okComputerRuntime,
)
filtered := prefs.filter([]Candidate{small, big})
if len(filtered) != 1 || filtered[0].ID != "big" {
if len(filtered) != 1 || filtered[0].ID != "lossless" {
t.Errorf("filter() = %v, want only the in-window candidate", filtered)
}
}
func TestAutoDownloadPrefsSizeFit(t *testing.T) {
func TestAutoDownloadPrefsBitrateFit(t *testing.T) {
t.Parallel()
const neutral = 0.5
tests := []struct {
name string
prefs AutoDownloadPrefs
totalSize int64
want float64
name string
prefs AutoDownloadPrefs
c Candidate
want float64
}{
{"no preference is neutral", AutoDownloadPrefs{}, 300 * mb, neutral},
{
"no preference is neutral",
AutoDownloadPrefs{},
kbpsCandidate("c", ".flac", 900), neutral,
},
{
"exact match scores 1",
AutoDownloadPrefs{PreferredSizeMB: 300},
300 * mb, 1.0,
AutoDownloadPrefs{PreferredKbps: 320},
kbpsCandidate("c", ".mp3", 320), 1.0,
},
{
"double the preferred size scores 0",
AutoDownloadPrefs{PreferredSizeMB: 300},
600 * mb, 0.0,
// The floor is neutral, not zero: this term carries 0.40
// of the quality score once a preference is set, and a
// span to zero would let "I like 320" quietly disqualify
// every FLAC from auto-pick.
"double the preferred rate falls to the neutral floor",
AutoDownloadPrefs{PreferredKbps: 320},
kbpsCandidate("c", ".flac", 640), neutral,
},
{
"half the preferred size scores 0",
AutoDownloadPrefs{PreferredSizeMB: 300},
150 * mb, 0.0,
"half the preferred rate falls to the neutral floor",
AutoDownloadPrefs{PreferredKbps: 320},
kbpsCandidate("c", ".mp3", 160), neutral,
},
{
"an unknowable rate is neutral",
AutoDownloadPrefs{PreferredKbps: 320},
kbpsCandidate("c", ".mp3", 320), neutral,
},
}
@@ -409,30 +519,223 @@ func TestAutoDownloadPrefsSizeFit(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := tt.prefs.sizeFit(tt.totalSize); got != tt.want {
t.Errorf("sizeFit(%d) = %f, want %f", tt.totalSize, got, tt.want)
// The last case deliberately withholds the runtime.
runtime := int64(okComputerRuntime)
if tt.name == "an unknowable rate is neutral" {
runtime = 0
}
if got := tt.prefs.bitrateFit(tt.c, runtime); got != tt.want {
t.Errorf("bitrateFit() = %f, want %f", got, tt.want)
}
})
}
}
// An otherwise-perfect candidate must not auto-pick when it falls
// outside the configured size guard: the guardrail applies before the
// match/quality/lead checks, not as one more input averaged into them.
func TestAutoPickableRejectsCandidateOutsideSizeGuard(t *testing.T) {
// outside the configured guardrails: they apply before the match and
// quality checks, not as one more input averaged into them.
func TestAutoPickableRejectsCandidateOutsideTheGuardrails(t *testing.T) {
t.Parallel()
dl := okComputer()
best := Score(dl, candidateFor("a", allTitles(), ".flac", 30_000_000), 50, AutoDownloadPrefs{})
best.TotalSize = 500 * mb
best := Score(dl, kbpsCandidate("a", ".flac", 900), 50, AutoDownloadPrefs{})
if !AutoPickable(dl, []Candidate{best}, AutoDownloadPrefs{}) {
t.Fatal("expected this candidate to be auto-pickable with no guardrails")
}
tight := AutoDownloadPrefs{MinSizeMB: 10, MaxSizeMB: 100}
if AutoPickable(dl, []Candidate{best}, AutoDownloadPrefs{MaxKbps: 320}) {
t.Error("candidate above the bitrate window must not auto-pick")
}
if AutoPickable(dl, []Candidate{best}, tight) {
t.Error("candidate outside the size guard must not auto-pick")
if AutoPickable(dl, []Candidate{best}, AutoDownloadPrefs{MaxSizeMB: 1}) {
t.Error("candidate above the size ceiling must not auto-pick")
}
}
// The refusal has to name the gate that refused.
//
// Before AutoPickVeto, every one of these came back as the same
// sentence built from `ranked[0]` — the best candidate before the size
// and format guardrails — so a request refused because the user's size
// window excluded every copy reported a match and a quality that both
// cleared their thresholds. A refusal quoting numbers that pass is
// what made the matcher look broken from outside.
func TestAutoPickVetoNamesTheGate(t *testing.T) {
t.Parallel()
dl := okComputer()
best := Score(
dl,
candidateFor("a", allTitles(), ".flac", 30_000_000),
50,
AutoDownloadPrefs{},
)
// candidateFor sizes the files and leaves TotalSize at 0, which is
// what the guardrails read.
sized := func(c Candidate, total int64) Candidate {
c.TotalSize = total
return c
}
tests := []struct {
name string
dl Download
ranked []Candidate
prefs AutoDownloadPrefs
wantSub string
}{
{
name: "nothing found",
dl: dl,
ranked: nil,
wantSub: "nothing found",
},
{
name: "free text",
dl: Download{Artist: "Radiohead", Album: "OK Computer"},
ranked: []Candidate{best},
wantSub: "free text",
},
{
name: "no tracklist behind the anchor",
dl: Download{
ReleaseMBID: "mbid-ok-computer",
Artist: "Radiohead",
Album: "OK Computer",
},
ranked: []Candidate{best},
wantSub: "no tracklist",
},
{
// The candidate is 120 MB and the window tops out at 1 MB:
// the old message reported its match and quality instead.
name: "outside the size window",
dl: dl,
ranked: []Candidate{sized(best, 120<<20)},
prefs: AutoDownloadPrefs{MaxSizeMB: 1},
wantSub: "bitrate, size or format limits",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := AutoPickVeto(tt.dl, tt.ranked, tt.prefs)
if !strings.Contains(got, tt.wantSub) {
t.Errorf("veto = %q, want it to mention %q", got, tt.wantSub)
}
})
}
}
// A clear winner has no veto at all — the sentence is empty, which is
// what AutoPickable reads.
func TestAutoPickVetoIsEmptyForAClearWinner(t *testing.T) {
t.Parallel()
dl := okComputer()
best := Score(
dl,
candidateFor("a", allTitles(), ".flac", 30_000_000),
50,
AutoDownloadPrefs{},
)
weak := Score(
dl,
candidateFor("b", allTitles()[:2], ".mp3", 1_000_000),
50,
AutoDownloadPrefs{},
)
if got := AutoPickVeto(dl, []Candidate{best, weak}, AutoDownloadPrefs{}); got != "" {
t.Errorf("veto = %q, want none", got)
}
}
// With several candidates that all clear the bar, the preferred
// bitrate decides which one is taken.
//
// This is what replaced the daylight requirement. Auto-pick no longer
// refuses when the field is close; it takes the copy nearest the shape
// the user asked for, which is the question they actually answered in
// Settings.
func TestPreferredBitrateBreaksTheTie(t *testing.T) {
t.Parallel()
dl := okComputer()
prefs := AutoDownloadPrefs{PreferredKbps: 320}
// Same album, same completeness, same health, same provider — the
// only difference between them is the rate.
lossless := kbpsCandidate("lossless", ".flac", 900)
perfect := kbpsCandidate("perfect", ".mp3", 320)
ranked := Rank(
dl, []Candidate{lossless, perfect}, nil, prefs,
)
if ranked[0].ID != "perfect" {
t.Errorf(
"winner = %q (fit %f) over %q (fit %f), want the 320 kbps copy",
ranked[0].ID, ranked[0].Quality.BitrateFit,
ranked[1].ID, ranked[1].Quality.BitrateFit,
)
}
if AutoPickVeto(dl, ranked, prefs) != "" {
t.Error("a close field must still auto-pick")
}
}
// With no preference set, nothing changes: BitrateFit is the same
// neutral value for every candidate and the older tie-breaks decide.
func TestNoPreferredBitrateLeavesRankingAlone(t *testing.T) {
t.Parallel()
dl := okComputer()
lossless := kbpsCandidate("lossless", ".flac", 900)
lossy := kbpsCandidate("lossy", ".mp3", 320)
ranked := Rank(
dl, []Candidate{lossy, lossless}, nil, AutoDownloadPrefs{},
)
if ranked[0].ID != "lossless" {
t.Errorf(
"winner = %q, want the lossless copy on format alone",
ranked[0].ID,
)
}
}
// A preferred bitrate promotes the copy that matches it and must never
// disqualify the ones that do not. It carries 0.40 of the quality
// score, so a fit spanning down to zero would put a perfectly good FLAC
// under minQuality and out of auto-pick — turning a preference into a
// prohibition without saying so. MinKbps and MaxKbps are how a user
// says that on purpose.
func TestAPreferredBitrateNeverDisqualifies(t *testing.T) {
t.Parallel()
dl := okComputer()
far := AutoDownloadPrefs{PreferredKbps: 128}
lossless := Score(dl, kbpsCandidate("flac", ".flac", 900), 50, far)
if lossless.Quality.Overall < minQuality {
t.Errorf(
"quality = %f under a far-off preference, want >= %f",
lossless.Quality.Overall, minQuality,
)
}
if veto := AutoPickVeto(dl, []Candidate{lossless}, far); veto != "" {
t.Errorf("a far-off preference vetoed the candidate: %s", veto)
}
}
+18 -2
View File
@@ -36,13 +36,24 @@ func newServiceFixture(t *testing.T) serviceFixture {
// assertion read it; the second is that same goroutine still writing
// into `t.TempDir()` after the test returned. One cause, two shapes.
//
// Putting the candidate outside the auto-pick size window stops the
// Putting the candidate outside the auto-pick guardrails stops the
// grab from ever starting, which is better than waiting for it: there
// is no goroutine to be slow, so the tests state what they mean
// ("the request exists, in this state") without a timing assumption
// underneath. A test that does want the download has `managerFixture`
// and sets its own preferences.
mf.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 1})
//
// The guard is a *format* the fake never produces, and it used to be
// `MaxSizeMB: 1`, which never fired: the size gates read
// `Candidate.TotalSize`, which real providers fill and the fake
// leaves at zero, and zero is under every ceiling. So the grab went
// ahead anyway and the second failure shape above — the TempDir
// cleanup race — kept happening, reproducibly, roughly one run in
// fifteen. A guard has to be keyed on something the fixture
// actually sets.
mf.manager.SetPreferences(AutoDownloadPrefs{
AllowedFormats: []Format{FormatWMA},
})
return serviceFixture{managerFixture: mf, svc: svc}
}
@@ -182,6 +193,11 @@ func TestManualDownloadSatisfiesRequestOnSuccess(t *testing.T) {
f := newServiceFixture(t)
ctx := context.Background()
// This is the one test here that is *about* the download, so it
// undoes the fixture's guard rather than relying on it — which is
// what it was doing implicitly while the guard did not work.
f.manager.SetPreferences(AutoDownloadPrefs{})
provider := fakeWithAlbum(1, "source", ".flac")
f.manager.installProvider(Config{ID: 1, Priority: 50}, provider)
+6 -1
View File
@@ -302,7 +302,12 @@ type QualityScore struct {
Bitrate float64 `json:"bitrate"`
Health float64 `json:"health"` // seeders, free slots
Priority float64 `json:"priority"` // user's per-provider preference
SizeFit float64 `json:"sizeFit"` // closeness to the preferred download size
// BitrateFit is closeness to the preferred *rate*, which is what
// the auto-download window is expressed in. It replaced a
// `SizeFit` measured in megabytes: a size means nothing without
// knowing how long the music is, so the same number described a
// generous single and a suspiciously small boxset.
BitrateFit float64 `json:"bitrateFit"`
// Mixed marks a candidate whose files are not all the same format,
// which usually means a hand-assembled folder rather than a rip.
+49 -5
View File
@@ -25,8 +25,26 @@ const (
// where cached cover art thumbnails are stored.
thumbnailDir = CoverArtCacheDirName
// thumbnailTimeout is the HTTP timeout for fetching a thumbnail.
thumbnailTimeout = 10 * time.Second
// thumbnailTimeout is the HTTP timeout for fetching a thumbnail,
// and it has to cover a redirect the Cover Art Archive does not
// serve itself.
//
// `coverartarchive.org` answers `front-250` with a 307 to an
// Internet Archive storage node (`dn######.us.archive.org`), and
// those nodes are routinely slow: measured against the twelve
// albums on Explore's own shelves, a successful fetch took 1416 s
// and a failing one 1317 s. At 10 s *every* cover on the page
// timed out — 24 cards, 5 of which had art, all of those from the
// disk cache — which reads as "Explore has no album art" rather
// than as a slow upstream, because a timeout writes nothing and
// says nothing.
//
// 30 s is chosen to clear that measured range with room, not to be
// generous: the fetch is off the critical path (each one is its own
// goroutine behind an 8/s limiter, and the frontend renders a
// placeholder until it lands), so the cost of waiting is nothing
// and the cost of giving up early is a blank page.
thumbnailTimeout = 30 * time.Second
// thumbnailMaxSize is the maximum image size to cache (2 MB).
thumbnailMaxSize = 2 * 1024 * 1024
@@ -97,6 +115,20 @@ func (p *CoverArtProxy) GetThumbnail(
return ""
}
// A 404 is an answer, and it is already on disk.
//
// `writeCache(mbid, nil)` has recorded "the archive has no art for
// this" as an empty file since this was written, and nothing has
// ever read it back: `readCache` returns "" for an empty file,
// which is indistinguishable from a miss, so every art-less release
// group was re-fetched from the network on every render that asked
// about it. On Explore's shelves a third of the cards are art-less,
// so that was a third of the page spending a live CAA request to be
// told again what the last one said.
if p.knownMissing(releaseGroupMBID) {
return ""
}
// Source 3: fetch from Cover Art Archive (slow, cached to disk).
url := CoverArtGroupURL(releaseGroupMBID)
data, cacheable, err := p.fetch(url)
@@ -177,8 +209,9 @@ func (p *CoverArtProxy) GetCandidateThumbnail(
}
}
// Network fetch on release group.
if releaseGroupMBID != "" {
// Network fetch on release group — unless a previous one was told
// there is none. See `knownMissing`.
if releaseGroupMBID != "" && !p.knownMissing(releaseGroupMBID) {
url := CoverArtGroupURL(releaseGroupMBID)
data, cacheable, err := p.fetch(url)
@@ -194,7 +227,7 @@ func (p *CoverArtProxy) GetCandidateThumbnail(
}
// Network fetch on release (fallback).
if releaseMBID != "" {
if releaseMBID != "" && !p.knownMissing(releaseMBID) {
url := CoverArtURL(releaseMBID)
data, cacheable, err := p.fetch(url)
@@ -285,6 +318,17 @@ func (p *CoverArtProxy) cachePath(mbid string) string {
return filepath.Join(p.cacheDir, mbid+".jpg")
}
// knownMissing reports whether a previous fetch was told the archive
// has no art for this MBID — the empty file `writeCache(mbid, nil)`
// leaves behind. It is deliberately separate from `readCache`, which
// answers "what are the bytes" and cannot express the difference
// between no answer and an answer of none.
func (p *CoverArtProxy) knownMissing(mbid string) bool {
info, err := os.Stat(p.cachePath(mbid))
return err == nil && info.Size() == 0
}
func (p *CoverArtProxy) readCache(mbid string) string {
path := p.cachePath(mbid)
+141 -15
View File
@@ -289,8 +289,13 @@ func (l *Library) scanInternal(
l.mu.Unlock()
}()
// The configured mode, not a hardcoded "auto". `ScanConcurrency`
// has been a validated config field with three values and one
// caller passing a constant, so choosing `ssd` or `hdd` by hand
// did nothing at all.
diskProfile := system.ProfileForPath(libraryPath)
workerCount := resolveScanWorkerCount(
ScanConcurrencyAuto,
l.conf.ScanConcurrency,
libraryPath,
)
@@ -300,6 +305,10 @@ func (l *Library) scanInternal(
"libraryName", libraryName,
"libraryPath", libraryPath,
"workers", workerCount,
"mode", l.conf.ScanConcurrency,
"device", diskProfile.Device,
"rotational", diskProfile.Rotational,
"queueDepth", diskProfile.QueueDepth,
)
// Helper to build a ScanProgress with library identification.
@@ -818,7 +827,7 @@ func (l *Library) scanInternal(
g := new(errgroup.Group)
g.SetLimit(workerCount)
for work := range workChan {
for work := range readaheadWork(scanCtx, workChan, diskProfile) {
g.Go(func() error {
if err := l.waitIfPaused(scanCtx); err != nil {
return err
@@ -1285,9 +1294,101 @@ func surveyAudioFiles(
return count, maxModTime
}
// hddWorkerCount is the maximum number of concurrent extraction
// workers when the library resides on a spinning disk.
const hddWorkerCount = 2
// How many extraction workers a spinning disk gets, and why it is two
// numbers rather than one.
//
// Extraction is not CPU work — every parser here reads headers and
// returns — so on a spinning disk the whole cost is seek latency, and
// the only question worth asking is how many reads should be in flight
// at once. That has two different right answers and the drive says
// which:
//
// - A drive with command queueing (NCQ: /sys/block/<dev>/device/
// queue_depth reports 31 or 32 on any SATA disk with it enabled)
// reorders outstanding reads into the order its head passes over
// them. Handing it several at once is most of why a parallel scan
// beats a serial one at all, and four is where the returns flatten:
// the drive needs a few requests to have anything to reorder, and
// past that it is queueing requests it was already going to
// service in that order.
// - A drive without it — queue_depth 1, which is what a USB bridge
// or a pre-2004 disk reports — services one command at a time in
// the order given. Every extra worker there is one more seek
// competing for one head, and the scan gets *slower* the harder it
// is pushed. Two is kept rather than one because the readahead
// hints (see readaheadWork) do the overlapping that concurrency
// was standing in for, and one worker cannot hide a stall.
//
// This used to be a flat 2 for anything rotational, which is a
// pre-NCQ assumption: it left a modern spinning disk with a quarter of
// the queue depth it can use.
const (
hddWorkerCountQueued = 4
hddWorkerCountSerial = 2
)
// Readahead tuning.
const (
// readaheadDepth is how many files ahead of the workers the
// prefetcher runs. It is the channel's buffer, so it is also the
// number of `WILLNEED` hints outstanding at once — comfortably more
// than a queueing drive's 32-command window is worth filling with
// one library, and small enough that a cancelled scan is not
// holding a long tail of queued reads.
readaheadDepth = 16
// readaheadBytes is how much of each file to pull in. Everything
// the scanner reads lives at the head: ID3v2 and FLAC's
// STREAMINFO/VORBIS_COMMENT/PICTURE blocks, and the first MPEG
// frame with its Xing header. 512 KB covers a tag carrying
// embedded cover art, which is the large case — and reading a
// little too much sequentially costs a spinning disk almost
// nothing next to the seek that got there.
readaheadBytes = 512 << 10
)
// readaheadWork forwards scan work while asking the kernel to fetch
// each file's header before a worker reaches it.
//
// The buffered channel *is* the lookahead: this goroutine runs ahead
// of the workers until the buffer fills, hinting every file as it goes,
// so by the time a worker takes an item the read it needs has been in
// flight for `readaheadDepth` files' worth of parsing. That is the
// only thing that helps a spinning disk here, because the per-file work
// is already header-only — every parser in `backend/metadata` reads a
// few hundred bytes and returns, so the scan is not waiting on CPU or
// on bytes, it is waiting on the head to arrive.
//
// It runs on rotational disks only. An SSD has no seek to hide and
// already has one worker per core; issuing hints there is pure syscall
// overhead against an OS readahead that is already ahead of us.
func readaheadWork(
ctx context.Context,
in <-chan scanWork,
profile system.DiskProfile,
) <-chan scanWork {
if !profile.Rotational {
return in
}
out := make(chan scanWork, readaheadDepth)
go func() {
defer close(out)
for work := range in {
hintReadahead(work.absolutePath, readaheadBytes)
select {
case out <- work:
case <-ctx.Done():
return
}
}
}()
return out
}
// resolveScanWorkerCount returns the number of concurrent
// extraction workers based on the configured concurrency mode
@@ -1296,20 +1397,45 @@ func resolveScanWorkerCount(
mode ScanConcurrency,
libraryPath string,
) int {
return workersForProfile(
mode,
system.ProfileForPath(libraryPath),
goruntime.NumCPU(),
)
}
// workersForProfile is the policy on its own, so it can be tested
// against drives this machine does not have.
//
// `hdd` and `ssd` override what the device says rather than being a
// separate branch: the mode is the user overruling detection, and
// detection is right about the queue depth either way — a user who
// picks `hdd` on a queueing drive still wants that drive's queue used.
func workersForProfile(
mode ScanConcurrency,
profile system.DiskProfile,
cpus int,
) int {
spinning := profile.Rotational
switch mode {
case ScanConcurrencySSD:
return goruntime.NumCPU()
spinning = false
case ScanConcurrencyHDD:
return min(hddWorkerCount, goruntime.NumCPU())
default: // auto
if system.IsRotationalDisk(libraryPath) {
return min(
hddWorkerCount, goruntime.NumCPU(),
)
}
return goruntime.NumCPU()
spinning = true
case ScanConcurrencyAuto:
}
if !spinning {
return cpus
}
workers := hddWorkerCountSerial
if profile.Queues() {
workers = hddWorkerCountQueued
}
return min(workers, cpus)
}
// scanWork represents a file to be processed by a worker.
+38
View File
@@ -0,0 +1,38 @@
//go:build linux
package library
import (
"os"
"golang.org/x/sys/unix"
)
// hintReadahead asks the kernel to start fetching the head of a file
// that is about to be read.
//
// `POSIX_FADV_WILLNEED` returns immediately and queues the read, which
// is the whole point: on a spinning disk the first access to a file
// costs a seek of several milliseconds, and that latency can only be
// hidden by having the next seek already in flight while the current
// file is being parsed. A drive with command queueing can then service
// the queued reads in head order rather than in the order they were
// asked for.
//
// Errors are dropped on purpose. This is a hint: a file that has since
// been deleted, a filesystem that does not implement fadvise, or a
// permission the walk saw and this open does not, all mean "no
// prefetch", never "fail the scan". The read that follows is what
// reports a genuine problem.
func hintReadahead(path string, bytes int64) {
f, err := os.Open(path)
if err != nil {
return
}
defer func() { _ = f.Close() }()
_ = unix.Fadvise(
int(f.Fd()), 0, bytes, unix.FADV_WILLNEED,
)
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !linux
package library
// hintReadahead is a no-op off Linux.
//
// macOS has `F_RDADVISE` and Windows has `FILE_FLAG_SEQUENTIAL_SCAN`,
// and neither is wired up here for the reason the scan concurrency
// heuristic is not either: this package cannot tell a spinning disk
// from an SSD on those platforms (see system.ProfileForPath), so it
// would be prefetching without knowing whether prefetching is what the
// device wants.
func hintReadahead(_ string, _ int64) {}
+122
View File
@@ -0,0 +1,122 @@
package library
import (
"context"
"testing"
"yellowjacket/backend/system"
)
// How many workers a scan gets is decided by two facts about the
// device, and the second one is new: a spinning disk that can queue
// commands wants several reads in flight, and one that cannot wants
// almost none. Before this it was a flat 2 for anything rotational,
// which is a pre-NCQ assumption — a modern SATA disk reports a queue
// depth of 32 and was being given a quarter of what it can use.
func TestWorkersForProfile(t *testing.T) {
t.Parallel()
const cpus = 16
ssd := system.DiskProfile{Device: "sda", QueueDepth: 32}
hddQueued := system.DiskProfile{
Device: "sdb", Rotational: true, QueueDepth: 32,
}
hddSerial := system.DiskProfile{
Device: "sdc", Rotational: true, QueueDepth: 1,
}
// Neither NVMe nor a device-mapper volume publishes queue_depth.
// An unknown depth must not be read as "cannot queue", or every
// such device would be scanned as if it were a 2003 drive.
unknown := system.DiskProfile{Device: "dm-0", Rotational: true}
tests := []struct {
name string
mode ScanConcurrency
profile system.DiskProfile
want int
}{
{"ssd auto", ScanConcurrencyAuto, ssd, cpus},
{"queueing hdd auto", ScanConcurrencyAuto, hddQueued, hddWorkerCountQueued},
{"serial hdd auto", ScanConcurrencyAuto, hddSerial, hddWorkerCountSerial},
{"unknown depth queues", ScanConcurrencyAuto, unknown, hddWorkerCountQueued},
// The mode overrules detection about the *disk*, never about
// its queue: forcing hdd on a queueing drive still uses it.
{"forced hdd on an ssd", ScanConcurrencyHDD, ssd, hddWorkerCountQueued},
{"forced ssd on an hdd", ScanConcurrencySSD, hddQueued, cpus},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := workersForProfile(tt.mode, tt.profile, cpus); got != tt.want {
t.Errorf(
"workersForProfile(%q, %+v) = %d, want %d",
tt.mode, tt.profile, got, tt.want,
)
}
})
}
}
// A machine with fewer cores than the policy asks for gets its cores.
func TestWorkersNeverExceedTheCPUCount(t *testing.T) {
t.Parallel()
hdd := system.DiskProfile{Rotational: true, QueueDepth: 32}
if got := workersForProfile(ScanConcurrencyAuto, hdd, 1); got != 1 {
t.Errorf("single-core hdd = %d workers, want 1", got)
}
}
// The prefetch stage must forward every item and nothing else: it is a
// pass-through with a side effect, and a scan that drops a file because
// of a *hint* would be a spectacular way to lose part of a library.
func TestReadaheadForwardsEveryFile(t *testing.T) {
t.Parallel()
in := make(chan scanWork, 4)
for _, p := range []string{"/a", "/b", "/c", "/d"} {
in <- scanWork{absolutePath: p}
}
close(in)
var got []string
for w := range readaheadWork(
context.Background(),
in,
system.DiskProfile{Rotational: true, QueueDepth: 32},
) {
got = append(got, w.absolutePath)
}
want := []string{"/a", "/b", "/c", "/d"}
if len(got) != len(want) {
t.Fatalf("forwarded %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Errorf("item %d = %q, want %q", i, got[i], want[i])
}
}
}
// On an SSD the stage is not inserted at all — the channel comes back
// unchanged, so a scan there pays nothing for a feature it cannot use.
func TestReadaheadIsSkippedOnSolidState(t *testing.T) {
t.Parallel()
in := make(chan scanWork)
out := readaheadWork(
context.Background(), in, system.DiskProfile{QueueDepth: 32},
)
if out != (<-chan scanWork)(in) {
t.Error("an ssd must get the original channel, unwrapped")
}
}
+134 -57
View File
@@ -16,32 +16,107 @@ var errNoBlockDevice = errors.New(
"no matching block device found",
)
// IsRotationalDisk reports whether the block device backing the
// given path is a rotational (spinning) disk. Detection uses the
// Linux sysfs interface at /sys/block/<dev>/queue/rotational.
// Returns false on any error (assumes SSD).
func IsRotationalDisk(path string) bool {
dev, err := deviceForPath(path)
if err != nil {
return false
}
// DiskProfile is what the scanner needs to know about the device a
// library sits on. Both fields are about the same question — how many
// reads should be in flight at once — and they answer different halves
// of it, so they travel together rather than as two probes.
type DiskProfile struct {
// Device is the whole-disk kernel name ("sdb"), or "" when the
// path could not be resolved to one.
Device string
rotational, err := os.ReadFile(
filepath.Join(
"/sys/block", dev, "queue", "rotational",
),
)
if err != nil {
return false
}
// Rotational is /sys/block/<dev>/queue/rotational: true for a
// spinning disk, where a seek costs milliseconds.
Rotational bool
return strings.TrimSpace(string(rotational)) == "1"
// QueueDepth is /sys/block/<dev>/device/queue_depth — how many
// commands the drive will accept and reorder at once. This is
// NCQ: a SATA disk with it enabled reports 31 or 32, and one
// without reports 1. Zero means the file was not there to read,
// which is the case for anything that is not a SCSI/SATA device
// (NVMe, MMC, device-mapper, loop, a VM's virtio disk).
//
// It is the difference between concurrency helping and hurting.
// With queueing, several outstanding reads let the drive service
// them in the order its head passes over them, which is most of
// why a parallel scan is faster at all. Without it, every extra
// worker is one more seek competing for one head, and the scan
// gets slower the harder it is pushed.
QueueDepth int
}
// deviceForPath resolves a filesystem path to its underlying block
// device name (e.g. "sda") by matching the device major:minor
// from stat(2) against /sys/block/ entries.
func deviceForPath(path string) (string, error) {
// Queues reports whether the drive can reorder outstanding commands.
//
// An unknown depth (0) counts as queueing: everything that does not
// publish this file is a device where concurrency is fine — NVMe has
// its own queues, virtio and device-mapper are not the physical layer
// at all. The only case worth being careful about is the one that
// says so explicitly.
func (p DiskProfile) Queues() bool {
return p.QueueDepth != 1
}
// IsRotationalDisk reports whether the block device backing the
// given path is a rotational (spinning) disk. Returns false on any
// error (assumes SSD).
func IsRotationalDisk(path string) bool {
return ProfileForPath(path).Rotational
}
// ProfileForPath describes the device backing a filesystem path. A
// path that cannot be resolved yields the zero profile, which reads as
// "not rotational, queueing" — the permissive answer, since assuming a
// spinning disk on an SSD would halve a scan for nothing.
func ProfileForPath(path string) DiskProfile {
dev, err := diskForPath(path)
if err != nil {
return DiskProfile{}
}
return DiskProfile{
Device: dev,
Rotational: sysfsInt(dev, "queue", "rotational") == 1,
QueueDepth: sysfsInt(dev, "device", "queue_depth"),
}
}
// sysfsInt reads one small integer out of /sys/block/<dev>/<parts...>,
// returning 0 when it is absent or unparseable. Every attribute here
// is optional: sysfs layout varies by driver, and a missing file is
// "this device does not say", never an error worth propagating.
func sysfsInt(dev string, parts ...string) int {
p := filepath.Join(
append([]string{"/sys/block", dev}, parts...)...,
)
data, err := os.ReadFile(p) //nolint:gosec // sysfs, name from the kernel
if err != nil {
return 0
}
n, err := strconv.Atoi(strings.TrimSpace(string(data)))
if err != nil {
return 0
}
return n
}
// diskForPath resolves a filesystem path to the *whole disk* backing
// it — "sdb" for a file on "sdb3".
//
// It goes through /sys/dev/block/<major>:<minor>, which the kernel
// maintains as a symlink to the device's own sysfs directory, and then
// walks up to the parent when that directory turns out to be a
// partition. The previous implementation scanned /sys/block comparing
// dev numbers and, failing an exact match, took the first entry whose
// *major* agreed — and every SATA disk shares major 8. So a library on
// /dev/sdb3 resolved to whatever /sys/block listed first, which is
// alphabetical, which is sda. On the machine this was found on that
// meant a 6 TB spinning disk was read as the SSD next to it and scanned
// with one worker per core. Matching on major alone cannot be right
// whenever a machine has two disks, which is the case this exists for.
func diskForPath(path string) (string, error) {
var st syscall.Stat_t
if err := syscall.Stat(path, &st); err != nil {
return "", fmt.Errorf(
@@ -49,48 +124,50 @@ func deviceForPath(path string) (string, error) {
)
}
// Extract major and minor device numbers.
major := (st.Dev >> 8) & 0xff
minor := st.Dev & 0xff
// Linux packs dev_t as 12 bits of major and 20 of minor, split
// across the word. Masking the low byte of each — which is what
// this used to do — is right only for the first 256 of either.
major := unixMajor(uint64(st.Dev))
minor := unixMinor(uint64(st.Dev))
// Scan /sys/block/ for a matching device.
entries, err := os.ReadDir("/sys/block")
link := filepath.Join(
"/sys/dev/block",
strconv.FormatUint(major, 10)+":"+
strconv.FormatUint(minor, 10),
)
target, err := filepath.EvalSymlinks(link)
if err != nil {
return "", fmt.Errorf(
"could not read /sys/block: %w", err,
"%w: %s (%w)", errNoBlockDevice, link, err,
)
}
majorStr := strconv.FormatUint(major, 10)
devStr := majorStr + ":" +
strconv.FormatUint(minor, 10)
// A partition's directory sits inside its disk's, and only the
// disk carries `queue`. Climb at most one level: sysfs nests a
// partition exactly one deep under its disk.
name := filepath.Base(target)
for _, entry := range entries {
devFile := filepath.Join(
"/sys/block", entry.Name(), "dev",
)
data, err := os.ReadFile(devFile)
if err != nil {
continue
}
content := strings.TrimSpace(string(data))
if content == devStr {
return entry.Name(), nil
}
// The filesystem might be on a partition (e.g. sda1)
// whose parent block device is sda. Check if the
// major number matches.
parts := strings.SplitN(content, ":", 2)
if len(parts) == 2 && parts[0] == majorStr {
return entry.Name(), nil
}
if _, err := os.Stat(filepath.Join(target, "queue")); err != nil {
name = filepath.Base(filepath.Dir(target))
}
return "", fmt.Errorf(
"%w for %s", errNoBlockDevice, devStr,
)
if name == "" || name == "." || name == string(filepath.Separator) {
return "", fmt.Errorf(
"%w for %d:%d", errNoBlockDevice, major, minor,
)
}
return name, nil
}
// unixMajor and unixMinor decode a Linux dev_t. Spelled out rather
// than taken from golang.org/x/sys/unix so this file stays readable
// beside the encoding it is undoing.
func unixMajor(dev uint64) uint64 {
return (dev>>8)&0xfff | (dev >> 32 & ^uint64(0xfff))
}
func unixMinor(dev uint64) uint64 {
return dev&0xff | (dev >> 12 & ^uint64(0xff))
}
+25
View File
@@ -2,9 +2,34 @@
package system
// DiskProfile is what the scanner needs to know about the device a
// library sits on. See the Linux implementation for what each field
// means; off Linux nothing fills them, because neither macOS nor
// Windows publishes an equivalent of sysfs's `rotational` and
// `queue_depth` without going through platform APIs this package
// deliberately does not link.
type DiskProfile struct {
Device string
Rotational bool
QueueDepth int
}
// Queues reports whether the drive can reorder outstanding commands.
// Always true here: an unknown depth is the permissive answer, and
// assuming otherwise would halve every scan on every Mac.
func (p DiskProfile) Queues() bool {
return p.QueueDepth != 1
}
// IsRotationalDisk reports whether the block device backing the
// given path is a rotational (spinning) disk. On non-Linux
// platforms this always returns false (assumes SSD).
func IsRotationalDisk(_ string) bool {
return false
}
// ProfileForPath describes the device backing a filesystem path. Off
// Linux that is the zero profile, which reads as "an SSD that queues".
func ProfileForPath(_ string) DiskProfile {
return DiskProfile{}
}