configurable scan parallelism based on hdd or sdd

This commit is contained in:
2026-02-19 13:59:02 -05:00
parent 81793975b4
commit d9ebd38382
19 changed files with 1579 additions and 142 deletions
+49
View File
@@ -144,6 +144,10 @@ func (c *Config) applyDefaults() {
} else {
c.Window.applyDefaults()
}
if c.Library != nil {
c.Library.ApplyDefaults()
}
}
// SetContext sets the Wails runtime context for event emission.
@@ -171,6 +175,11 @@ func (c *Config) SetLibraryDirectory(dir string) error {
)
}
// Preserve existing scan concurrency setting.
if c.Library != nil {
newLibConf.ScanConcurrency = c.Library.ScanConcurrency
}
c.Library = newLibConf
if err := c.Save(); err != nil {
@@ -196,3 +205,43 @@ func (c *Config) SetLibraryDirectory(dir string) error {
return nil
}
// GetScanConcurrency returns the configured scan concurrency mode.
func (c *Config) GetScanConcurrency() string {
if c.Library == nil {
return string(library.DefaultScanConcurrency)
}
return string(c.Library.ScanConcurrency)
}
// SetScanConcurrency validates and saves a new scan concurrency
// mode. The change takes effect on the next scan.
func (c *Config) SetScanConcurrency(mode string) error {
if c.Library == nil {
c.Library = &library.Config{}
c.Library.ApplyDefaults()
}
c.Library.ScanConcurrency = library.ScanConcurrency(
mode,
)
if err := c.Library.Validate(); err != nil {
return fmt.Errorf(
"invalid scan concurrency mode: %w", err,
)
}
if err := c.Save(); err != nil {
return fmt.Errorf(
"could not save config: %w", err,
)
}
c.logger.Info(
"scan concurrency updated", "mode", mode,
)
return nil
}
+64 -6
View File
@@ -7,11 +7,39 @@ import (
"os"
)
var errNotDirectory = errors.New("path is not a directory")
var (
errNotDirectory = errors.New("path is not a directory")
errUnknownScanConcurrency = errors.New("unknown scan concurrency mode")
)
// ScanConcurrency controls how many parallel workers the scanner
// uses for metadata extraction. The choice directly affects I/O
// throughput on spinning disks vs SSDs.
type ScanConcurrency string
// Valid ScanConcurrency modes.
const (
// ScanConcurrencyAuto detects whether the library resides on
// a rotational disk and chooses workers accordingly.
ScanConcurrencyAuto ScanConcurrency = "auto"
// ScanConcurrencySSD uses runtime.NumCPU() workers, maximising
// throughput on solid-state storage.
ScanConcurrencySSD ScanConcurrency = "ssd"
// ScanConcurrencyHDD uses a small number of workers to limit
// I/O contention on spinning disks.
ScanConcurrencyHDD ScanConcurrency = "hdd"
)
// DefaultScanConcurrency is the mode used when no value is
// configured.
const DefaultScanConcurrency = ScanConcurrencyAuto
// Config holds Library config data.
type Config struct {
DirectoryPath Directory `form:"Directory" schema:"directory,required"`
DirectoryPath Directory `toml:"DirectoryPath"`
ScanConcurrency ScanConcurrency `toml:"ScanConcurrency"`
}
// Directory represents a filesystem path to a music directory.
@@ -23,24 +51,54 @@ func NewConfig(dir string) (*Config, error) {
DirectoryPath: Directory(dir),
}
if err := config.Validate(); err != nil {
return nil, fmt.Errorf("validation error for new library config: %w", err)
return nil, fmt.Errorf(
"validation error for new library config: %w",
err,
)
}
return config, nil
}
// Validate checks that the configured directory exists.
// ApplyDefaults fills zero-value fields with sensible defaults.
func (c *Config) ApplyDefaults() {
if c.ScanConcurrency == "" {
c.ScanConcurrency = DefaultScanConcurrency
}
}
// Validate checks that the configured directory exists and that
// the scan concurrency mode is recognised.
func (c *Config) Validate() error {
c.ApplyDefaults()
if len(c.DirectoryPath) != 0 {
dirInfo, err := os.Stat(string(c.DirectoryPath))
if err != nil {
return fmt.Errorf("problem getting info on library dir (%s): %w", c.DirectoryPath, err)
return fmt.Errorf(
"problem getting info on library dir (%s): %w",
c.DirectoryPath, err,
)
}
if !dirInfo.IsDir() {
return fmt.Errorf("%s: %w", c.DirectoryPath, errNotDirectory)
return fmt.Errorf(
"%s: %w", c.DirectoryPath, errNotDirectory,
)
}
}
switch c.ScanConcurrency {
case ScanConcurrencyAuto,
ScanConcurrencySSD,
ScanConcurrencyHDD:
// Valid.
default:
return fmt.Errorf(
"%w: %q", errUnknownScanConcurrency,
c.ScanConcurrency,
)
}
return nil
}
+105 -12
View File
@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/image/draw"
@@ -28,6 +29,14 @@ type thumbnailTier struct {
Quality int
}
// thumbnailWork is a unit of work for the async thumbnail worker pool.
type thumbnailWork struct {
imgData []byte
dir string
hashStr string
metrics *ScanMetrics
}
// thumbnailTiers lists all generated size variants, ordered smallest to largest.
var thumbnailTiers = []thumbnailTier{
{Suffix: "_sm", MaxSize: 100, Quality: 75},
@@ -56,14 +65,21 @@ func isSizedVariant(name string) bool {
}
// saveCoverArt saves embedded cover art to the cache directory.
// Returns the file path where the art was saved, or empty string if no picture data.
// Returns the file path where the art was saved, or empty string
// if no picture data. Timing is recorded in the provided metrics.
// When thumbChan is non-nil, thumbnail generation is dispatched
// asynchronously to a worker pool instead of running inline.
func (l *Library) saveCoverArt(
pic *metadata.PictureData,
metrics *ScanMetrics,
thumbChan chan<- thumbnailWork,
) (string, error) {
if pic == nil || len(pic.Data) == 0 {
return "", nil
}
saveStart := time.Now()
// Get the data directory for storing cover art.
dataDir, err := system.GetUserDataDirPath()
if err != nil {
@@ -113,19 +129,31 @@ func (l *Library) saveCoverArt(
)
}
metrics.addCoverArtSave(time.Since(saveStart))
l.logger.Debug(
"saved cover art",
"path", filePath, "size", len(pic.Data),
)
// Generate all sized variants alongside the original.
if err := l.generateSizedVariants(
pic.Data, coverDir, hashStr,
); err != nil {
l.logger.Warn(
"could not generate sized variants",
"path", filePath, "err", err,
)
// Dispatch thumbnail generation to the async worker pool
// if available, otherwise generate inline.
if thumbChan != nil {
thumbChan <- thumbnailWork{
imgData: pic.Data,
dir: coverDir,
hashStr: hashStr,
metrics: metrics,
}
} else {
if err := l.generateSizedVariantsWithMetrics(
pic.Data, coverDir, hashStr, metrics,
); err != nil {
l.logger.Warn(
"could not generate sized variants",
"path", filePath, "err", err,
)
}
}
return filePath, nil
@@ -144,6 +172,73 @@ func (l *Library) generateSizedVariants(
)
}
l.generateTiersFromImage(src, dir, hashStr)
return nil
}
// generateSizedVariantsWithMetrics is like generateSizedVariants
// but records per-tier timing in the provided metrics.
func (l *Library) generateSizedVariantsWithMetrics(
imgData []byte,
dir, hashStr string,
metrics *ScanMetrics,
) error {
src, _, err := image.Decode(bytes.NewReader(imgData))
if err != nil {
return fmt.Errorf(
"could not decode image for thumbnails: %w", err,
)
}
bounds := src.Bounds()
srcW := bounds.Dx()
srcH := bounds.Dy()
for _, tier := range thumbnailTiers {
tierStart := time.Now()
tierPath := filepath.Join(
dir,
fmt.Sprintf("%s%s.jpg", hashStr, tier.Suffix),
)
w, h := fitDimensions(srcW, srcH, tier.MaxSize)
if err := encodeAndSaveImage(
src, tierPath, w, h, tier.Quality,
); err != nil {
l.logger.Warn(
"could not generate sized variant",
"tier", tier.Suffix,
"path", tierPath,
"err", err,
)
continue
}
metrics.addThumbnailTier(
tier.Suffix, time.Since(tierStart),
)
l.logger.Debug(
"saved sized variant",
"tier", tier.Suffix,
"path", tierPath,
"dimensions", fmt.Sprintf("%dx%d", w, h),
)
}
return nil
}
// generateTiersFromImage creates all thumbnail tiers from an
// already-decoded image.
func (l *Library) generateTiersFromImage(
src image.Image,
dir, hashStr string,
) {
bounds := src.Bounds()
srcW := bounds.Dx()
srcH := bounds.Dy()
@@ -176,8 +271,6 @@ func (l *Library) generateSizedVariants(
"dimensions", fmt.Sprintf("%dx%d", w, h),
)
}
return nil
}
// fitDimensions calculates the output dimensions that fit within maxSize
@@ -206,7 +299,7 @@ func encodeAndSaveImage(
w, h, quality int,
) error {
dst := image.NewRGBA(image.Rect(0, 0, w, h))
draw.CatmullRom.Scale(
draw.ApproxBiLinear.Scale(
dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil,
)
+257 -63
View File
@@ -14,6 +14,7 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"github.com/wailsapp/wails/v2/pkg/runtime"
"golang.org/x/sync/errgroup"
@@ -22,6 +23,7 @@ import (
"yellowjacket/backend/database/sql/sqlcgen"
"yellowjacket/backend/events"
"yellowjacket/backend/metadata"
"yellowjacket/backend/system"
)
var errLibraryDirNotConfigured = errors.New("library directory not configured")
@@ -148,24 +150,38 @@ func (l *Library) registerEventHandlers() {
}
// Scan syncs the library by adding new files and removing deleted ones.
// Files that exist but have incomplete metadata (recording_id = 0) will be updated.
func (l *Library) Scan() error {
// Files that exist but have incomplete metadata (recording_id = 0)
// will be updated. The returned ScanMetrics contains timing and
// count data for every phase of the scan.
func (l *Library) Scan() (*ScanMetrics, error) {
metrics := newScanMetrics()
scanStart := time.Now()
if len(l.conf.DirectoryPath) == 0 {
return metrics, errLibraryDirNotConfigured
}
workerCount := resolveScanWorkerCount(
l.conf.ScanConcurrency,
string(l.conf.DirectoryPath),
)
l.logger.Info(
"beginning library scan", "workers", scanWorkerCount,
"beginning library scan",
"workers", workerCount,
"concurrencyMode", l.conf.ScanConcurrency,
)
runtime.EventsEmit(l.ctx, events.LibraryScanStarted)
if len(l.conf.DirectoryPath) == 0 {
return errLibraryDirNotConfigured
}
// --- Phase 1: load existing files from DB ---
loadStart := time.Now()
// Load existing file paths from the database into a sync.Map for concurrent access.
// The map tracks path → audioFile; entries are removed as files are "seen" during the walk.
// Any entries remaining after the walk are orphans (files deleted from disk).
existingFiles, err := l.db.Queries.GetAllAudioFiles(l.ctx)
if err != nil {
return fmt.Errorf("could not load existing audio files: %w", err)
return metrics, fmt.Errorf(
"could not load existing audio files: %w", err,
)
}
existingPaths := &sync.Map{}
@@ -173,6 +189,8 @@ func (l *Library) Scan() error {
existingPaths.Store(f.FilePath, f)
}
metrics.LoadExisting = time.Since(loadStart)
l.logger.Debug(
"loaded existing files from database",
"count", len(existingFiles),
@@ -189,16 +207,25 @@ func (l *Library) Scan() error {
var errMu sync.Mutex
// Walker goroutine: traverse directory and send work items to workers
// --- Phase 2: directory walk ---
walkStart := time.Now()
go func() {
defer close(workChan)
defer func() {
metrics.WalkDuration = time.Since(walkStart)
close(workChan)
}()
walkErr := fs.WalkDir(
os.DirFS(basePath),
".",
func(path string, d fs.DirEntry, err error) error {
if err != nil {
l.logger.Error("problem walking directory", "path", path, "err", err)
l.logger.Error(
"problem walking directory",
"path", path, "err", err,
)
return nil // continue walking
}
@@ -207,7 +234,9 @@ func (l *Library) Scan() error {
return nil
}
absoluteFilePath := filepath.Join(basePath, path)
absoluteFilePath := filepath.Join(
basePath, path,
)
fileExt := filepath.Ext(d.Name())
fileType, isSupportedAudioFile := metadata.GetSupportedFileType(fileExt)
@@ -215,13 +244,15 @@ func (l *Library) Scan() error {
return nil
}
// Check if file already exists in database
// Check if file already exists in database.
if existing, exists := existingPaths.LoadAndDelete(absoluteFilePath); exists {
audioFile := existing.(sqlcgen.AudioFile)
// Check if this file needs metadata update (recording_id = 0)
if audioFile.RecordingID == 0 {
l.logger.Debug("file needs metadata update", "path", absoluteFilePath)
l.logger.Debug(
"file needs metadata update",
"path", absoluteFilePath,
)
select {
case workChan <- scanWork{
@@ -248,11 +279,16 @@ func (l *Library) Scan() error {
return nil
}
l.logger.Debug("queueing file for import", "path", absoluteFilePath)
l.logger.Debug(
"queueing file for import",
"path", absoluteFilePath,
)
// Send to workers for processing
select {
case workChan <- scanWork{absolutePath: absoluteFilePath, fileType: fileType}:
case workChan <- scanWork{
absolutePath: absoluteFilePath,
fileType: fileType,
}:
case <-l.ctx.Done():
return l.ctx.Err()
}
@@ -265,15 +301,44 @@ func (l *Library) Scan() error {
errMu.Lock()
scanErr = errors.Join(
scanErr,
fmt.Errorf("problem walking library directory: %w", walkErr),
fmt.Errorf(
"problem walking library directory: %w",
walkErr,
),
)
errMu.Unlock()
}
}()
// DB writer goroutine: serialize all database writes to avoid SQLite
// contention. Results are committed in batches to amortize the cost
// of SQLite's fsync-per-commit.
// --- Thumbnail worker pool (async, decoupled from DB writer) ---
thumbChan := make(chan thumbnailWork, 100)
var thumbWg sync.WaitGroup
for range workerCount {
thumbWg.Add(1)
go func() {
defer thumbWg.Done()
for work := range thumbChan {
if err := l.generateSizedVariantsWithMetrics(
work.imgData,
work.dir,
work.hashStr,
work.metrics,
); err != nil {
l.logger.Warn(
"could not generate thumbnails",
"hash", work.hashStr,
"err", err,
)
}
}
}()
}
// --- Phase 4: DB writer goroutine ---
var dbWg sync.WaitGroup
dbWg.Add(1)
@@ -283,53 +348,79 @@ func (l *Library) Scan() error {
cache := newEntityCache()
var batch []importResult
var (
batch []importResult
dbStarted bool
dbStartVal time.Time
)
flushBatch := func() {
if len(batch) == 0 {
return
}
batchStart := time.Now()
if batchErr := l.commitBatch(
batch, cache, &added, &updated,
batch, cache, metrics,
&added, &updated,
thumbChan,
); batchErr != nil {
errMu.Lock()
scanErr = errors.Join(scanErr, batchErr)
errMu.Unlock()
}
metrics.BatchCommits += time.Since(batchStart)
batch = batch[:0]
}
for result := range resultChan {
if !dbStarted {
dbStartVal = time.Now()
dbStarted = true
}
batch = append(batch, result)
if len(batch) >= scanBatchSize {
flushBatch()
}
}
// Flush any remaining results.
flushBatch()
if dbStarted {
metrics.DBWritesWallClock = time.Since(
dbStartVal,
)
}
}()
// Worker pool: extract metadata concurrently, send results to DB writer
// --- Phase 3: worker pool ---
extractStart := time.Now()
g := new(errgroup.Group)
g.SetLimit(scanWorkerCount)
g.SetLimit(workerCount)
for work := range workChan {
g.Go(func() error {
result, err := l.extractAudioMetadata(work)
result, err := l.extractAudioMetadata(
work, metrics,
)
if err != nil {
l.logger.Warn("failed to extract metadata", "path", work.absolutePath, "err", err)
l.logger.Warn(
"failed to extract metadata",
"path", work.absolutePath,
"err", err,
)
errMu.Lock()
scanErr = errors.Join(scanErr, err)
errMu.Unlock()
return nil // continue processing other files
return nil
}
// Send to DB writer
select {
case resultChan <- result:
case <-l.ctx.Done():
@@ -340,21 +431,40 @@ func (l *Library) Scan() error {
})
}
_ = g.Wait() // Wait for all metadata extraction to complete
_ = g.Wait()
close(resultChan) // Signal DB writer to finish
dbWg.Wait() // Wait for all DB writes to complete
metrics.ExtractionWallClock = time.Since(extractStart)
close(resultChan)
dbWg.Wait()
// Close thumbnail channel and wait for all thumbnail workers
// to finish. The DB writer has stopped sending work at this
// point so it is safe to close.
thumbStart := time.Now()
close(thumbChan)
thumbWg.Wait()
metrics.ThumbnailWallClock = time.Since(thumbStart)
// --- Phase 5: orphan cleanup ---
orphanStart := time.Now()
// Orphan cleanup: any entries remaining in existingPaths are files deleted from disk
var removed atomic.Int64
existingPaths.Range(func(key, value any) bool {
path := key.(string)
audioFile := value.(sqlcgen.AudioFile)
l.logger.Debug("removing orphaned database entry", "path", path, "id", audioFile.ID)
l.logger.Debug(
"removing orphaned database entry",
"path", path, "id", audioFile.ID,
)
if err := l.db.Queries.DeleteAudioFile(l.ctx, audioFile.ID); err != nil {
if err := l.db.Queries.DeleteAudioFile(
l.ctx, audioFile.ID,
); err != nil {
l.logger.Warn(
"failed to delete orphaned audio file",
"path", path,
@@ -370,8 +480,11 @@ func (l *Library) Scan() error {
return true
})
// Generate sized variants for any cover art missing them,
// and migrate legacy _thumb files.
metrics.OrphanCleanup = time.Since(orphanStart)
// --- Phase 6: post-scan variant generation ---
variantStart := time.Now()
if err := l.generateMissingSizedVariants(); err != nil {
l.logger.Warn(
"could not generate missing sized variants",
@@ -379,23 +492,58 @@ func (l *Library) Scan() error {
)
}
metrics.PostScanVariants = time.Since(variantStart)
// --- Finalize ---
metrics.Added = added.Load()
metrics.Updated = updated.Load()
metrics.Skipped = skipped.Load()
metrics.Removed = removed.Load()
metrics.Total = time.Since(scanStart)
l.logger.Info(
"library scan complete",
"added", added.Load(),
"updated", updated.Load(),
"removed", removed.Load(),
"skipped", skipped.Load(),
"added", metrics.Added,
"updated", metrics.Updated,
"removed", metrics.Removed,
"skipped", metrics.Skipped,
"total", metrics.Total,
"library", l.conf.DirectoryPath,
)
runtime.EventsEmit(l.ctx, events.LibraryScanComplete)
runtime.EventsEmit(
l.ctx, events.LibraryScanComplete, metrics,
)
return scanErr
return metrics, scanErr
}
// scanWorkerCount controls the number of concurrent file processors.
// TODO: make configurable via Config.
var scanWorkerCount = goruntime.NumCPU()
// hddWorkerCount is the maximum number of concurrent extraction
// workers when the library resides on a spinning disk.
const hddWorkerCount = 2
// resolveScanWorkerCount returns the number of concurrent
// extraction workers based on the configured concurrency mode
// and the storage type of the library directory.
func resolveScanWorkerCount(
mode ScanConcurrency,
libraryPath string,
) int {
switch mode {
case ScanConcurrencySSD:
return goruntime.NumCPU()
case ScanConcurrencyHDD:
return min(hddWorkerCount, goruntime.NumCPU())
default: // auto
if system.IsRotationalDisk(libraryPath) {
return min(
hddWorkerCount, goruntime.NumCPU(),
)
}
return goruntime.NumCPU()
}
}
// scanWork represents a file to be processed by a worker.
type scanWork struct {
@@ -417,8 +565,12 @@ type importResult struct {
}
// extractAudioMetadata reads and extracts metadata from an audio file.
// It opens the file once, extracting both tags and duration in a single pass.
func (l *Library) extractAudioMetadata(work scanWork) (importResult, error) {
// It opens the file once, extracting both tags and duration in a
// single pass, and records per-file timing in the shared metrics.
func (l *Library) extractAudioMetadata(
work scanWork,
metrics *ScanMetrics,
) (importResult, error) {
result := importResult{
absolutePath: work.absolutePath,
fileType: work.fileType,
@@ -429,9 +581,18 @@ func (l *Library) extractAudioMetadata(work scanWork) (importResult, error) {
// Skip duration decode if we already have it from a previous import.
skipDuration := work.needsUpdate && work.existingLength > 0
tags, lengthMillis, err := metadata.ExtractAllMetadata(
tags, lengthMillis, timing, err := metadata.ExtractAllMetadata(
work.absolutePath, skipDuration,
)
if timing != nil {
metrics.addExtraction(
string(work.fileType),
timing.TagExtraction,
timing.DurationExtraction,
)
}
if err != nil {
return result, fmt.Errorf(
"could not extract metadata for %s: %w",
@@ -454,11 +615,14 @@ func (l *Library) extractAudioMetadata(work scanWork) (importResult, error) {
// commitBatch wraps a slice of import results in a single database
// transaction, creating all related records and audio file entries.
// Individual file failures are logged and accumulated but do not
// abort the entire batch.
// abort the entire batch. thumbChan dispatches thumbnail generation
// to the async worker pool.
func (l *Library) commitBatch(
batch []importResult,
cache *entityCache,
metrics *ScanMetrics,
added, updated *atomic.Int64,
thumbChan chan<- thumbnailWork,
) error {
tx, err := l.db.BeginTx()
if err != nil {
@@ -475,12 +639,18 @@ func (l *Library) commitBatch(
var saveErr error
if result.needsUpdate {
saveErr = l.updateAudioFileMetadata(txq, cache, *result)
saveErr = l.updateAudioFileMetadata(
txq, cache, metrics, *result,
thumbChan,
)
if saveErr == nil {
updated.Add(1)
}
} else {
saveErr = l.saveAudioFile(txq, cache, *result)
saveErr = l.saveAudioFile(
txq, cache, metrics, *result,
thumbChan,
)
if saveErr == nil {
added.Add(1)
}
@@ -511,7 +681,9 @@ func (l *Library) commitBatch(
func (l *Library) saveAudioFile(
q *sqlcgen.Queries,
cache *entityCache,
metrics *ScanMetrics,
result importResult,
thumbChan chan<- thumbnailWork,
) error {
l.logger.Debug(
"saving audio file to db",
@@ -526,7 +698,9 @@ func (l *Library) saveAudioFile(
)
// Process metadata and create related records.
recordingID, err := l.processMetadata(q, cache, result)
recordingID, err := l.processMetadata(
q, cache, metrics, result, thumbChan,
)
if err != nil {
return fmt.Errorf("could not process metadata: %w", err)
}
@@ -560,7 +734,9 @@ func (l *Library) saveAudioFile(
func (l *Library) updateAudioFileMetadata(
q *sqlcgen.Queries,
cache *entityCache,
metrics *ScanMetrics,
result importResult,
thumbChan chan<- thumbnailWork,
) error {
l.logger.Debug(
"updating audio file metadata",
@@ -569,7 +745,9 @@ func (l *Library) updateAudioFileMetadata(
)
// Process metadata and create related records.
recordingID, err := l.processMetadata(q, cache, result)
recordingID, err := l.processMetadata(
q, cache, metrics, result, thumbChan,
)
if err != nil {
return fmt.Errorf("could not process metadata: %w", err)
}
@@ -596,10 +774,14 @@ func (l *Library) updateAudioFileMetadata(
// and returns the recording ID. It uses the provided queries object
// (which may be transaction-scoped) and the entity cache to avoid
// redundant upserts for repeated artist/album/cover-art values.
// When thumbChan is non-nil, thumbnail generation is dispatched
// asynchronously.
func (l *Library) processMetadata(
q *sqlcgen.Queries,
cache *entityCache,
metrics *ScanMetrics,
result importResult,
thumbChan chan<- thumbnailWork,
) (int64, error) {
tags := result.tags
if tags == nil {
@@ -607,7 +789,9 @@ func (l *Library) processMetadata(
}
// 1. Handle cover art (if present).
coverArtID := l.processCoverArt(q, cache, tags)
coverArtID := l.processCoverArt(
q, cache, metrics, tags, thumbChan,
)
// 2. Get or create artist credit for track artist.
artistName := tags.Artist
@@ -681,17 +865,23 @@ func (l *Library) processMetadata(
}
// processCoverArt saves cover art to disk and upserts the DB record,
// using the cache to skip work for previously seen images.
// using the cache to skip work for previously seen images. When
// thumbChan is non-nil, thumbnail generation is dispatched to the
// async worker pool.
func (l *Library) processCoverArt(
q *sqlcgen.Queries,
cache *entityCache,
metrics *ScanMetrics,
tags *metadata.TrackMetadata,
thumbChan chan<- thumbnailWork,
) sql.NullInt64 {
if tags.Picture == nil {
return sql.NullInt64{}
}
coverPath, err := l.saveCoverArt(tags.Picture)
coverPath, err := l.saveCoverArt(
tags.Picture, metrics, thumbChan,
)
if err != nil {
l.logger.Warn("could not save cover art", "err", err)
@@ -937,10 +1127,14 @@ func (l *Library) handleConfigUpdate(updatedConfigValues Config) error {
l.logger.Info("new library, scanning")
l.conf.DirectoryPath = updatedConfigValues.DirectoryPath
if err := l.Scan(); err != nil {
if _, err := l.Scan(); err != nil {
updateErr = errors.Join(
updateErr,
fmt.Errorf("problem scanning library on config update: %w", err),
fmt.Errorf(
"problem scanning library on config update: %w",
err,
),
)
}
}
+103
View File
@@ -0,0 +1,103 @@
package library
import (
"sync"
"time"
)
// ScanMetrics holds timing and count data collected during a library scan.
// Worker-pool fields are protected by a mutex; DB-writer fields are
// single-threaded and use plain addition.
type ScanMetrics struct {
mu sync.Mutex
// Top-level phases (wall-clock).
Total time.Duration `json:"total"`
LoadExisting time.Duration `json:"loadExisting"`
WalkDuration time.Duration `json:"walkDuration"`
ExtractionWallClock time.Duration `json:"extractionWallClock"`
DBWritesWallClock time.Duration `json:"dbWritesWallClock"`
OrphanCleanup time.Duration `json:"orphanCleanup"`
PostScanVariants time.Duration `json:"postScanVariants"`
// Per-format extraction (cumulative across workers).
FormatExtraction map[string]int64 `json:"formatExtraction"`
FormatCount map[string]int64 `json:"formatCount"`
// Sub-operation cumulative times (across workers).
TagExtraction time.Duration `json:"tagExtraction"`
DurationExtraction time.Duration `json:"durationExtraction"`
// DB sub-operations (cumulative, single-threaded DB writer).
BatchCommits time.Duration `json:"batchCommits"`
CoverArtSave time.Duration `json:"coverArtSave"`
// Thumbnail generation (async worker pool).
ThumbnailWallClock time.Duration `json:"thumbnailWallClock"`
ThumbnailGeneration time.Duration `json:"thumbnailGeneration"`
ThumbnailSmall time.Duration `json:"thumbnailSmall"`
ThumbnailMedium time.Duration `json:"thumbnailMedium"`
ThumbnailLarge time.Duration `json:"thumbnailLarge"`
// Full-rescan-specific phases.
ClearQueue time.Duration `json:"clearQueue"`
ClearDatabase time.Duration `json:"clearDatabase"`
ClearCoverFiles time.Duration `json:"clearCoverFiles"`
// File counts.
Added int64 `json:"added"`
Updated int64 `json:"updated"`
Skipped int64 `json:"skipped"`
Removed int64 `json:"removed"`
}
func newScanMetrics() *ScanMetrics {
return &ScanMetrics{
FormatExtraction: make(map[string]int64),
FormatCount: make(map[string]int64),
}
}
// addExtraction records per-file extraction timing from a worker
// goroutine. It is safe for concurrent use.
func (m *ScanMetrics) addExtraction(
fileType string,
tagTime, durationTime time.Duration,
) {
m.mu.Lock()
defer m.mu.Unlock()
total := tagTime + durationTime
m.FormatExtraction[fileType] += total.Milliseconds()
m.FormatCount[fileType]++
m.TagExtraction += tagTime
m.DurationExtraction += durationTime
}
// addCoverArtSave records the time spent saving an original cover
// art file. Called from the single-threaded DB writer.
func (m *ScanMetrics) addCoverArtSave(d time.Duration) {
m.CoverArtSave += d
}
// addThumbnailTier records the time spent generating a single
// thumbnail tier. Safe for concurrent use from the thumbnail
// worker pool.
func (m *ScanMetrics) addThumbnailTier(
suffix string,
d time.Duration,
) {
m.mu.Lock()
defer m.mu.Unlock()
m.ThumbnailGeneration += d
switch suffix {
case "_sm":
m.ThumbnailSmall += d
case "_md":
m.ThumbnailMedium += d
case "_lg":
m.ThumbnailLarge += d
}
}
+36 -16
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/wailsapp/wails/v2/pkg/runtime"
@@ -13,42 +14,61 @@ import (
// FullRescan clears the queue and player, wipes all library data
// (database records and cover art files), and performs a fresh
// scan from scratch.
func (l *Library) FullRescan() error {
// scan from scratch. The returned ScanMetrics includes timing
// for the clear phases in addition to the normal scan metrics.
func (l *Library) FullRescan() (*ScanMetrics, error) {
l.logger.Info("beginning full library rescan")
runtime.EventsEmit(l.ctx, events.LibraryScanStarted)
// Stop playback and clear the queue before wiping data so
// the player is not referencing now-deleted tracks.
clearQueueStart := time.Now()
if l.queue != nil {
l.queue.Clear()
}
if err := l.clearLibraryData(); err != nil {
return fmt.Errorf("could not clear library data: %w", err)
}
clearQueueDur := time.Since(clearQueueStart)
return l.Scan()
}
// clearLibraryData removes all library-related records from the
// database and deletes all cover art files from disk. The deletes
// are executed in FK-safe order within a single transaction.
func (l *Library) clearLibraryData() error {
l.logger.Info("clearing all library data")
// Clear all library data (DB + cover art files).
clearDBStart := time.Now()
if err := l.clearLibraryTables(); err != nil {
return err
return nil, fmt.Errorf(
"could not clear library tables: %w", err,
)
}
clearDBDur := time.Since(clearDBStart)
clearFilesStart := time.Now()
if err := l.clearCoverArtFiles(); err != nil {
return err
return nil, fmt.Errorf(
"could not clear cover art files: %w", err,
)
}
clearFilesDur := time.Since(clearFilesStart)
l.logger.Info("library data cleared successfully")
return nil
// Run the full scan and merge clear-phase times into
// the metrics it returns.
metrics, err := l.Scan()
if metrics != nil {
metrics.ClearQueue = clearQueueDur
metrics.ClearDatabase = clearDBDur
metrics.ClearCoverFiles = clearFilesDur
// Include clear-phase durations in the total so the
// displayed value reflects true wall-clock time.
metrics.Total += clearQueueDur +
clearDBDur + clearFilesDur
}
return metrics, err
}
// clearLibraryTables deletes all library-related rows in FK-safe
+28 -7
View File
@@ -4,8 +4,16 @@ import (
"fmt"
"io"
"os"
"time"
)
// ExtractionTiming holds sub-operation durations from a single
// ExtractAllMetadata call so callers can build per-format aggregates.
type ExtractionTiming struct {
TagExtraction time.Duration
DurationExtraction time.Duration
}
// AudioFileExtension represents a supported audio file extension.
type AudioFileExtension string
@@ -55,13 +63,16 @@ func GetTrackLengthMillis(path string) (int64, error) {
// ExtractAllMetadata opens the file once and extracts both tags and duration.
// This avoids the overhead of opening the file twice when both are needed.
// If skipDuration is true, only tags are extracted and lengthMillis is 0.
// The returned ExtractionTiming records how long each sub-operation took.
func ExtractAllMetadata(
path string,
skipDuration bool,
) (*TrackMetadata, int64, error) {
) (*TrackMetadata, int64, *ExtractionTiming, error) {
timing := &ExtractionTiming{}
f, err := os.Open(path)
if err != nil {
return nil, 0, fmt.Errorf(
return nil, 0, timing, fmt.Errorf(
"could not open file: %w", err,
)
}
@@ -69,30 +80,40 @@ func ExtractAllMetadata(
defer func() { _ = f.Close() }()
// Extract tags first (reads only headers, fast).
tagStart := time.Now()
tags, err := ExtractTagsFromReader(f)
timing.TagExtraction = time.Since(tagStart)
if err != nil {
return nil, 0, fmt.Errorf(
return nil, 0, timing, fmt.Errorf(
"could not extract tags from %s: %w", path, err,
)
}
if skipDuration {
return tags, 0, nil
return tags, 0, timing, nil
}
// Seek back to the beginning for duration extraction.
if _, err := f.Seek(0, io.SeekStart); err != nil {
return tags, 0, fmt.Errorf(
return tags, 0, timing, fmt.Errorf(
"could not seek file for duration: %w", err,
)
}
durStart := time.Now()
lengthMillis, err := getTrackDuration(f)
timing.DurationExtraction = time.Since(durStart)
if err != nil {
return tags, 0, fmt.Errorf(
return tags, 0, timing, fmt.Errorf(
"error getting duration for %s: %w", path, err,
)
}
return tags, lengthMillis, nil
return tags, lengthMillis, timing, nil
}
+96
View File
@@ -0,0 +1,96 @@
//go:build linux
package system
import (
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
)
var errNoBlockDevice = errors.New(
"no matching block device found",
)
// IsRotationalDisk reports whether the block device backing the
// given path is a rotational (spinning) disk. Detection uses the
// Linux sysfs interface at /sys/block/<dev>/queue/rotational.
// Returns false on any error (assumes SSD).
func IsRotationalDisk(path string) bool {
dev, err := deviceForPath(path)
if err != nil {
return false
}
rotational, err := os.ReadFile(
filepath.Join(
"/sys/block", dev, "queue", "rotational",
),
)
if err != nil {
return false
}
return strings.TrimSpace(string(rotational)) == "1"
}
// deviceForPath resolves a filesystem path to its underlying block
// device name (e.g. "sda") by matching the device major:minor
// from stat(2) against /sys/block/ entries.
func deviceForPath(path string) (string, error) {
var st syscall.Stat_t
if err := syscall.Stat(path, &st); err != nil {
return "", fmt.Errorf(
"could not stat path: %w", err,
)
}
// Extract major and minor device numbers.
major := (st.Dev >> 8) & 0xff
minor := st.Dev & 0xff
// Scan /sys/block/ for a matching device.
entries, err := os.ReadDir("/sys/block")
if err != nil {
return "", fmt.Errorf(
"could not read /sys/block: %w", err,
)
}
majorStr := strconv.FormatUint(major, 10)
devStr := majorStr + ":" +
strconv.FormatUint(minor, 10)
for _, entry := range entries {
devFile := filepath.Join(
"/sys/block", entry.Name(), "dev",
)
data, err := os.ReadFile(devFile)
if err != nil {
continue
}
content := strings.TrimSpace(string(data))
if content == devStr {
return entry.Name(), nil
}
// The filesystem might be on a partition (e.g. sda1)
// whose parent block device is sda. Check if the
// major number matches.
parts := strings.SplitN(content, ":", 2)
if len(parts) == 2 && parts[0] == majorStr {
return entry.Name(), nil
}
}
return "", fmt.Errorf(
"%w for %s", errNoBlockDevice, devStr,
)
}
+10
View File
@@ -0,0 +1,10 @@
//go:build !linux
package system
// IsRotationalDisk reports whether the block device backing the
// given path is a rotational (spinning) disk. On non-Linux
// platforms this always returns false (assumes SSD).
func IsRotationalDisk(_ string) bool {
return false
}