package library import ( "context" "database/sql" "errors" "fmt" "io/fs" "log/slog" "os" "path/filepath" goruntime "runtime" "slices" "strings" "sync" "sync/atomic" "github.com/wailsapp/wails/v2/pkg/runtime" "golang.org/x/sync/errgroup" "yellowjacket/backend/database" "yellowjacket/backend/database/sql/sqlcgen" "yellowjacket/backend/events" "yellowjacket/backend/metadata" ) var errLibraryDirNotConfigured = errors.New("library directory not configured") // scanBatchSize controls how many files are committed in a single // database transaction during a scan. Larger batches amortize // SQLite's fsync cost but increase the blast radius of a failed commit. const scanBatchSize = 50 // entityCache holds recently resolved database entities so that // repeated upserts for the same artist/album/cover art within a scan // can be served from memory instead of hitting the database. // It is only accessed from the single DB-writer goroutine and // therefore needs no synchronisation. type entityCache struct { artistCredits map[string]sqlcgen.ArtistCredit artists map[string]sqlcgen.Artist releaseGroups map[string]sqlcgen.ReleaseGroup coverArt map[string]sqlcgen.CoverArt // linkedCredits tracks artist-credit-artist links already created // so we skip the duplicate INSERT. Key is "artistID:creditID". linkedCredits map[string]struct{} } func newEntityCache() *entityCache { return &entityCache{ artistCredits: make(map[string]sqlcgen.ArtistCredit), artists: make(map[string]sqlcgen.Artist), releaseGroups: make(map[string]sqlcgen.ReleaseGroup), coverArt: make(map[string]sqlcgen.CoverArt), linkedCredits: make(map[string]struct{}), } } // queueClearer is a narrow interface for clearing the playback queue. type queueClearer interface { Clear() } // Library manages scanning and querying the music collection. type Library struct { ctx context.Context logger *slog.Logger conf *Config db *database.DB queue queueClearer } // SetQueue provides the library with a reference to the queue so // that destructive operations like FullRescan can clear the queue // and stop playback before wiping data. func (l *Library) SetQueue(q queueClearer) { l.queue = q } // NewLibrary creates a new library with the given configuration. // A nil config is permitted; the library will be inert until a valid // configuration is supplied via the LibraryConfigChanged event. func NewLibrary( ctx context.Context, logger *slog.Logger, conf *Config, db *database.DB, ) (*Library, error) { if conf == nil { conf = &Config{} } if err := conf.Validate(); err != nil { return nil, fmt.Errorf("invalid library config %#v: %w", conf, err) } library := &Library{ ctx: ctx, logger: logger, conf: conf, db: db, } return library, nil } // SetContext sets the Wails runtime context and registers event handlers. func (l *Library) SetContext(ctx context.Context) { l.ctx = ctx l.registerEventHandlers() } func (l *Library) registerEventHandlers() { if l.ctx == nil { l.logger.Error("Context is nil, cannot register event handlers") return } runtime.EventsOn(l.ctx, events.LibraryConfigChanged, func(data ...any) { l.logger.Info("Received LibraryConfigChanged event") if len(data) == 0 { l.logger.Error("LibraryConfigChanged event received with no data") return } configMap, ok := data[0].(map[string]any) if !ok { l.logger.Error("LibraryConfigChanged event data is not a map", "data", data[0]) return } dir, ok := configMap["DirectoryPath"].(string) if !ok { l.logger.Error("DirectoryPath not found or not a string in config event") return } updatedConfig := Config{DirectoryPath: Directory(dir)} if err := l.handleConfigUpdate(updatedConfig); err != nil { l.logger.Error("Failed to handle config update", "err", err) } }) } // Scan syncs the library by adding new files and removing deleted ones. // Files that exist but have incomplete metadata (recording_id = 0) will be updated. func (l *Library) Scan() error { l.logger.Info( "beginning library scan", "workers", scanWorkerCount, ) runtime.EventsEmit(l.ctx, events.LibraryScanStarted) if len(l.conf.DirectoryPath) == 0 { return errLibraryDirNotConfigured } // Load existing file paths from the database into a sync.Map for concurrent access. // The map tracks path → audioFile; entries are removed as files are "seen" during the walk. // Any entries remaining after the walk are orphans (files deleted from disk). existingFiles, err := l.db.Queries.GetAllAudioFiles(l.ctx) if err != nil { return fmt.Errorf("could not load existing audio files: %w", err) } existingPaths := &sync.Map{} for _, f := range existingFiles { existingPaths.Store(f.FilePath, f) } l.logger.Debug( "loaded existing files from database", "count", len(existingFiles), "library-directory", l.conf.DirectoryPath, ) basePath := string(l.conf.DirectoryPath) workChan := make(chan scanWork, 100) resultChan := make(chan importResult, 100) var added, skipped, updated atomic.Int64 var scanErr error var errMu sync.Mutex // Walker goroutine: traverse directory and send work items to workers go func() { defer close(workChan) walkErr := fs.WalkDir( os.DirFS(basePath), ".", func(path string, d fs.DirEntry, err error) error { if err != nil { l.logger.Error("problem walking directory", "path", path, "err", err) return nil // continue walking } if d.IsDir() { return nil } absoluteFilePath := filepath.Join(basePath, path) fileExt := filepath.Ext(d.Name()) fileType, isSupportedAudioFile := metadata.GetSupportedFileType(fileExt) if !isSupportedAudioFile { return nil } // Check if file already exists in database if existing, exists := existingPaths.LoadAndDelete(absoluteFilePath); exists { audioFile := existing.(sqlcgen.AudioFile) // Check if this file needs metadata update (recording_id = 0) if audioFile.RecordingID == 0 { l.logger.Debug("file needs metadata update", "path", absoluteFilePath) select { case workChan <- scanWork{ absolutePath: absoluteFilePath, fileType: fileType, existingFileID: audioFile.ID, needsUpdate: true, existingLength: audioFile.LengthMilliseconds, }: case <-l.ctx.Done(): return l.ctx.Err() } return nil } l.logger.Debug( "file already in library with metadata, skipping", "path", absoluteFilePath, ) skipped.Add(1) return nil } l.logger.Debug("queueing file for import", "path", absoluteFilePath) // Send to workers for processing select { case workChan <- scanWork{absolutePath: absoluteFilePath, fileType: fileType}: case <-l.ctx.Done(): return l.ctx.Err() } return nil }, ) if walkErr != nil { errMu.Lock() scanErr = errors.Join( scanErr, fmt.Errorf("problem walking library directory: %w", walkErr), ) errMu.Unlock() } }() // DB writer goroutine: serialize all database writes to avoid SQLite // contention. Results are committed in batches to amortize the cost // of SQLite's fsync-per-commit. var dbWg sync.WaitGroup dbWg.Add(1) go func() { defer dbWg.Done() cache := newEntityCache() var batch []importResult flushBatch := func() { if len(batch) == 0 { return } if batchErr := l.commitBatch( batch, cache, &added, &updated, ); batchErr != nil { errMu.Lock() scanErr = errors.Join(scanErr, batchErr) errMu.Unlock() } batch = batch[:0] } for result := range resultChan { batch = append(batch, result) if len(batch) >= scanBatchSize { flushBatch() } } // Flush any remaining results. flushBatch() }() // Worker pool: extract metadata concurrently, send results to DB writer g := new(errgroup.Group) g.SetLimit(scanWorkerCount) for work := range workChan { g.Go(func() error { result, err := l.extractAudioMetadata(work) if err != nil { l.logger.Warn("failed to extract metadata", "path", work.absolutePath, "err", err) errMu.Lock() scanErr = errors.Join(scanErr, err) errMu.Unlock() return nil // continue processing other files } // Send to DB writer select { case resultChan <- result: case <-l.ctx.Done(): return l.ctx.Err() } return nil }) } _ = g.Wait() // Wait for all metadata extraction to complete close(resultChan) // Signal DB writer to finish dbWg.Wait() // Wait for all DB writes to complete // Orphan cleanup: any entries remaining in existingPaths are files deleted from disk var removed atomic.Int64 existingPaths.Range(func(key, value any) bool { path := key.(string) audioFile := value.(sqlcgen.AudioFile) l.logger.Debug("removing orphaned database entry", "path", path, "id", audioFile.ID) if err := l.db.Queries.DeleteAudioFile(l.ctx, audioFile.ID); err != nil { l.logger.Warn( "failed to delete orphaned audio file", "path", path, "id", audioFile.ID, "err", err, ) return true } removed.Add(1) return true }) // Generate sized variants for any cover art missing them, // and migrate legacy _thumb files. if err := l.generateMissingSizedVariants(); err != nil { l.logger.Warn( "could not generate missing sized variants", "err", err, ) } l.logger.Info( "library scan complete", "added", added.Load(), "updated", updated.Load(), "removed", removed.Load(), "skipped", skipped.Load(), "library", l.conf.DirectoryPath, ) runtime.EventsEmit(l.ctx, events.LibraryScanComplete) return scanErr } // scanWorkerCount controls the number of concurrent file processors. // TODO: make configurable via Config. var scanWorkerCount = goruntime.NumCPU() // scanWork represents a file to be processed by a worker. type scanWork struct { absolutePath string fileType metadata.AudioFileExtension existingFileID int64 // non-zero if this is an update needsUpdate bool existingLength int64 // existing length if updating } // importResult holds metadata extracted by workers, ready for DB insertion. type importResult struct { absolutePath string fileType metadata.AudioFileExtension lengthMillis int64 tags *metadata.TrackMetadata existingFileID int64 // non-zero if this is an update needsUpdate bool } // extractAudioMetadata reads and extracts metadata from an audio file. // It opens the file once, extracting both tags and duration in a single pass. func (l *Library) extractAudioMetadata(work scanWork) (importResult, error) { result := importResult{ absolutePath: work.absolutePath, fileType: work.fileType, existingFileID: work.existingFileID, needsUpdate: work.needsUpdate, } // Skip duration decode if we already have it from a previous import. skipDuration := work.needsUpdate && work.existingLength > 0 tags, lengthMillis, err := metadata.ExtractAllMetadata( work.absolutePath, skipDuration, ) if err != nil { return result, fmt.Errorf( "could not extract metadata for %s: %w", work.absolutePath, err, ) } result.tags = tags if skipDuration { result.lengthMillis = work.existingLength } else { result.lengthMillis = lengthMillis } return result, nil } // commitBatch wraps a slice of import results in a single database // transaction, creating all related records and audio file entries. // Individual file failures are logged and accumulated but do not // abort the entire batch. func (l *Library) commitBatch( batch []importResult, cache *entityCache, added, updated *atomic.Int64, ) error { tx, err := l.db.BeginTx() if err != nil { return fmt.Errorf("could not begin transaction: %w", err) } txq := l.db.Queries.WithTx(tx) var batchErr error for i := range batch { result := &batch[i] var saveErr error if result.needsUpdate { saveErr = l.updateAudioFileMetadata(txq, cache, *result) if saveErr == nil { updated.Add(1) } } else { saveErr = l.saveAudioFile(txq, cache, *result) if saveErr == nil { added.Add(1) } } if saveErr != nil { l.logger.Warn( "failed to save audio file", "path", result.absolutePath, "err", saveErr, ) batchErr = errors.Join(batchErr, saveErr) } } if commitErr := tx.Commit(); commitErr != nil { return fmt.Errorf( "could not commit batch of %d files: %w", len(batch), commitErr, ) } return batchErr } // saveAudioFile writes audio file metadata to the database (new files). func (l *Library) saveAudioFile( q *sqlcgen.Queries, cache *entityCache, result importResult, ) error { l.logger.Debug( "saving audio file to db", "absolute-path", result.absolutePath, "track-length-millis", result.lengthMillis, "file-type", int64( slices.Index( metadata.SupportedFileExtensions, result.fileType, ), ), ) // Process metadata and create related records. recordingID, err := l.processMetadata(q, cache, result) if err != nil { return fmt.Errorf("could not process metadata: %w", err) } if _, err := q.CreateAudioFile( l.ctx, sqlcgen.CreateAudioFileParams{ FilePath: result.absolutePath, LengthMilliseconds: result.lengthMillis, FileTypeID: int64( slices.Index( metadata.SupportedFileExtensions, result.fileType, ), ), RecordingID: recordingID, }); err != nil { return fmt.Errorf( "could not save audio file to db: %w", err, ) } l.logger.Debug( "added audio file to library", "path", result.absolutePath, ) return nil } // updateAudioFileMetadata updates an existing audio file with extracted metadata. func (l *Library) updateAudioFileMetadata( q *sqlcgen.Queries, cache *entityCache, result importResult, ) error { l.logger.Debug( "updating audio file metadata", "absolute-path", result.absolutePath, "file-id", result.existingFileID, ) // Process metadata and create related records. recordingID, err := l.processMetadata(q, cache, result) if err != nil { return fmt.Errorf("could not process metadata: %w", err) } if err := q.UpdateAudioFileRecording( l.ctx, sqlcgen.UpdateAudioFileRecordingParams{ RecordingID: recordingID, ID: result.existingFileID, }); err != nil { return fmt.Errorf( "could not update audio file recording: %w", err, ) } l.logger.Debug( "updated audio file metadata", "path", result.absolutePath, ) return nil } // processMetadata creates all related database records for metadata // and returns the recording ID. It uses the provided queries object // (which may be transaction-scoped) and the entity cache to avoid // redundant upserts for repeated artist/album/cover-art values. func (l *Library) processMetadata( q *sqlcgen.Queries, cache *entityCache, result importResult, ) (int64, error) { tags := result.tags if tags == nil { tags = &metadata.TrackMetadata{} } // 1. Handle cover art (if present). coverArtID := l.processCoverArt(q, cache, tags) // 2. Get or create artist credit for track artist. artistName := tags.Artist if artistName == "" { artistName = "Unknown Artist" } artistCredit, err := l.cachedUpsertArtistCredit( q, cache, artistName, ) if err != nil { return 0, fmt.Errorf( "could not upsert artist credit: %w", err, ) } l.cachedLinkArtist(q, cache, artistName, artistCredit.ID) // 3. Get or create artist credit for album artist. albumArtistCreditID := l.resolveAlbumArtistCredit( q, cache, tags, artistCredit.ID, ) // 4. Get or create release group (album). releaseGroupID := l.resolveReleaseGroup( q, cache, tags, albumArtistCreditID, coverArtID, ) // 5. Create recording. recording, err := q.CreateRecordingFull( l.ctx, sqlcgen.CreateRecordingFullParams{ Name: l.getRecordingName( tags, result.absolutePath, ), ArtistCreditID: artistCredit.ID, TrackNumber: toNullInt64(tags.TrackNumber), DiscNumber: toNullInt64(tags.DiscNumber), Year: toNullInt64(tags.Year), Genre: toNullString(tags.Genre), Composer: toNullString(tags.Composer), Lyrics: toNullString(tags.Lyrics), Comment: toNullString(tags.Comment), }, ) if err != nil { return 0, fmt.Errorf( "could not create recording: %w", err, ) } // 6. Link recording to release group. if releaseGroupID.Valid { _, err = q.CreateReleaseGroupRecording( l.ctx, sqlcgen.CreateReleaseGroupRecordingParams{ ReleaseGroupID: releaseGroupID.Int64, RecordingID: recording.ID, TrackNumber: toNullInt64(tags.TrackNumber), DiscNumber: toNullInt64(tags.DiscNumber), }, ) if err != nil { l.logger.Warn( "could not link recording to release group", "err", err, ) } } return recording.ID, nil } // processCoverArt saves cover art to disk and upserts the DB record, // using the cache to skip work for previously seen images. func (l *Library) processCoverArt( q *sqlcgen.Queries, cache *entityCache, tags *metadata.TrackMetadata, ) sql.NullInt64 { if tags.Picture == nil { return sql.NullInt64{} } coverPath, err := l.saveCoverArt(tags.Picture) if err != nil { l.logger.Warn("could not save cover art", "err", err) return sql.NullInt64{} } if coverPath == "" { return sql.NullInt64{} } // Check cache first. if cached, ok := cache.coverArt[coverPath]; ok { return sql.NullInt64{Int64: cached.ID, Valid: true} } ca, err := q.UpsertCoverArt(l.ctx, sqlcgen.UpsertCoverArtParams{ IsEmbedded: true, FilePath: coverPath, MimeType: tags.Picture.MIMEType, }) if err != nil { l.logger.Warn( "could not create cover art record", "err", err, ) return sql.NullInt64{} } cache.coverArt[coverPath] = ca return sql.NullInt64{Int64: ca.ID, Valid: true} } // cachedUpsertArtistCredit returns the artist credit for the given // name, using the cache when possible. func (l *Library) cachedUpsertArtistCredit( q *sqlcgen.Queries, cache *entityCache, name string, ) (sqlcgen.ArtistCredit, error) { if cached, ok := cache.artistCredits[name]; ok { return cached, nil } ac, err := q.UpsertArtistCredit(l.ctx, name) if err != nil { return sqlcgen.ArtistCredit{}, err } cache.artistCredits[name] = ac return ac, nil } // cachedLinkArtist upserts the artist record and creates the // artist-credit-artist link, skipping work already done. func (l *Library) cachedLinkArtist( q *sqlcgen.Queries, cache *entityCache, name string, creditID int64, ) { artist, ok := cache.artists[name] if !ok { var err error artist, err = q.UpsertArtist(l.ctx, name) if err != nil { l.logger.Warn( "could not upsert artist", "err", err, ) return } cache.artists[name] = artist } linkKey := fmt.Sprintf("%d:%d", artist.ID, creditID) if _, done := cache.linkedCredits[linkKey]; done { return } _, _ = q.CreateArtistCreditArtist( l.ctx, sqlcgen.CreateArtistCreditArtistParams{ ArtistID: artist.ID, CreditID: creditID, }, ) cache.linkedCredits[linkKey] = struct{}{} } // resolveAlbumArtistCredit returns the album artist credit ID. // When the AlbumArtist tag is absent or matches the track artist, // the track artist credit is reused. func (l *Library) resolveAlbumArtistCredit( q *sqlcgen.Queries, cache *entityCache, tags *metadata.TrackMetadata, trackArtistCreditID int64, ) sql.NullInt64 { if tags.AlbumArtist == "" || tags.AlbumArtist == tags.Artist { return sql.NullInt64{ Int64: trackArtistCreditID, Valid: true, } } albumArtistCredit, err := l.cachedUpsertArtistCredit( q, cache, tags.AlbumArtist, ) if err != nil { l.logger.Warn( "could not upsert album artist credit", "err", err, ) return sql.NullInt64{} } l.cachedLinkArtist( q, cache, tags.AlbumArtist, albumArtistCredit.ID, ) return sql.NullInt64{ Int64: albumArtistCredit.ID, Valid: true, } } // resolveReleaseGroup returns the release group ID for the album, // using the cache when possible. func (l *Library) resolveReleaseGroup( q *sqlcgen.Queries, cache *entityCache, tags *metadata.TrackMetadata, albumArtistCreditID sql.NullInt64, coverArtID sql.NullInt64, ) sql.NullInt64 { if tags.Album == "" { return sql.NullInt64{} } // Check cache first. if cached, ok := cache.releaseGroups[tags.Album]; ok { // If the cached release group lacks cover art and we now // have it, update it. if coverArtID.Valid && !cached.CoverArtID.Valid { err := q.UpdateReleaseGroupCoverArt( l.ctx, sqlcgen.UpdateReleaseGroupCoverArtParams{ CoverArtID: coverArtID, ID: cached.ID, }, ) if err != nil { l.logger.Warn( "could not update release group cover art", "err", err, ) } else { cached.CoverArtID = coverArtID cache.releaseGroups[tags.Album] = cached } } return sql.NullInt64{Int64: cached.ID, Valid: true} } rg, err := q.UpsertReleaseGroup( l.ctx, sqlcgen.UpsertReleaseGroupParams{ Name: tags.Album, AlbumArtistCreditID: albumArtistCreditID, Year: toNullInt64(tags.Year), }, ) if err != nil { l.logger.Warn( "could not upsert release group", "err", err, ) return sql.NullInt64{} } // Update cover art if this album doesn't have one yet. if coverArtID.Valid && !rg.CoverArtID.Valid { err := q.UpdateReleaseGroupCoverArt( l.ctx, sqlcgen.UpdateReleaseGroupCoverArtParams{ CoverArtID: coverArtID, ID: rg.ID, }, ) if err != nil { l.logger.Warn( "could not update release group cover art", "err", err, ) } else { rg.CoverArtID = coverArtID } } cache.releaseGroups[tags.Album] = rg return sql.NullInt64{Int64: rg.ID, Valid: true} } // getRecordingName returns the track title, or falls back to the filename. func (l *Library) getRecordingName(tags *metadata.TrackMetadata, filePath string) string { if tags.Title != "" { return tags.Title } // Fallback to filename without extension base := filepath.Base(filePath) return strings.TrimSuffix(base, filepath.Ext(base)) } // toNullInt64 converts an int to sql.NullInt64, treating 0 as null. func toNullInt64(v int) sql.NullInt64 { if v == 0 { return sql.NullInt64{} } return sql.NullInt64{Int64: int64(v), Valid: true} } // toNullString converts a string to sql.NullString, treating empty as null. func toNullString(v string) sql.NullString { if v == "" { return sql.NullString{} } return sql.NullString{String: v, Valid: true} } func (l *Library) handleConfigUpdate(updatedConfigValues Config) error { l.logger.Info("handling config update", "updated", updatedConfigValues) var updateErr error if l.conf.DirectoryPath != updatedConfigValues.DirectoryPath { l.logger.Info("new library, scanning") l.conf.DirectoryPath = updatedConfigValues.DirectoryPath if err := l.Scan(); err != nil { updateErr = errors.Join( updateErr, fmt.Errorf("problem scanning library on config update: %w", err), ) } } return updateErr }