feat(jobs): surface background jobs with progress, logs and controls

Add a central job registry that library scans and search index builds
report into, so background work is visible instead of buried in the
settings page.

- backend/jobs: registry with per-job ring-buffer logs, capability-driven
  controls, and one coalesced JobsChanged snapshot at 4Hz
- pause survives restart via a job_state table; a paused scan is adopted
  back on launch and skipped by the soft scan
- top-bar indicator, popover, details drawer and a Jobs page replacing
  the config page's scan UI; per-library start/stop retained
- scan timing breakdown moves into the job log, Full rescan to the Jobs
  page; delete the orphaned library-manager component

Also add cmd/indexbuild and cmd/indexexport so the explore index can be
built once centrally rather than by every install, which today streams
~205GB from the ListenBrainz spark dump on first run. indexbuild picks
build/refresh/rebuild from index state; the Gitea workflow runs it on
push, weekly, or manually and publishes only when content changed.

fresh-install no longer defaults YJ_HOME under /tmp: it is tmpfs on most
distros, and the import needs ~6GB of real disk.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-25 14:42:22 -04:00
co-authored by Claude Opus 5
parent aead8eaef4
commit 01bc5f2094
48 changed files with 6656 additions and 2271 deletions
+42 -20
View File
@@ -23,6 +23,7 @@ import (
"yellowjacket/backend/database"
"yellowjacket/backend/database/sql/sqlcgen"
"yellowjacket/backend/events"
"yellowjacket/backend/jobs"
"yellowjacket/backend/metadata"
"yellowjacket/backend/system"
)
@@ -106,6 +107,10 @@ type Library struct {
scanPaused bool
scanPauseCh chan struct{}
// jobs is the background job registry. Nil in tests that do not
// exercise progress reporting; every use site must nil-check.
jobs *jobs.Registry
// Scan queue fields — protected by mu.
scanQueue []scanQueueEntry
currentScanLibraryID int64
@@ -218,6 +223,25 @@ func (l *Library) scanInternal(
metrics.LibraryName = libraryName
scanStart := time.Now()
// Register the background job before any work starts so the UI
// indicator appears immediately, even during the pre-walk count.
jobHandle := l.startScanJob(scanQueueEntry{
libraryID: libraryID,
libraryName: libraryName,
libraryPath: libraryPath,
})
// Stream non-fatal issues into the job log as they happen rather
// than dumping them all at completion — the point of the log pane
// is to answer "what is it doing right now".
metrics.onWarning = func(w ScanWarning) {
if jobHandle == nil {
return
}
jobHandle.LogDetail(jobs.LevelWarn, w.Phase+": "+w.Err, w.FilePath)
}
scanCtx, scanCancel := context.WithCancel(l.ctx)
defer scanCancel()
@@ -281,6 +305,14 @@ func (l *Library) scanInternal(
}
}
// emitProgress publishes one progress update to both consumers: the
// legacy LibraryScanProgress event and the shared job registry.
// Routing everything through here keeps the two from drifting.
emitProgress := func(p ScanProgress) {
runtime.EventsEmit(l.ctx, events.LibraryScanProgress, p)
reportScanProgress(jobHandle, p)
}
runtime.EventsEmit(l.ctx, events.LibraryScanStarted, map[string]any{
"libraryId": libraryID,
"libraryName": libraryName,
@@ -289,9 +321,7 @@ func (l *Library) scanInternal(
basePath := libraryPath
// --- Pre-walk: count audio files for progress reporting ---
runtime.EventsEmit(l.ctx, events.LibraryScanProgress,
mkProgress("counting", 0, 0, 0, 0, 0),
)
emitProgress(mkProgress("counting", 0, 0, 0, 0, 0))
totalFiles := countAudioFiles(basePath)
@@ -488,14 +518,10 @@ func (l *Library) scanInternal(
s := skipped.Load()
u := updated.Load()
runtime.EventsEmit(
l.ctx,
events.LibraryScanProgress,
mkProgress(
"scanning", totalFiles,
a+s+u, a, s, u,
),
)
emitProgress(mkProgress(
"scanning", totalFiles,
a+s+u, a, s, u,
))
case <-stopProgress:
return
}
@@ -618,18 +644,14 @@ func (l *Library) scanInternal(
s := skipped.Load()
u := updated.Load()
runtime.EventsEmit(l.ctx, events.LibraryScanProgress,
mkProgress("scanning", totalFiles, a+s+u, a, s, u),
)
emitProgress(mkProgress("scanning", totalFiles, a+s+u, a, s, u))
// Close thumbnail channel and wait for all thumbnail workers
// to finish. The DB writer has stopped sending work at this
// point so it is safe to close.
thumbStart := time.Now()
runtime.EventsEmit(l.ctx, events.LibraryScanProgress,
mkProgress("thumbnails", totalFiles, a+s+u, a, s, u),
)
emitProgress(mkProgress("thumbnails", totalFiles, a+s+u, a, s, u))
close(thumbChan)
thumbWg.Wait()
@@ -648,9 +670,7 @@ func (l *Library) scanInternal(
l.logger.Info("scan cancelled, skipping orphan cleanup")
} else {
// --- Phase 5: orphan cleanup ---
runtime.EventsEmit(l.ctx, events.LibraryScanProgress,
mkProgress("orphans", totalFiles, a+s+u, a, s, u),
)
emitProgress(mkProgress("orphans", totalFiles, a+s+u, a, s, u))
orphanStart := time.Now()
@@ -753,6 +773,8 @@ func (l *Library) scanInternal(
"total", metrics.Total,
)
finishScanJob(jobHandle, metrics, cancelled)
if cancelled {
runtime.EventsEmit(
l.ctx, events.LibraryScanCancelled, metrics,