feat(jobs): surface background jobs with progress, logs and controls
Add a central job registry that library scans and search index builds report into, so background work is visible instead of buried in the settings page. - backend/jobs: registry with per-job ring-buffer logs, capability-driven controls, and one coalesced JobsChanged snapshot at 4Hz - pause survives restart via a job_state table; a paused scan is adopted back on launch and skipped by the soft scan - top-bar indicator, popover, details drawer and a Jobs page replacing the config page's scan UI; per-library start/stop retained - scan timing breakdown moves into the job log, Full rescan to the Jobs page; delete the orphaned library-manager component Also add cmd/indexbuild and cmd/indexexport so the explore index can be built once centrally rather than by every install, which today streams ~205GB from the ListenBrainz spark dump on first run. indexbuild picks build/refresh/rebuild from index state; the Gitea workflow runs it on push, weekly, or manually and publishes only when content changed. fresh-install no longer defaults YJ_HOME under /tmp: it is tmpfs on most distros, and the import needs ~6GB of real disk. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stubState lets the decision table run without a database. It mirrors
|
||||
// the three pieces of index state `decide` consults.
|
||||
type stubState struct {
|
||||
complete bool
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func (s stubState) IndexImportComplete() bool { return s.complete }
|
||||
func (s stubState) IndexLastImported() time.Time { return s.last }
|
||||
|
||||
func TestDecide(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
rebuildAfter = 90 * 24 * time.Hour
|
||||
refreshAfter = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode mode
|
||||
state stubState
|
||||
want mode
|
||||
}{
|
||||
{
|
||||
name: "first run with no import builds",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: false},
|
||||
want: modeBuild,
|
||||
},
|
||||
{
|
||||
name: "partial import resumes as a build",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: false, last: time.Now().Add(-time.Hour)},
|
||||
want: modeBuild,
|
||||
},
|
||||
{
|
||||
name: "recent import refreshes",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: true, last: time.Now().Add(-24 * time.Hour)},
|
||||
want: modeRefresh,
|
||||
},
|
||||
{
|
||||
name: "import just under the rebuild age still refreshes",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: true, last: time.Now().Add(-89 * 24 * time.Hour)},
|
||||
want: modeRefresh,
|
||||
},
|
||||
{
|
||||
name: "import past the rebuild age rebuilds",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: true, last: time.Now().Add(-91 * 24 * time.Hour)},
|
||||
want: modeRebuild,
|
||||
},
|
||||
{
|
||||
name: "unreadable timestamp rebuilds rather than wedging",
|
||||
mode: modeAuto,
|
||||
state: stubState{complete: true},
|
||||
want: modeRebuild,
|
||||
},
|
||||
{
|
||||
name: "explicit mode overrides state",
|
||||
mode: modeRefresh,
|
||||
state: stubState{complete: false},
|
||||
want: modeRefresh,
|
||||
},
|
||||
{
|
||||
name: "explicit rebuild overrides a fresh import",
|
||||
mode: modeRebuild,
|
||||
state: stubState{complete: true, last: time.Now()},
|
||||
want: modeRebuild,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, why := decideFrom(
|
||||
opts{
|
||||
mode: tt.mode,
|
||||
rebuildAfter: rebuildAfter,
|
||||
refreshAfter: refreshAfter,
|
||||
},
|
||||
tt.state,
|
||||
)
|
||||
if got != tt.want {
|
||||
t.Errorf("decide = %q (%s), want %q", got, why, tt.want)
|
||||
}
|
||||
|
||||
if why == "" {
|
||||
t.Error("expected a non-empty reason")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
// Command indexbuild maintains the explore search index outside the
|
||||
// desktop app, so the catalog can be built once centrally instead of by
|
||||
// every install.
|
||||
//
|
||||
// It decides what to do from the index's own state rather than needing
|
||||
// the caller to know:
|
||||
//
|
||||
// no completed import → build (first run, or resume a partial one)
|
||||
// import older than 3mo → rebuild (re-import from the newest dump)
|
||||
// otherwise → refresh (fold in new incremental listens)
|
||||
//
|
||||
// A full build streams ~205GB from the ListenBrainz spark dump — far
|
||||
// more than one CI job should attempt — so builds are budgeted and
|
||||
// resumable: the importer checkpoints its absolute stream offset, and
|
||||
// each run continues where the last stopped. A refresh is cheap
|
||||
// (~250MB incremental dumps) and finishes in one run.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// YJ_HOME=/cache indexbuild -budget 3h
|
||||
//
|
||||
// Exit codes:
|
||||
//
|
||||
// 0 up to date — nothing left to do
|
||||
// 3 build incomplete — schedule another run to resume
|
||||
// 1 error
|
||||
//
|
||||
// When GITHUB_OUTPUT is set, `complete` and `changed` are appended to it
|
||||
// so a workflow can decide whether to publish a new artifact.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/explore"
|
||||
"yellowjacket/backend/system"
|
||||
)
|
||||
|
||||
// exitIncomplete tells the caller the build made progress but has not
|
||||
// finished, so another run should follow. Distinct from a failure: the
|
||||
// checkpoint is valid and resuming is the correct action.
|
||||
const exitIncomplete = 3
|
||||
|
||||
// mode is what this run decided to do.
|
||||
type mode string
|
||||
|
||||
const (
|
||||
modeAuto mode = "auto"
|
||||
modeBuild mode = "build"
|
||||
modeRefresh mode = "refresh"
|
||||
modeRebuild mode = "rebuild"
|
||||
)
|
||||
|
||||
// errNoHome is returned when YJ_HOME is unset. The default per-user data
|
||||
// directory is deliberately not used: a build host should always write
|
||||
// to an explicit, persistent location.
|
||||
var errNoHome = errors.New(
|
||||
"YJ_HOME must be set to a persistent directory on real disk",
|
||||
)
|
||||
|
||||
// errIncomplete signals a clean stop with work remaining.
|
||||
var errIncomplete = errors.New("build incomplete")
|
||||
|
||||
var errBadMode = errors.New("unknown mode")
|
||||
|
||||
type opts struct {
|
||||
budget time.Duration
|
||||
mode mode
|
||||
rebuildAfter time.Duration
|
||||
refreshAfter time.Duration
|
||||
verbose bool
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
budget = flag.Duration("budget", 3*time.Hour,
|
||||
"stop and checkpoint a build after this long (0 = no limit)")
|
||||
modeFlag = flag.String("mode", string(modeAuto),
|
||||
"auto | build | refresh | rebuild")
|
||||
rebuildAfter = flag.Duration("rebuild-after", 90*24*time.Hour,
|
||||
"re-import from a fresh dump once the last import is older than this")
|
||||
refreshAfter = flag.Duration("refresh-after", 7*24*time.Hour,
|
||||
"minimum gap between incremental refreshes (0 = always)")
|
||||
verbose = flag.Bool("v", false, "debug logging")
|
||||
)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
err := run(opts{
|
||||
budget: *budget,
|
||||
mode: mode(*modeFlag),
|
||||
rebuildAfter: *rebuildAfter,
|
||||
refreshAfter: *refreshAfter,
|
||||
verbose: *verbose,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, errIncomplete) {
|
||||
os.Exit(exitIncomplete)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stderr, "indexbuild:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(o opts) error {
|
||||
logger := newLogger(o.verbose)
|
||||
|
||||
if os.Getenv("YJ_HOME") == "" {
|
||||
return errNoHome
|
||||
}
|
||||
|
||||
dataDir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve data dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := database.NewDB(logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
// NewExploreService is reused rather than reconstructing its
|
||||
// dependency graph here, so the headless path cannot drift from what
|
||||
// the app does. SetContext is never called: it installs the Wails
|
||||
// runtime context, and emitting Wails events against a non-Wails
|
||||
// context terminates the process.
|
||||
svc := explore.NewExploreService(logger.WithGroup("explore"), db)
|
||||
|
||||
chosen, why := decide(o, svc)
|
||||
|
||||
logger.Info("index maintenance",
|
||||
"mode", string(chosen),
|
||||
"reason", why,
|
||||
"dataDir", dataDir,
|
||||
"staging", filepath.Join(dataDir, "explore-staging"),
|
||||
"lastImported", stamp(svc.IndexLastImported()),
|
||||
"baselineSeries", svc.IndexBaselineSeries(),
|
||||
)
|
||||
|
||||
seriesBefore := svc.IndexBaselineSeries()
|
||||
|
||||
switch chosen {
|
||||
case modeRefresh:
|
||||
err = doRefresh(logger, svc, o.refreshAfter)
|
||||
case modeRebuild:
|
||||
svc.PrepareIndexRebuild()
|
||||
|
||||
err = doBuild(logger, svc, o.budget)
|
||||
case modeBuild:
|
||||
err = doBuild(logger, svc, o.budget)
|
||||
case modeAuto:
|
||||
return fmt.Errorf("%w: auto should have resolved", errBadMode)
|
||||
default:
|
||||
return fmt.Errorf("%w: %q", errBadMode, chosen)
|
||||
}
|
||||
|
||||
complete := svc.IndexImportComplete() && !errors.Is(err, errIncomplete)
|
||||
|
||||
// "Changed" means there is something new worth publishing, so it is
|
||||
// only ever true for a finished import: a build stamps the listens
|
||||
// series early, long before its rows are assembled, and reporting a
|
||||
// change off that would be a lie about a half-built index.
|
||||
changed := complete &&
|
||||
(svc.IndexBaselineSeries() != seriesBefore || chosen != modeRefresh)
|
||||
|
||||
report(logger, svc, chosen, complete, changed)
|
||||
|
||||
if writeErr := writeOutputs(complete, changed); writeErr != nil {
|
||||
logger.Warn("could not write workflow outputs", "err", writeErr)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func newLogger(verbose bool) *slog.Logger {
|
||||
level := slog.LevelInfo
|
||||
if verbose {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
|
||||
return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: level,
|
||||
}))
|
||||
}
|
||||
|
||||
// indexState is the slice of the index that the mode decision depends
|
||||
// on. Narrowing it to an interface keeps the decision table testable
|
||||
// without standing up a database.
|
||||
type indexState interface {
|
||||
IndexImportComplete() bool
|
||||
IndexLastImported() time.Time
|
||||
}
|
||||
|
||||
// decide picks the mode from the index's own state, so callers (a cron,
|
||||
// a push hook, a human) need no knowledge of where the index stands.
|
||||
func decide(o opts, svc *explore.Service) (mode, string) {
|
||||
return decideFrom(o, svc)
|
||||
}
|
||||
|
||||
func decideFrom(o opts, state indexState) (mode, string) {
|
||||
if o.mode != modeAuto {
|
||||
return o.mode, "explicitly requested"
|
||||
}
|
||||
|
||||
if !state.IndexImportComplete() {
|
||||
return modeBuild, "no completed import yet"
|
||||
}
|
||||
|
||||
last := state.IndexLastImported()
|
||||
if last.IsZero() {
|
||||
// Marker present but unparseable — treat as due rather than
|
||||
// letting a malformed timestamp wedge the rebuild cadence.
|
||||
return modeRebuild, "import timestamp unreadable"
|
||||
}
|
||||
|
||||
if age := time.Since(last); age >= o.rebuildAfter {
|
||||
return modeRebuild, fmt.Sprintf("last import %s ago (>= %s)",
|
||||
age.Round(time.Hour), o.rebuildAfter)
|
||||
}
|
||||
|
||||
return modeRefresh, "import current, folding in new listens"
|
||||
}
|
||||
|
||||
func doBuild(
|
||||
logger *slog.Logger, svc *explore.Service, budget time.Duration,
|
||||
) error {
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
defer signal.Stop(stop)
|
||||
|
||||
var timer <-chan time.Time
|
||||
|
||||
if budget > 0 {
|
||||
t := time.NewTimer(budget)
|
||||
defer t.Stop()
|
||||
|
||||
timer = t.C
|
||||
}
|
||||
|
||||
// finished is closed by this function alone, so the watchdog only
|
||||
// reads it — no double close, and it exits whether the build ended
|
||||
// on its own or was stopped.
|
||||
finished := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-finished:
|
||||
case sig := <-stop:
|
||||
logger.Info("build: signal received, checkpointing",
|
||||
"signal", sig.String())
|
||||
svc.StopIndexBuild()
|
||||
case <-timer:
|
||||
logger.Info("build: budget reached, checkpointing")
|
||||
svc.StopIndexBuild()
|
||||
}
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
|
||||
svc.StartIndexBuild()
|
||||
svc.WaitForIndexIdle()
|
||||
close(finished)
|
||||
|
||||
logger.Info("build stopped", "elapsed", time.Since(start).Round(time.Second))
|
||||
|
||||
if !svc.IndexImportComplete() {
|
||||
return errIncomplete
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func doRefresh(
|
||||
logger *slog.Logger, svc *explore.Service, minInterval time.Duration,
|
||||
) error {
|
||||
start := time.Now()
|
||||
|
||||
svc.RefreshIndexNow(minInterval)
|
||||
|
||||
logger.Info("refresh finished",
|
||||
"elapsed", time.Since(start).Round(time.Second))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func report(
|
||||
logger *slog.Logger, svc *explore.Service,
|
||||
chosen mode, complete, changed bool,
|
||||
) {
|
||||
status := svc.GetIndexStatus()
|
||||
|
||||
logger.Info("index state",
|
||||
"mode", string(chosen),
|
||||
"complete", complete,
|
||||
"changed", changed,
|
||||
"artists", status.Artists,
|
||||
"releaseGroups", status.ReleaseGroups,
|
||||
"recordings", status.Recordings,
|
||||
"totalRows", status.TotalRows,
|
||||
"baselineSeries", svc.IndexBaselineSeries(),
|
||||
)
|
||||
|
||||
for _, tier := range status.Tiers {
|
||||
logger.Info(" stage",
|
||||
"name", tier.Name,
|
||||
"state", tier.State,
|
||||
"completed", tier.Completed,
|
||||
"total", tier.Total,
|
||||
"error", tier.Error,
|
||||
)
|
||||
}
|
||||
|
||||
if !complete {
|
||||
logger.Info("build incomplete — rerun to resume from checkpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// writeOutputs appends step outputs when running under a workflow, so
|
||||
// the caller can publish only when something actually changed.
|
||||
func writeOutputs(complete, changed bool) error {
|
||||
path := os.Getenv("GITHUB_OUTPUT")
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open outputs: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
_, err = fmt.Fprintf(f, "complete=%s\nchanged=%s\n",
|
||||
strconv.FormatBool(complete), strconv.FormatBool(changed))
|
||||
if err != nil {
|
||||
return fmt.Errorf("write outputs: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func stamp(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "never"
|
||||
}
|
||||
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
// Command indexexport turns a fully built explore index into the
|
||||
// compact "core" artifact that ships to users.
|
||||
//
|
||||
// The full dump-built index is far too large to distribute (~900MB by
|
||||
// current budget estimates). The core artifact keeps the most-listened
|
||||
// artists and their discography slice — enough for Explore to be useful
|
||||
// on a fresh install — and leaves the long tail to the existing lazy
|
||||
// per-artist fetch paths.
|
||||
//
|
||||
// The artifact deliberately contains no FTS table. The importing client
|
||||
// inserts these rows into its own explore_index, whose AFTER INSERT
|
||||
// trigger populates explore_index_fts as a side effect, so shipping a
|
||||
// search index would be redundant weight.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// YJ_HOME=/var/cache/yellowjacket-index indexexport -o core-index.db
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"yellowjacket/backend/system"
|
||||
)
|
||||
|
||||
// Columns copied into the artifact: the global catalog only.
|
||||
//
|
||||
// Deliberately excluded are the per-user columns — in_library,
|
||||
// is_similar, local_artist_id, local_release_group_id,
|
||||
// local_recording_id — which describe one person's library and are
|
||||
// recomputed locally by PopulateLocalCrossReferences after import.
|
||||
const catalogColumns = `entity_type, mbid, title, artist_name, artist_mbid,
|
||||
aliases, popularity, listener_count, duration, caa_release_mbid,
|
||||
release_name, primary_type, secondary_types, release_date,
|
||||
artist_type, country, disambiguation, sort_name, discog_fetched`
|
||||
|
||||
var errNoHome = errors.New(
|
||||
"YJ_HOME must be set to the directory holding the built index",
|
||||
)
|
||||
|
||||
var errEmptyIndex = errors.New(
|
||||
"source index has no rows — run indexbuild to completion first",
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := flag.String("o", "core-index.db", "output artifact path")
|
||||
artists := flag.Int("artists", 50_000,
|
||||
"number of top artists (by listen count) to include")
|
||||
perArtistRGs := flag.Int("rgs-per-artist", 15,
|
||||
"max release groups per included artist")
|
||||
perArtistRecs := flag.Int("recs-per-artist", 30,
|
||||
"max recordings per included artist")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if err := run(*out, *artists, *perArtistRGs, *perArtistRecs); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "indexexport:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(out string, artists, perArtistRGs, perArtistRecs int) error {
|
||||
if os.Getenv("YJ_HOME") == "" {
|
||||
return errNoHome
|
||||
}
|
||||
|
||||
dataDir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve data dir: %w", err)
|
||||
}
|
||||
|
||||
srcPath := filepath.Join(dataDir, "yj.db")
|
||||
|
||||
// Read-only so an export can never disturb a build that is still
|
||||
// running against the same working directory.
|
||||
db, err := sql.Open("sqlite",
|
||||
"file:"+srcPath+"?_pragma=busy_timeout(10000)&mode=ro")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source index: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
var srcRows int
|
||||
if err := db.QueryRow(
|
||||
"SELECT COUNT(*) FROM explore_index",
|
||||
).Scan(&srcRows); err != nil {
|
||||
return fmt.Errorf("count source rows: %w", err)
|
||||
}
|
||||
|
||||
if srcRows == 0 {
|
||||
return errEmptyIndex
|
||||
}
|
||||
|
||||
fmt.Printf("source: %s (%d rows)\n", srcPath, srcRows)
|
||||
|
||||
if err := os.Remove(out); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("clear output: %w", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`ATTACH DATABASE ? AS core`, out); err != nil {
|
||||
return fmt.Errorf("attach output: %w", err)
|
||||
}
|
||||
|
||||
if err := createSchema(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := copyRows(db, artists, perArtistRGs, perArtistRecs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stampMeta(db, srcRows); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DETACH before VACUUM: sqlite cannot vacuum an attached database.
|
||||
if _, err := db.Exec(`DETACH DATABASE core`); err != nil {
|
||||
return fmt.Errorf("detach output: %w", err)
|
||||
}
|
||||
|
||||
if err := vacuum(out); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return report(out)
|
||||
}
|
||||
|
||||
// createSchema builds the artifact's tables. No FTS and no triggers —
|
||||
// the importing client's own trigger rebuilds its FTS on insert.
|
||||
func createSchema(db *sql.DB) error {
|
||||
stmts := []string{
|
||||
`CREATE TABLE core.explore_index (
|
||||
entity_type TEXT NOT NULL,
|
||||
mbid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
artist_name TEXT NOT NULL,
|
||||
artist_mbid TEXT NOT NULL,
|
||||
aliases TEXT NOT NULL DEFAULT '',
|
||||
popularity INTEGER NOT NULL DEFAULT 0,
|
||||
listener_count INTEGER NOT NULL DEFAULT 0,
|
||||
duration INTEGER NOT NULL DEFAULT 0,
|
||||
caa_release_mbid TEXT NOT NULL DEFAULT '',
|
||||
release_name TEXT NOT NULL DEFAULT '',
|
||||
primary_type TEXT NOT NULL DEFAULT '',
|
||||
secondary_types TEXT NOT NULL DEFAULT '',
|
||||
release_date TEXT NOT NULL DEFAULT '',
|
||||
artist_type TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
disambiguation TEXT NOT NULL DEFAULT '',
|
||||
sort_name TEXT NOT NULL DEFAULT '',
|
||||
discog_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (mbid)
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE TABLE core.artifact_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)`,
|
||||
}
|
||||
|
||||
for _, stmt := range stmts {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
return fmt.Errorf("create artifact schema: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyRows selects the core subset: the top artists by listen count,
|
||||
// then a bounded slice of each one's release groups and recordings.
|
||||
//
|
||||
// The per-artist window mirrors the S2 coverage already in
|
||||
// dumpcatalog.go — a flat global top-N would give a handful of
|
||||
// superstars everything and everyone else nothing.
|
||||
func copyRows(db *sql.DB, artists, perArtistRGs, perArtistRecs int) error {
|
||||
if _, err := db.Exec(`
|
||||
CREATE TEMP TABLE core_artists AS
|
||||
SELECT mbid FROM main.explore_index
|
||||
WHERE entity_type = 'artist'
|
||||
ORDER BY popularity DESC
|
||||
LIMIT ?`, artists,
|
||||
); err != nil {
|
||||
return fmt.Errorf("select core artists: %w", err)
|
||||
}
|
||||
|
||||
copied, err := insertSelect(db, `
|
||||
INSERT INTO core.explore_index (`+catalogColumns+`)
|
||||
SELECT `+catalogColumns+`
|
||||
FROM main.explore_index
|
||||
WHERE entity_type = 'artist'
|
||||
AND mbid IN (SELECT mbid FROM core_artists)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf(" artists: %d\n", copied)
|
||||
|
||||
for _, sel := range []struct {
|
||||
label string
|
||||
entity string
|
||||
limit int
|
||||
}{
|
||||
{"release groups", "release_group", perArtistRGs},
|
||||
{"recordings", "recording", perArtistRecs},
|
||||
} {
|
||||
// The window is over artist_mbid so each artist contributes at
|
||||
// most `limit` rows, ranked by their own listen counts.
|
||||
n, err := insertSelect(db, `
|
||||
INSERT INTO core.explore_index (`+catalogColumns+`)
|
||||
SELECT `+catalogColumns+` FROM (
|
||||
SELECT *, ROW_NUMBER() OVER (
|
||||
PARTITION BY artist_mbid ORDER BY popularity DESC
|
||||
) AS rn
|
||||
FROM main.explore_index
|
||||
WHERE entity_type = ?
|
||||
AND artist_mbid IN (SELECT mbid FROM core_artists)
|
||||
) WHERE rn <= ?`, sel.entity, sel.limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf(" %-15s %d\n", sel.label+":", n)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertSelect(db *sql.DB, query string, args ...any) (int64, error) {
|
||||
res, err := db.Exec(query, args...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("copy rows: %w", err)
|
||||
}
|
||||
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("rows affected: %w", err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// stampMeta records what the importing client needs to know: that the
|
||||
// catalog half is already populated, and which incremental listens
|
||||
// series the popularity numbers are baselined on, so the incremental
|
||||
// refresh resumes from the right point instead of reapplying deltas.
|
||||
func stampMeta(db *sql.DB, srcRows int) error {
|
||||
series := lookupMeta(db, "listens_applied_series")
|
||||
built := lookupMeta(db, "dump_import_done")
|
||||
|
||||
if built == "" {
|
||||
built = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
entries := map[string]string{
|
||||
"artifact_version": "1",
|
||||
"built_at": built,
|
||||
"source_rows": strconv.Itoa(srcRows),
|
||||
"listens_applied_series": series,
|
||||
}
|
||||
|
||||
for k, v := range entries {
|
||||
if _, err := db.Exec(
|
||||
`INSERT OR REPLACE INTO core.artifact_meta (key, value) VALUES (?, ?)`,
|
||||
k, v,
|
||||
); err != nil {
|
||||
return fmt.Errorf("stamp %s: %w", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func lookupMeta(db *sql.DB, key string) string {
|
||||
var value string
|
||||
|
||||
row := db.QueryRow(
|
||||
`SELECT value FROM main.explore_index_meta WHERE key = ?`, key)
|
||||
if err := row.Scan(&value); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
func vacuum(path string) error {
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reopen artifact: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
if _, err := db.Exec("VACUUM"); err != nil {
|
||||
return fmt.Errorf("vacuum artifact: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func report(path string) error {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat artifact: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\nartifact: %s (%.1f MB)\n",
|
||||
path, float64(fi.Size())/(1<<20))
|
||||
fmt.Println("compress with: zstd -19 -T0", path)
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user