package explore import ( "context" "encoding/json" "errors" "fmt" "log/slog" "sort" "strconv" "strings" "sync" "time" "github.com/wailsapp/wails/v2/pkg/runtime" "golang.org/x/sync/singleflight" "yellowjacket/backend/database" "yellowjacket/backend/events" ) // Index build parameters. const ( // indexMaxRGs is the number of release groups fetched per // artist by the API discography path (new library artists). indexMaxRGs = 50 // indexMaxRecs is the number of recordings fetched per artist // by the API discography path (new library artists). indexMaxRecs = 200 // indexMinPopularity is the minimum listen count for an entry // to be indexed. Cuts noise from long-tail entries. indexMinPopularity = 50 // championPopThreshold is the popularity floor for a row to join // the champion FTS. Rows below it (unless owned) can never win a // generic short-prefix query, whose ranking is popularity-dominated, // so excluding them is lossless for those queries. ~90k rows. championPopThreshold = 10000 // genericMaxTokenLen classifies a query as "generic" when its // longest token is at most this many runes. Such queries ("the", // "a", "u2") match a huge slice of the index with no selective term, // so they route to the champion tier; anything with a longer token // is selective enough that the full index is already fast. genericMaxTokenLen = 3 // Typo-tolerant rescue (see fuzzyRescue). When ordinary prefix-FTS // retrieval returns fewer than fuzzyRescueThinHits rows — usually a // misspelled query that shares no token prefix with any indexed name // — a bigram-similarity pass over the champion set kicks in. // fuzzyRescueMinLen guards against very short queries (too few // bigrams to discriminate); fuzzyRescueMinScore is the minimum bigram // Dice coefficient for a candidate to count as a plausible match. fuzzyRescueThinHits = 3 fuzzyRescueMinLen = 4 fuzzyRescueMinScore = 0.34 // prefixCacheTTL bounds how long a memoised generic-query result is // served before it is recomputed; prefixCacheMaxEntries caps the map. prefixCacheTTL = 10 * time.Minute prefixCacheMaxEntries = 512 // championBuiltKey marks in explore_index_meta that the champion FTS // has been populated at least once, so it is trusted across restarts. championBuiltKey = "champion_built" // localXrefReadyKey marks that PopulateLocalCrossReferences has synced // the current library into the index at least once. The unchanged- // library launch path skips the (write-heavy) re-sync when it is set; // a scan re-runs the sync unconditionally, and an explore_index wipe // clears explore_index_meta with it. localXrefReadyKey = "local_xref_ready" // lyricsIndexReadyKey marks that the lyrics FTS has been rebuilt from // the library at least once. It is kept in sync incrementally by the // LRCLIB backfill thereafter, so the unchanged-library launch path // skips the redundant full rebuild when it is set. lyricsIndexReadyKey = "lyrics_index_ready" // indexBatchSize is the number of rows per INSERT transaction. indexBatchSize = 100 // indexerRate is the requests-per-second for the background // indexer's dedicated rate limiter (LB allows 30/10s). indexerRate = 3 // indexSimilarPerArtist is how many similar artists to store // per library artist in similar_artist_map. indexSimilarPerArtist = 20 // labsBaseURL is the base URL for the ListenBrainz labs API. labsBaseURL = "https://labs.api.listenbrainz.org" // labsSimilarAlgorithm is the algorithm parameter for the // similar-artists endpoint. labsSimilarAlgorithm = "session_based_days_7500_session_300_contribution_5_threshold_10_limit_100_filter_True_skip_30" // similarArtistsBatchSize is the number of seed MBIDs processed // in one logging "batch" during Tier 4. The labs multi-seed // POST form is broken, so we actually issue one GET per seed // (concurrency bounded by indexerRate); batching here just // keeps progress log output bounded. similarArtistsBatchSize = 50 ) // SearchIndexResult is a single hit from the local popularity index. type SearchIndexResult struct { EntityType string `json:"entityType"` MBID string `json:"mbid"` Title string `json:"title"` ArtistName string `json:"artistName"` ArtistMBID string `json:"artistMbid"` Aliases string `json:"aliases,omitempty"` // Popularity signals. Popularity int `json:"popularity"` ListenerCount int `json:"listenerCount"` // Recording-specific fields. Duration int `json:"duration"` // milliseconds CAAReleaseMBID string `json:"caaReleaseMbid"` ReleaseName string `json:"releaseName"` // Release-group-specific fields. PrimaryType string `json:"primaryType"` SecondaryTypes string `json:"secondaryTypes"` // comma-separated ReleaseDate string `json:"releaseDate"` // Artist-specific fields (from MB lookup). ArtistType string `json:"artistType"` Country string `json:"country"` Disambiguation string `json:"disambiguation"` SortName string `json:"sortName"` // Personalization. InLibrary bool `json:"inLibrary"` IsSimilar bool `json:"isSimilar"` // DiscogFetched marks a row as having had its full MusicBrainz // metadata fetched. For artist rows it means the discography // (release groups + recordings) was pulled by the indexer // pipeline, and indexedArtistMBIDs() uses it to skip already- // processed artists in tier 2/3. For release_group rows it means // LookupReleaseGroup already did its one-time MB enrichment (e.g. // secondary_types), so the album page's background lookup fires at // most once per RG instead of on every visit. DiscogFetched bool `json:"-"` // Local library cross-reference (0 if not owned). LocalArtistID int64 `json:"localArtistId,omitempty"` LocalReleaseGroupID int64 `json:"localReleaseGroupId,omitempty"` LocalRecordingID int64 `json:"localRecordingId,omitempty"` // Schema version for staleness detection. SchemaVersion int `json:"-"` } // currentSchemaVersion is bumped when we add new fields that should // trigger re-indexing of existing rows. The build logic checks each // artist's rows against this version and re-fetches if stale. const currentSchemaVersion = 1 // lbSitewideArtist is the response shape from the LB sitewide // top-artists endpoint. type lbSitewideArtist struct { ArtistMBID string `json:"artist_mbid"` ArtistName string `json:"artist_name"` ListenCount int `json:"listen_count"` } // SearchIndex maintains a local SQLite FTS5 index of popular // albums and tracks. The index is populated in the background from // the MetaBrainz dumps (see dumpimport.go) and patched incrementally // via the ListenBrainz API: // // - Initial build: listens dump (popularity) + canonical dump (catalog) // - Post-scan: API discographies for new library artists // - Ongoing: organic growth from user browsing (AddFromCache) type SearchIndex struct { db *database.DB lb *ListenBrainzClient artistImg *ArtistImageProvider logger *slog.Logger runtimeCtx context.Context // Wails runtime context for event emission cancel context.CancelFunc done chan struct{} mu sync.RWMutex ready bool // championReady is set once the champion FTS (see // RebuildChampionIndex) has been populated; championBuilding guards // against overlapping rebuilds. Both are protected by mu. championReady bool championBuilding bool // prefixCache memoises the raw index hits for generic short-prefix // queries — the expensive ones — so repeats are served from memory. prefixCacheMu sync.Mutex prefixCache map[string]prefixCacheEntry // discogSF dedupes concurrent lazy discography fetches for the same // artist — the detail page fires its top-tracks, top-releases, and // similar-artists requests in parallel, so without this the same // artist would be fetched several times at once on first view. discogSF singleflight.Group // Build status tracking — read by GetIndexStatus for the UI. buildStatus IndexStatus } // prefixCacheEntry is one memoised generic-query result. type prefixCacheEntry struct { hits []SearchIndexResult created time.Time } // TierStatus represents the state of a single index tier. type TierStatus struct { Name string `json:"name"` State string `json:"state"` // "pending", "running", "complete", "error", "skipped" Total int `json:"total"` Completed int `json:"completed"` Error string `json:"error,omitempty"` } // IndexStatus is the full index build status, exposed to the frontend. type IndexStatus struct { Building bool `json:"building"` Ready bool `json:"ready"` LastBuilt string `json:"lastBuilt,omitempty"` // RFC3339 timestamp of last complete build Tiers []TierStatus `json:"tiers"` Artists int `json:"artists"` Recordings int `json:"recordings"` ReleaseGroups int `json:"releaseGroups"` TotalRows int `json:"totalRows"` } // NewSearchIndex creates a search index backed by the given // database. Call StartBuild to kick off the background populate. func NewSearchIndex( db *database.DB, lb *ListenBrainzClient, artistImg *ArtistImageProvider, logger *slog.Logger, ) *SearchIndex { return &SearchIndex{ db: db, lb: lb, artistImg: artistImg, logger: logger, } } // SetContext injects the Wails runtime context for event emission. func (si *SearchIndex) SetContext(ctx context.Context) { si.runtimeCtx = ctx // Initialize buildStatus with empty (non-nil) tiers so the // frontend always receives a valid array, not JSON null. si.mu.Lock() if si.buildStatus.Tiers == nil { si.buildStatus.Tiers = []TierStatus{} } si.mu.Unlock() // Load current row counts + last-built timestamp from DB. si.refreshStatusCounts() // Start a background ticker that emits status every 3 seconds. // This replaces frontend polling — the Wails binding dispatcher // can be blocked by other calls, but EventsEmit bypasses it. go func() { ticker := time.NewTicker(3 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: si.emitStatus() } } }() } // EnsureArtistDiscography lazily fetches an artist's top release groups // and recordings from ListenBrainz and persists them into the index — // the first time the artist is actually needed (e.g. their detail page // opens). It is a no-op once the artist is marked discog_fetched, so // each artist costs at most one set of API calls, ever; every later // view is served from the index with no network. This replaces the old // post-scan sweep that eagerly fetched every library artist on startup. // // Concurrent calls for the same artist collapse into one via discogSF, // since the detail page fires several requests for the same artist at // once. Safe to call synchronously from a request handler: bounded work // on its own rate-limited client. func (si *SearchIndex) EnsureArtistDiscography(ctx context.Context, artistMBID string) { if si.lb == nil || artistMBID == "" { return } if si.artistDiscogFetched(artistMBID) { return } _, _, _ = si.discogSF.Do(artistMBID, func() (any, error) { // Re-check under the singleflight: a sibling call may have just // finished fetching this artist while we were queued. if si.artistDiscogFetched(artistMBID) { return nil, nil } indexLB := NewListenBrainzClient( NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"), ) si.indexOneArtist(ctx, indexLB, lbSitewideArtist{ ArtistMBID: artistMBID, ArtistName: si.artistDisplayName(artistMBID), }) return nil, nil }) } // artistDiscogFetched reports whether the artist row already has its // discography fetched, so EnsureArtistDiscography can skip the network. func (si *SearchIndex) artistDiscogFetched(mbid string) bool { rows, err := si.db.QueryContext( "SELECT 1 FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND discog_fetched = 1 LIMIT 1", mbid, ) if err != nil { return false } defer func() { _ = rows.Close() }() return rows.Next() } // artistDisplayName resolves a human-readable name for an artist MBID, // preferring the index title, then the local library, then the MBID // itself. Used to seed the discography fetch's artist entry. func (si *SearchIndex) artistDisplayName(mbid string) string { for _, q := range []string{ "SELECT title FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND title != '' AND title != mbid LIMIT 1", "SELECT name FROM artists WHERE mbid = ? AND name != '' LIMIT 1", } { rows, err := si.db.QueryContext(q, mbid) if err != nil { continue } if rows.Next() { var name string if err := rows.Scan(&name); err == nil && name != "" { _ = rows.Close() return name } } _ = rows.Close() } return mbid } // PersistSimilarArtists stores ListenBrainz similar-artist results into // the similar_artist_map so future lookups are served locally instead of // re-hitting the labs API on every artist-page view. func (si *SearchIndex) PersistSimilarArtists(sourceMBID string, similar []LBSimilarArtist) { if len(similar) == 0 { return } wire := make([]lbSimilarArtistWire, len(similar)) for i, s := range similar { wire[i] = lbSimilarArtistWire{ ArtistMBID: s.ArtistMBID, Name: s.Name, Score: int(s.Score), } } si.storeSimilarArtists(sourceMBID, wire) } // StartBuild launches the background index build goroutine. // Returns immediately. func (si *SearchIndex) StartBuild(ctx context.Context) { si.mu.Lock() // Don't start if already running. if si.cancel != nil { si.mu.Unlock() return } si.done = make(chan struct{}) si.mu.Unlock() buildCtx, cancel := context.WithCancel(ctx) si.mu.Lock() si.cancel = cancel si.mu.Unlock() go func() { defer func() { si.mu.Lock() si.cancel = nil si.mu.Unlock() close(si.done) // The index rows (and their popularities) may have changed, so // refresh the champion tier once the build settles. si.scheduleChampionRebuild() }() si.runDumpBuild(buildCtx) }() } // StopBuild cancels an in-flight build and waits for it to finish. // Safe to call even if no build is running. func (si *SearchIndex) StopBuild() { si.mu.RLock() cancel := si.cancel done := si.done si.mu.RUnlock() if cancel != nil { cancel() } if done != nil { <-done } } // WaitForIdle blocks until no build or indexing goroutine is running. // Unlike StopBuild, this does NOT cancel a running build. func (si *SearchIndex) WaitForIdle() { si.mu.RLock() done := si.done si.mu.RUnlock() if done != nil { <-done } } // IsReady returns true once the index has been built at least once. func (si *SearchIndex) IsReady() bool { si.mu.RLock() defer si.mu.RUnlock() return si.ready } // GetIndexStatus returns the current index build status for the UI. // Entirely in-memory — no DB queries — to avoid blocking the Wails // UI thread when the index build holds a write lock. func (si *SearchIndex) GetIndexStatus() IndexStatus { si.mu.RLock() status := si.buildStatus status.Ready = si.ready status.Building = si.cancel != nil si.mu.RUnlock() return status } // refreshStatusCounts updates the row counts and last-built timestamp // in buildStatus from the DB. Called between tiers when the DB is idle. func (si *SearchIndex) refreshStatusCounts() { var artists, recordings, rgs int rows, err := si.db.QueryContext(` SELECT entity_type, COUNT(*) FROM explore_index GROUP BY entity_type `) if err == nil { defer func() { _ = rows.Close() }() for rows.Next() { var ( et string count int ) if err := rows.Scan(&et, &count); err == nil { switch et { case "artist": artists = count case "recording": recordings = count case "release_group": rgs = count } } } } var lastBuilt string metaRow, err := si.db.QueryContext( "SELECT value FROM explore_index_meta WHERE key = 'dump_import_done'", ) if err == nil { defer func() { _ = metaRow.Close() }() if metaRow.Next() { _ = metaRow.Scan(&lastBuilt) } } si.mu.Lock() si.buildStatus.Artists = artists si.buildStatus.Recordings = recordings si.buildStatus.ReleaseGroups = rgs si.buildStatus.TotalRows = artists + recordings + rgs si.buildStatus.LastBuilt = lastBuilt si.mu.Unlock() si.emitStatus() } // setTierStatus updates the build status for a named tier. func (si *SearchIndex) setTierStatus(name, state string, total, completed int) { si.mu.Lock() for i := range si.buildStatus.Tiers { if si.buildStatus.Tiers[i].Name == name { si.buildStatus.Tiers[i].State = state si.buildStatus.Tiers[i].Total = total si.buildStatus.Tiers[i].Completed = completed si.mu.Unlock() si.emitStatus() return } } si.buildStatus.Tiers = append(si.buildStatus.Tiers, TierStatus{ Name: name, State: state, Total: total, Completed: completed, }) si.mu.Unlock() si.emitStatus() } // setTierError marks a build stage as errored. func (si *SearchIndex) setTierError(name, errMsg string) { si.mu.Lock() for i := range si.buildStatus.Tiers { if si.buildStatus.Tiers[i].Name == name { si.buildStatus.Tiers[i].State = "error" si.buildStatus.Tiers[i].Error = errMsg si.mu.Unlock() si.emitStatus() return } } si.mu.Unlock() } // emitStatus pushes the current index status to the frontend via Wails event. func (si *SearchIndex) emitStatus() { if si.runtimeCtx == nil { return } si.mu.RLock() status := si.buildStatus status.Ready = si.ready status.Building = si.cancel != nil si.mu.RUnlock() runtime.EventsEmit(si.runtimeCtx, events.IndexStatusChanged, status) } // GetPopularity returns the cached popularity (listen count) for // the given MBID from the local index. Returns 0 if not found. func (si *SearchIndex) GetPopularity(mbid string) int { if mbid == "" { return 0 } rows, err := si.db.QueryContext( "SELECT popularity FROM explore_index WHERE mbid = ? LIMIT 1", mbid, ) if err != nil { return 0 } defer func() { _ = rows.Close() }() if rows.Next() { var pop int if err := rows.Scan(&pop); err == nil { return pop } } return 0 } // PopularityBatchResult contains popularity, listener count, library // status, and similarity scores for a batch of MBIDs. type PopularityBatchResult struct { Popularity map[string]int ListenerCount map[string]int InLibrary map[string]bool SimilarityScores map[string]int // max similarity score (0 = not similar) } // GetPopularityBatch returns popularity (listen count) and library // status for multiple MBIDs in a single query. func (si *SearchIndex) GetPopularityBatch(mbids []string) *PopularityBatchResult { if len(mbids) == 0 { return nil } placeholders := make([]string, len(mbids)) args := make([]any, len(mbids)) for i, m := range mbids { placeholders[i] = "?" args[i] = m } query := "SELECT mbid, popularity, listener_count, in_library FROM explore_index WHERE mbid IN (" + strings.Join( placeholders, ",", ) + ")" rows, err := si.db.QueryContext(query, args...) if err != nil { return nil } defer func() { _ = rows.Close() }() result := &PopularityBatchResult{ Popularity: make(map[string]int, len(mbids)), ListenerCount: make(map[string]int), InLibrary: make(map[string]bool), SimilarityScores: make(map[string]int), } for rows.Next() { var ( mbid string pop int listeners int inLib int ) if err := rows.Scan(&mbid, &pop, &listeners, &inLib); err == nil { existing, ok := result.Popularity[mbid] if !ok || pop > existing { result.Popularity[mbid] = pop } if listeners > 0 { result.ListenerCount[mbid] = listeners } if inLib == 1 { result.InLibrary[mbid] = true } } } // Fetch similarity scores from the map table. result.SimilarityScores = si.GetSimilarityScores(mbids) return result } // IsInLibrary returns whether the given MBID is marked as in the // user's local library in the search index. func (si *SearchIndex) IsInLibrary(mbid string) bool { if mbid == "" { return false } rows, err := si.db.QueryContext( "SELECT in_library FROM explore_index WHERE mbid = ? AND in_library = 1 LIMIT 1", mbid, ) if err != nil { return false } defer func() { _ = rows.Close() }() return rows.Next() } // LookupArtistByMBID reads a single artist row from the index, including // all metadata fields (type, country, disambiguation, sort_name). // Returns nil if the artist isn't indexed. func (si *SearchIndex) LookupArtistByMBID(mbid string) *SearchIndexResult { rows, err := si.db.QueryContext( `SELECT title, artist_name, artist_mbid, popularity, listener_count, artist_type, country, disambiguation, sort_name, aliases, in_library, is_similar, COALESCE(local_artist_id, 0) FROM explore_index WHERE mbid = ? AND entity_type = 'artist' LIMIT 1`, mbid, ) if err != nil { return nil } defer func() { _ = rows.Close() }() if !rows.Next() { return nil } r := SearchIndexResult{ EntityType: "artist", MBID: mbid, } if err := rows.Scan( &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.ArtistType, &r.Country, &r.Disambiguation, &r.SortName, &r.Aliases, &r.InLibrary, &r.IsSimilar, &r.LocalArtistID, ); err != nil { return nil } return &r } // ReleaseGroupMBIDsForCAAReleaseMBIDs takes a list of release MBIDs // (from recording.caa_release_mbid) and returns a map from release // MBID → release group MBID, by joining against the release_group // rows whose caa_release_mbid matches. Used to find parent release // groups for tracks so we can fetch cover art via the existing // release-group endpoint instead of the per-release endpoint. func (si *SearchIndex) ReleaseGroupMBIDsForCAAReleaseMBIDs( caaReleaseMBIDs []string, ) map[string]string { if len(caaReleaseMBIDs) == 0 { return nil } // Filter out empty strings — an empty input MBID would match // every release_group row that also has an empty caa_release_mbid, // producing false positives that resolve to release groups with // no actual cover art (e.g. bootlegs, demos). filtered := make([]string, 0, len(caaReleaseMBIDs)) seen := make(map[string]struct{}, len(caaReleaseMBIDs)) for _, m := range caaReleaseMBIDs { if m == "" { continue } if _, dup := seen[m]; dup { continue } seen[m] = struct{}{} filtered = append(filtered, m) } if len(filtered) == 0 { return nil } placeholders := make([]string, len(filtered)) args := make([]any, len(filtered)) for i, m := range filtered { placeholders[i] = "?" args[i] = m } query := `SELECT caa_release_mbid, mbid FROM explore_index WHERE entity_type = 'release_group' AND caa_release_mbid != '' AND caa_release_mbid IN (` + strings.Join(placeholders, ",") + `)` rows, err := si.db.QueryContext(query, args...) if err != nil { return nil } defer func() { _ = rows.Close() }() out := make(map[string]string, len(filtered)) for rows.Next() { var caaMBID, rgMBID string if err := rows.Scan(&caaMBID, &rgMBID); err == nil { out[caaMBID] = rgMBID } } return out } // LookupReleaseGroupByMBID reads a single release group row from the index. func (si *SearchIndex) LookupReleaseGroupByMBID(mbid string) *SearchIndexResult { rows, err := si.db.QueryContext( `SELECT title, artist_name, artist_mbid, popularity, listener_count, primary_type, secondary_types, release_date, in_library, COALESCE(local_release_group_id, 0), discog_fetched FROM explore_index WHERE mbid = ? AND entity_type = 'release_group' LIMIT 1`, mbid, ) if err != nil { return nil } defer func() { _ = rows.Close() }() if !rows.Next() { return nil } r := SearchIndexResult{ EntityType: "release_group", MBID: mbid, } if err := rows.Scan( &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, &r.InLibrary, &r.LocalReleaseGroupID, &r.DiscogFetched, ); err != nil { return nil } return &r } // PersistReleaseGroupLookup writes the MB-enriched metadata for a // single release group back into the index and marks it DiscogFetched // so LookupReleaseGroup's background enrichment runs at most once per // RG. Popularity is left at 0 here — upsertBatch keeps the higher of // old/new, so the dump-provided popularity is never clobbered. func (si *SearchIndex) PersistReleaseGroupLookup(rg *MBReleaseGroup) { if rg == nil || rg.MBID == "" { return } si.upsertBatch([]SearchIndexResult{{ EntityType: "release_group", MBID: rg.MBID, Title: rg.Title, ArtistName: rg.ArtistCredit, PrimaryType: rg.PrimaryType, SecondaryTypes: strings.Join(rg.SecondaryTypes, ","), ReleaseDate: rg.FirstReleaseDate, DiscogFetched: true, }}) } // TopRecordingsByArtist returns the most popular recordings for an // artist MBID from the index, ordered by popularity descending. // Falls back to returning entries without popularity data if there // aren't enough popular ones. func (si *SearchIndex) TopRecordingsByArtist(artistMBID string, limit int) []SearchIndexResult { rows, err := si.db.QueryContext( `SELECT mbid, title, artist_name, popularity, listener_count, duration, caa_release_mbid, release_name, in_library, COALESCE(local_recording_id, 0) FROM explore_index WHERE artist_mbid = ? AND entity_type = 'recording' ORDER BY popularity DESC LIMIT ?`, artistMBID, limit, ) if err != nil { return nil } defer func() { _ = rows.Close() }() var results []SearchIndexResult for rows.Next() { var r SearchIndexResult if err := rows.Scan( &r.MBID, &r.Title, &r.ArtistName, &r.Popularity, &r.ListenerCount, &r.Duration, &r.CAAReleaseMBID, &r.ReleaseName, &r.InLibrary, &r.LocalRecordingID, ); err == nil { r.EntityType = "recording" r.ArtistMBID = artistMBID results = append(results, r) } } return results } // TopReleaseGroupsByArtist returns the most popular release groups // for an artist MBID from the index, ordered by popularity descending. // Falls back to returning entries without popularity data if there // aren't enough popular ones. func (si *SearchIndex) TopReleaseGroupsByArtist(artistMBID string, limit int) []SearchIndexResult { rows, err := si.db.QueryContext( `SELECT mbid, title, artist_name, popularity, listener_count, primary_type, secondary_types, release_date, in_library, COALESCE(local_release_group_id, 0) FROM explore_index WHERE artist_mbid = ? AND entity_type = 'release_group' ORDER BY popularity DESC LIMIT ?`, artistMBID, limit, ) if err != nil { return nil } defer func() { _ = rows.Close() }() var results []SearchIndexResult for rows.Next() { var r SearchIndexResult if err := rows.Scan( &r.MBID, &r.Title, &r.ArtistName, &r.Popularity, &r.ListenerCount, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, &r.InLibrary, &r.LocalReleaseGroupID, ); err == nil { r.EntityType = "release_group" r.ArtistMBID = artistMBID results = append(results, r) } } return results } // AddFromCache inserts entries from a cached discography browse // into the search index (Tier 5: organic growth). Called when a // user views an artist page and the discography is fetched. func (si *SearchIndex) AddFromCache(artistName, artistMBID string, rgs []MBReleaseGroup) { if len(rgs) == 0 { return } entries := make([]SearchIndexResult, 0, len(rgs)+1) // Add the artist itself. entries = append(entries, SearchIndexResult{ EntityType: "artist", MBID: artistMBID, Title: artistName, ArtistName: artistName, ArtistMBID: artistMBID, Popularity: 0, // Unknown from this path. }) for _, rg := range rgs { entries = append(entries, SearchIndexResult{ EntityType: "release_group", MBID: rg.MBID, Title: rg.Title, ArtistName: artistName, ArtistMBID: artistMBID, Popularity: 0, PrimaryType: rg.PrimaryType, SecondaryTypes: strings.Join(rg.SecondaryTypes, ","), ReleaseDate: rg.FirstReleaseDate, }) } si.upsertBatch(entries) si.logger.Debug("search index: organic add", "artist", artistName, "releaseGroups", len(rgs), ) } // ExactMatches returns index rows whose normalized title (or artist // name) exactly equals the given query. Used by the top-results // intent pipeline as a dedicated retrieval source — exact matches // against high-popularity entities are almost always the right // answer and should bypass the noise of MB text search. // // Returns up to `perCategory` matches per entity type, ordered by // popularity descending. Case-insensitive; trims whitespace. func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndexResult { if !si.IsReady() { return nil } q := strings.ToLower(strings.TrimSpace(query)) if q == "" { return nil } if perCategory <= 0 { perCategory = 3 } // UNION of two equality lookups rather than `LOWER(title) = ? OR // LOWER(artist_name) = ?`. The OR form forces SQLite to scan all // ~240k rows; splitting into two equalities lets each branch seek // its partial expression index (idx_explore_title_lower / // idx_explore_artist_lower). Ordering is done in Go below since // an ORDER BY here would also defeat the index seek. rows, err := si.db.QueryContext(` SELECT entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count, duration, primary_type, secondary_types, release_date, caa_release_mbid, release_name, artist_type, country, disambiguation, sort_name, in_library, is_similar, COALESCE(local_artist_id, 0), COALESCE(local_release_group_id, 0), COALESCE(local_recording_id, 0) FROM explore_index WHERE LOWER(title) = ? AND popularity > 0 UNION SELECT entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count, duration, primary_type, secondary_types, release_date, caa_release_mbid, release_name, artist_type, country, disambiguation, sort_name, in_library, is_similar, COALESCE(local_artist_id, 0), COALESCE(local_release_group_id, 0), COALESCE(local_recording_id, 0) FROM explore_index WHERE LOWER(artist_name) = ? AND popularity > 0 `, q, q) if err != nil { return nil } defer func() { _ = rows.Close() }() var matches []SearchIndexResult for rows.Next() { var r SearchIndexResult if err := rows.Scan( &r.EntityType, &r.MBID, &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.Duration, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, &r.CAAReleaseMBID, &r.ReleaseName, &r.ArtistType, &r.Country, &r.Disambiguation, &r.SortName, &r.InLibrary, &r.IsSimilar, &r.LocalArtistID, &r.LocalReleaseGroupID, &r.LocalRecordingID, ); err != nil { continue } // For artists, only match on title (name). For recordings // and release groups, match on either title or artist name // — that way "miley cyrus" surfaces both the artist and // her recordings. titleMatch := strings.ToLower(r.Title) == q artistMatch := strings.ToLower(r.ArtistName) == q if r.EntityType == "artist" && !titleMatch { continue } if r.EntityType != "artist" && !titleMatch && !artistMatch { continue } matches = append(matches, r) } // Sort by popularity desc so the per-category cap keeps the most // popular entries (the old SQL ORDER BY guaranteed this; UNION // does not preserve order). sort.SliceStable(matches, func(i, j int) bool { return matches[i].Popularity > matches[j].Popularity }) // Group by entity type, capping at perCategory each. buckets := map[string][]SearchIndexResult{ "artist": nil, "release_group": nil, "recording": nil, } for _, r := range matches { bucket := buckets[r.EntityType] if len(bucket) >= perCategory { continue } buckets[r.EntityType] = append(bucket, r) } var out []SearchIndexResult out = append(out, buckets["artist"]...) out = append(out, buckets["release_group"]...) out = append(out, buckets["recording"]...) return out } // Search queries the local FTS5 index and returns matches ordered // by relevance (popularity-blended). Returns nil when the index // hasn't finished its initial build. func (si *SearchIndex) Search(ctx context.Context, query string, limit int) []SearchIndexResult { if !si.IsReady() { return nil } if limit <= 0 { limit = 20 } ftsQuery := buildFTSQuery(query) if ftsQuery == "" { return nil } generic := isGenericQuery(query) // Generic short-prefix queries are the slow ones; serve repeats from // the in-memory cache and route the rest through the champion tier. if generic { if hits, ok := si.prefixCacheGet(ftsQuery, limit); ok { return hits } } hits := si.runSearch(ctx, ftsQuery, limit, generic) // Typo-tolerant fallback: a thin result usually means the query is // misspelled and matched no token prefix. Rescue with bigram // similarity over the champion set before giving up. Skipped once a // search is superseded (ctx cancelled) — the result is discarded // anyway. if len(hits) < fuzzyRescueThinHits && ctx.Err() == nil { hits = si.appendFuzzyRescue(ctx, query, limit, hits) } // Cache only fully-formed generic results — never a partial list from // a superseded (cancelled) query. if generic && ctx.Err() == nil { si.prefixCachePut(ftsQuery, limit, hits) } return hits } // appendFuzzyRescue runs the bigram rescue pass and merges its hits into // the existing (thin) result, de-duplicated by MBID and preserving the // original hits first. Returns the original slice unchanged when rescue // finds nothing new. func (si *SearchIndex) appendFuzzyRescue( ctx context.Context, query string, limit int, hits []SearchIndexResult, ) []SearchIndexResult { rescued := si.fuzzyRescue(ctx, query, limit) if len(rescued) == 0 { return hits } seen := make(map[string]struct{}, len(hits)+len(rescued)) for _, h := range hits { seen[h.MBID] = struct{}{} } for _, r := range rescued { if _, dup := seen[r.MBID]; dup { continue } seen[r.MBID] = struct{}{} hits = append(hits, r) if len(hits) >= limit { break } } return hits } // fuzzyRescue is the typo-tolerant fallback for Search. It scans the // champion set (high-popularity + owned rows) and scores each candidate // by character-bigram overlap (Dice) against its title and artist name, // so a misspelled query still surfaces its intended entity even though it // shares no token prefix with any indexed name. Returns up to `limit` // hits above fuzzyRescueMinScore, ordered by fuzzy score with popularity // as a tie-break. Bounded work that fires only on the rare thin query, // so it never touches the fast common path. func (si *SearchIndex) fuzzyRescue( ctx context.Context, query string, limit int, ) []SearchIndexResult { qSet := fuzzyBigrams(query) if len([]rune(fuzzyNormalize(query))) < fuzzyRescueMinLen || len(qSet) == 0 { return nil } // Champion membership predicate, mirrored from rebuildChampionIndex: // a row below the popularity floor that isn't owned can't be a // meaningful popular match, so scoring it would only add noise. rows, err := si.db.QueryContextWith(ctx, ` SELECT id, title, artist_name, popularity FROM explore_index WHERE popularity >= ? OR in_library = 1 `, championPopThreshold) if err != nil { if !errors.Is(err, context.Canceled) { si.logger.Warn("fuzzy rescue scan error", "error", err) } return nil } defer func() { _ = rows.Close() }() type scored struct { id int64 score float64 pop int } var candidates []scored scanned := 0 for rows.Next() { // Periodically honour cancellation: a superseded search shouldn't // keep scoring tens of thousands of rows. scanned++ if scanned%4096 == 0 && ctx.Err() != nil { return nil } var ( id int64 title string artist string pop int ) if err := rows.Scan(&id, &title, &artist, &pop); err != nil { continue } score := diceCoefficient(qSet, fuzzyBigrams(title)) if s := diceCoefficient(qSet, fuzzyBigrams(artist)); s > score { score = s } if score >= fuzzyRescueMinScore { candidates = append(candidates, scored{id: id, score: score, pop: pop}) } } if len(candidates) == 0 { return nil } // Best bigram match first; popularity breaks near-ties so the more // prominent entity wins when two names are equally close. sort.SliceStable(candidates, func(i, j int) bool { if candidates[i].score != candidates[j].score { return candidates[i].score > candidates[j].score } return candidates[i].pop > candidates[j].pop }) if len(candidates) > limit { candidates = candidates[:limit] } ids := make([]int64, len(candidates)) for i, c := range candidates { ids[i] = c.id } return si.rowsByIDs(ctx, ids) } // rowsByIDs loads full index rows for the given explore_index ids, // returned in the same order as `ids` (rows are keyed by id in a map, // then re-emitted in input order). Used by fuzzyRescue to hydrate the // lightweight candidate ids it scored into full search results. func (si *SearchIndex) rowsByIDs(ctx context.Context, ids []int64) []SearchIndexResult { if len(ids) == 0 { return nil } placeholders := make([]string, len(ids)) args := make([]any, len(ids)) for i, id := range ids { placeholders[i] = "?" args[i] = id } query := ` SELECT id, entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count, duration, primary_type, secondary_types, release_date, caa_release_mbid, release_name, artist_type, country, disambiguation, sort_name, in_library, is_similar, COALESCE(local_artist_id, 0), COALESCE(local_release_group_id, 0), COALESCE(local_recording_id, 0) FROM explore_index WHERE id IN (` + strings.Join(placeholders, ",") + `)` rows, err := si.db.QueryContextWith(ctx, query, args...) if err != nil { if !errors.Is(err, context.Canceled) { si.logger.Warn("fuzzy rescue hydrate error", "error", err) } return nil } defer func() { _ = rows.Close() }() byID := make(map[int64]SearchIndexResult, len(ids)) for rows.Next() { var ( id int64 r SearchIndexResult ) if err := rows.Scan( &id, &r.EntityType, &r.MBID, &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.Duration, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, &r.CAAReleaseMBID, &r.ReleaseName, &r.ArtistType, &r.Country, &r.Disambiguation, &r.SortName, &r.InLibrary, &r.IsSimilar, &r.LocalArtistID, &r.LocalReleaseGroupID, &r.LocalRecordingID, ); err != nil { continue } byID[id] = r } out := make([]SearchIndexResult, 0, len(ids)) for _, id := range ids { if r, ok := byID[id]; ok { out = append(out, r) } } return out } // runSearch dispatches an FTS query to the champion tier when the query // is generic and the champion index is ready, falling back to the full // index when the champion result is thin (few high-popularity matches) // or the champion index isn't built yet. func (si *SearchIndex) runSearch( ctx context.Context, ftsQuery string, limit int, generic bool, ) []SearchIndexResult { if generic { if si.championIsReady() { cStart := time.Now() hits := si.queryFTS(ctx, "explore_champion_fts", ftsQuery, limit) if len(hits) >= limit { si.logger.Info("search path: champion", "fts", ftsQuery, "rows", len(hits), "elapsed", time.Since(cStart).Round(time.Millisecond), ) return hits } // Thin champion result: the full index is authoritative, and a // query the champion couldn't fill matches few rows there too. si.logger.Info("search path: champion thin -> full", "fts", ftsQuery, "champion_rows", len(hits), "champion_elapsed", time.Since(cStart).Round(time.Millisecond), ) } else { // First generic query before the champion exists — build it in // the background so the next one is fast. si.logger.Info("search path: champion not ready -> full", "fts", ftsQuery) si.scheduleChampionRebuild() } } fStart := time.Now() hits := si.queryFTS(ctx, "explore_index_fts", ftsQuery, limit) if generic { si.logger.Info("search path: full", "fts", ftsQuery, "rows", len(hits), "elapsed", time.Since(fStart).Round(time.Millisecond), ) } return hits } // queryFTS runs the popularity-blended ranking query against the named // FTS table (the full index or the champion tier — identical schema, so // the only difference is how many rows MATCH). func (si *SearchIndex) queryFTS( ctx context.Context, ftsTable, ftsQuery string, limit int, ) []SearchIndexResult { // ftsTable is a trusted in-package constant, never user input. sqlText := fmt.Sprintf(` SELECT i.entity_type, i.mbid, i.title, i.artist_name, i.artist_mbid, i.popularity, i.listener_count, i.duration, i.primary_type, i.secondary_types, i.release_date, i.caa_release_mbid, i.release_name, i.artist_type, i.country, i.disambiguation, i.sort_name, i.in_library, i.is_similar, COALESCE(i.local_artist_id, 0), COALESCE(i.local_release_group_id, 0), COALESCE(i.local_recording_id, 0) FROM explore_index i JOIN %[1]s f ON f.rowid = i.id WHERE %[1]s MATCH ? ORDER BY bm25(%[1]s, 3.0, 1.0, 0.5) - (ln(i.popularity + 1) * 1.5) - (i.in_library * 3.0) - (i.is_similar * 1.5) LIMIT ? `, ftsTable) rows, err := si.db.QueryContextWith(ctx, sqlText, ftsQuery, limit) if err != nil { // A superseded search has its context cancelled on purpose; that // is expected shutdown, not a query fault, so don't warn on it. if !errors.Is(err, context.Canceled) { si.logger.Warn("search index query error", "table", ftsTable, "ftsQuery", ftsQuery, "error", err, ) } return nil } defer func() { _ = rows.Close() }() var results []SearchIndexResult for rows.Next() { var r SearchIndexResult if err := rows.Scan( &r.EntityType, &r.MBID, &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.Duration, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, &r.CAAReleaseMBID, &r.ReleaseName, &r.ArtistType, &r.Country, &r.Disambiguation, &r.SortName, &r.InLibrary, &r.IsSimilar, &r.LocalArtistID, &r.LocalReleaseGroupID, &r.LocalRecordingID, ); err != nil { si.logger.Warn("search index scan error", "error", err) continue } results = append(results, r) } return results } // isGenericQuery reports whether a query has no selective (long) token // and therefore matches a broad slice of the index — the case the // champion tier and prefix cache exist to accelerate. func isGenericQuery(query string) bool { words := splitWords(query) if len(words) == 0 { return false } for _, w := range words { if len([]rune(w)) > genericMaxTokenLen { return false } } return true } // --------------------------------------------------------------------------- // Champion tier // --------------------------------------------------------------------------- // championIsReady reports whether the champion FTS is populated. func (si *SearchIndex) championIsReady() bool { si.mu.RLock() defer si.mu.RUnlock() return si.championReady } // scheduleChampionRebuild rebuilds the champion FTS in the background, // unless one is already running or the main index isn't ready yet. The // rebuild runs in a single write transaction, so concurrent searches // keep reading the previous champion snapshot (WAL) until it commits. func (si *SearchIndex) scheduleChampionRebuild() { si.mu.Lock() if si.championBuilding || !si.ready { si.mu.Unlock() return } si.championBuilding = true si.mu.Unlock() go func() { // Use a stable context, not any per-search one, so a superseded // search can't cancel the rebuild midway. ctx := si.runtimeCtx if ctx == nil { ctx = context.Background() } start := time.Now() err := si.rebuildChampionIndex(ctx) si.mu.Lock() si.championBuilding = false // Keep a previously-good champion usable if a refresh failed; mark // ready on success. si.championReady = si.championReady || err == nil si.mu.Unlock() // The champion set changed, so any memoised generic results are // stale. si.clearPrefixCache() if err != nil { si.logger.Warn("champion index rebuild failed", "error", err) return } si.setMeta(championBuiltKey, time.Now().UTC().Format(time.RFC3339)) si.logger.Info("champion index rebuilt", "elapsed", time.Since(start).Round(time.Millisecond), ) }() } // rebuildChampionIndex repopulates the champion FTS from the high- // popularity and owned rows of explore_index. A row below the // popularity floor can never place in a generic query's top results // (ranking there is popularity-dominated), so dropping it is lossless // for those queries; owned rows are always kept for their in-library // boost. func (si *SearchIndex) rebuildChampionIndex(ctx context.Context) error { tx, err := si.db.BeginTx() if err != nil { return fmt.Errorf("begin champion rebuild: %w", err) } defer func() { _ = tx.Rollback() }() if _, err := tx.ExecContext(ctx, `INSERT INTO explore_champion_fts(explore_champion_fts) VALUES('delete-all')`, ); err != nil { return fmt.Errorf("clear champion fts: %w", err) } if _, err := tx.ExecContext(ctx, ` INSERT INTO explore_champion_fts(rowid, title, artist_name, aliases) SELECT id, title, artist_name, aliases FROM explore_index WHERE popularity >= ? OR in_library = 1 `, championPopThreshold); err != nil { return fmt.Errorf("populate champion fts: %w", err) } if err := tx.Commit(); err != nil { return fmt.Errorf("commit champion rebuild: %w", err) } return nil } // --------------------------------------------------------------------------- // Prefix result cache // --------------------------------------------------------------------------- func prefixCacheKey(ftsQuery string, limit int) string { return strconv.Itoa(limit) + "\x00" + ftsQuery } func (si *SearchIndex) prefixCacheGet(ftsQuery string, limit int) ([]SearchIndexResult, bool) { si.prefixCacheMu.Lock() defer si.prefixCacheMu.Unlock() key := prefixCacheKey(ftsQuery, limit) entry, ok := si.prefixCache[key] if !ok { return nil, false } if time.Since(entry.created) > prefixCacheTTL { delete(si.prefixCache, key) return nil, false } return entry.hits, true } func (si *SearchIndex) prefixCachePut(ftsQuery string, limit int, hits []SearchIndexResult) { si.prefixCacheMu.Lock() defer si.prefixCacheMu.Unlock() if si.prefixCache == nil { si.prefixCache = make(map[string]prefixCacheEntry) } // Generic keys are few and cheap to recompute; when the cap is hit, // drop everything rather than track per-entry LRU state. if len(si.prefixCache) >= prefixCacheMaxEntries { si.prefixCache = make(map[string]prefixCacheEntry) } si.prefixCache[prefixCacheKey(ftsQuery, limit)] = prefixCacheEntry{ hits: hits, created: time.Now(), } } func (si *SearchIndex) clearPrefixCache() { si.prefixCacheMu.Lock() si.prefixCache = nil si.prefixCacheMu.Unlock() } // --------------------------------------------------------------------------- // FTS query building // --------------------------------------------------------------------------- func buildFTSQuery(query string) string { words := splitWords(query) if len(words) == 0 { return "" } var b strings.Builder for i, w := range words { if i > 0 { b.WriteByte(' ') } b.WriteString(w) b.WriteByte('*') } return b.String() } func splitWords(s string) []string { var words []string current := "" for _, r := range s { if isWordChar(r) { current += string(r) } else if current != "" { words = append(words, current) current = "" } } if current != "" { words = append(words, current) } return words } func isWordChar(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r >= 0x80 } type lbSimilarArtistWire struct { ArtistMBID string `json:"artist_mbid"` Name string `json:"name"` Score int `json:"score"` ReferenceMBID string `json:"reference_mbid"` // which seed artist this result belongs to } // fetchSimilarArtistsBatch queries the labs similar-artists endpoint // for multiple seed MBIDs. Despite the name, this actually fans // out one request per seed: the labs API's multi-seed mode is // broken (results for different seeds get mis-labeled, and some // seeds return zero), so batching with multiple artist_mbids is // not viable. Concurrency is bounded by indexerRate to respect // the labs rate limit; each call goes through the provided LB // client's rate limiter and cache. func (si *SearchIndex) fetchSimilarArtistsBatch( ctx context.Context, lb *ListenBrainzClient, seedMBIDs []string, ) map[string][]lbSimilarArtistWire { if len(seedMBIDs) == 0 { return nil } var ( mu sync.Mutex grouped = make(map[string][]lbSimilarArtistWire, len(seedMBIDs)) wg sync.WaitGroup ) sem := make(chan struct{}, indexerRate) for _, seedMBID := range seedMBIDs { if ctx.Err() != nil { break } sem <- struct{}{} wg.Add(1) go func(seed string) { defer func() { <-sem wg.Done() }() // Use the LB client's per-seed GET form — goes through // the shared rate limiter and cache. The multi-seed // POST form is not viable (see function comment). similar, err := lb.SimilarArtists(ctx, seed) if err != nil || len(similar) == 0 { return } // Convert to the internal wire type used by the caller // and trim to indexSimilarPerArtist. if len(similar) > indexSimilarPerArtist { similar = similar[:indexSimilarPerArtist] } results := make([]lbSimilarArtistWire, len(similar)) for i, s := range similar { results[i] = lbSimilarArtistWire{ ArtistMBID: s.ArtistMBID, Name: s.Name, Score: int(s.Score), ReferenceMBID: seed, } } mu.Lock() grouped[seed] = results mu.Unlock() }(seedMBID) } wg.Wait() return grouped } // --------------------------------------------------------------------------- // Shared: index artist discographies // --------------------------------------------------------------------------- func (si *SearchIndex) indexOneArtist( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, ) { if ctx.Err() != nil { return } rgLimit, recLimit := indexMaxRGs, indexMaxRecs // Run LB discography fetches and MB artist image resolution // concurrently — they use different rate limiters so they // don't block each other. var ( rgs []SearchIndexResult recs []SearchIndexResult wg sync.WaitGroup ) // LB pipeline: top release groups + top recordings. wg.Add(1) go func() { defer wg.Done() rgs = si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit) recs = si.fetchTopRecordings(ctx, lb, artist, recLimit) }() // MB pipeline: resolve + cache artist image (uses MB rate limiter). wg.Add(1) go func() { defer wg.Done() if si.artistImg != nil { si.artistImg.GetArtistImage(artist.ArtistMBID) } }() wg.Wait() // Write the artist entry into the index so indexedArtistMBIDs() // recognises this artist as processed on subsequent builds. // Only mark DiscogFetched=true if at least one of the discography // fetches actually returned data — a transient API failure should // allow a retry on the next build, not permanently claim the // artist as indexed. Also stores aliases and detail fields from // the now-cached MB rels (populated by the image resolution above) // for FTS search. if si.artistImg != nil { gotData := len(rgs) > 0 || len(recs) > 0 artistEntry := SearchIndexResult{ EntityType: "artist", MBID: artist.ArtistMBID, Title: artist.ArtistName, ArtistName: artist.ArtistName, ArtistMBID: artist.ArtistMBID, Popularity: artist.ListenCount, DiscogFetched: gotData, } if details := si.artistImg.GetArtistDetails(artist.ArtistMBID); details != nil { artistEntry.ArtistType = details.Type artistEntry.Country = details.Country artistEntry.Disambiguation = details.Disambiguation artistEntry.SortName = details.SortName artistEntry.Aliases = details.Aliases } si.upsertBatch([]SearchIndexResult{artistEntry}) } // Batch write discography results. all := make([]SearchIndexResult, 0, len(rgs)+len(recs)) all = append(all, rgs...) all = append(all, recs...) for i := 0; i < len(all); i += indexBatchSize { end := i + indexBatchSize if end > len(all) { end = len(all) } si.upsertBatch(all[i:end]) } } func (si *SearchIndex) fetchTopReleaseGroups( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, maxCount int, ) []SearchIndexResult { url := fmt.Sprintf( "%s/1/popularity/top-release-groups-for-artist/%s", listenBrainzBaseURL, artist.ArtistMBID, ) body, err := lb.doGet(ctx, url) if err != nil { si.logger.Debug("search index: top RGs failed", "artist", artist.ArtistName, "error", err, ) return nil } var raw []struct { ReleaseGroupMBID string `json:"release_group_mbid"` TotalListenCount int `json:"total_listen_count"` ReleaseGroup struct { Name string `json:"name"` Type string `json:"type"` Date string `json:"date"` CAAReleaseMBID string `json:"caa_release_mbid"` } `json:"release_group"` Artist struct { Artists []struct { ArtistMBID string `json:"artist_mbid"` Name string `json:"name"` } `json:"artists"` } `json:"artist"` } if err := json.Unmarshal(body, &raw); err != nil { return nil } limit := maxCount if limit > len(raw) { limit = len(raw) } results := make([]SearchIndexResult, 0, limit) for _, r := range raw[:limit] { if r.TotalListenCount < indexMinPopularity { continue } artistName := artist.ArtistName artistMBID := artist.ArtistMBID if len(r.Artist.Artists) > 0 { artistName = r.Artist.Artists[0].Name artistMBID = r.Artist.Artists[0].ArtistMBID } results = append(results, SearchIndexResult{ EntityType: "release_group", MBID: r.ReleaseGroupMBID, Title: r.ReleaseGroup.Name, ArtistName: artistName, ArtistMBID: artistMBID, Popularity: r.TotalListenCount, PrimaryType: r.ReleaseGroup.Type, ReleaseDate: r.ReleaseGroup.Date, CAAReleaseMBID: r.ReleaseGroup.CAAReleaseMBID, }) } return results } func (si *SearchIndex) fetchTopRecordings( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, maxCount int, ) []SearchIndexResult { url := fmt.Sprintf( "%s/1/popularity/top-recordings-for-artist/%s", listenBrainzBaseURL, artist.ArtistMBID, ) body, err := lb.doGet(ctx, url) if err != nil { return nil } var raw []lbTopRecordingWire if err := json.Unmarshal(body, &raw); err != nil { return nil } limit := maxCount if limit > len(raw) { limit = len(raw) } results := make([]SearchIndexResult, 0, limit) for _, r := range raw[:limit] { if r.TotalListenCount < indexMinPopularity { continue } results = append(results, SearchIndexResult{ EntityType: "recording", MBID: r.RecordingMBID, Title: r.RecordingName, ArtistName: r.ArtistName, ArtistMBID: artist.ArtistMBID, Popularity: r.TotalListenCount, Duration: r.Length, CAAReleaseMBID: r.CAAReleaseMBID, ReleaseName: r.ReleaseName, }) } return results } // chunkStrings splits a slice into chunks of at most size n. func chunkStrings(s []string, n int) [][]string { var chunks [][]string for i := 0; i < len(s); i += n { end := i + n if end > len(s) { end = len(s) } chunks = append(chunks, s[i:end]) } return chunks } // --------------------------------------------------------------------------- // Database writes // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Unified write API // --------------------------------------------------------------------------- // // All writes to explore_index go through upsertBatch. There are no // side-channel write paths — if data needs to land in the index, it // flows through a SearchIndexResult struct that carries every field. // The merge semantics are: non-empty incoming values replace existing // empty values, and numeric fields use "highest wins" for popularity/ // listener_count/duration so older richer data survives refreshes. // upsertBatch writes a batch of SearchIndexResult entries to the index // inside a single transaction. This is the ONE function that all // writes go through. All fields are handled — callers don't need to // know which columns exist for which entity types. func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { if len(entries) == 0 { return } tx, err := si.db.BeginTx() if err != nil { si.logger.Warn("search index: begin tx error", "error", err) return } for _, e := range entries { if e.MBID == "" { continue // skip entries without MBIDs — can't be looked up } inLib := 0 if e.InLibrary { inLib = 1 } isSim := 0 if e.IsSimilar { isSim = 1 } discogFetched := 0 if e.DiscogFetched { discogFetched = 1 } if _, err := tx.Exec(` INSERT INTO explore_index ( entity_type, mbid, title, artist_name, artist_mbid, aliases, popularity, listener_count, duration, caa_release_mbid, release_name, primary_type, secondary_types, release_date, artist_type, country, disambiguation, sort_name, in_library, is_similar, local_artist_id, local_release_group_id, local_recording_id, discog_fetched, schema_version ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, 0), NULLIF(?, 0), NULLIF(?, 0), ?, ? ) ON CONFLICT(mbid) DO UPDATE SET -- Title and artist info: don't clobber a good value with -- an empty string or with the MBID itself (which can sneak -- in via fallback paths in AddFromCache). title = CASE WHEN excluded.title != '' AND excluded.title != excluded.mbid THEN excluded.title ELSE title END, artist_name = CASE WHEN excluded.artist_name != '' AND excluded.artist_name != excluded.artist_mbid THEN excluded.artist_name ELSE artist_name END, artist_mbid = CASE WHEN excluded.artist_mbid != '' THEN excluded.artist_mbid ELSE artist_mbid END, aliases = CASE WHEN excluded.aliases != '' THEN excluded.aliases ELSE aliases END, -- Highest wins for popularity + listener_count (refreshes can go up). popularity = CASE WHEN excluded.popularity > popularity THEN excluded.popularity ELSE popularity END, listener_count = CASE WHEN excluded.listener_count > listener_count THEN excluded.listener_count ELSE listener_count END, -- Non-empty wins for all other optional fields (never clobber with empty). duration = CASE WHEN excluded.duration > 0 THEN excluded.duration ELSE duration END, caa_release_mbid = CASE WHEN excluded.caa_release_mbid != '' THEN excluded.caa_release_mbid ELSE caa_release_mbid END, release_name = CASE WHEN excluded.release_name != '' THEN excluded.release_name ELSE release_name END, primary_type = CASE WHEN excluded.primary_type != '' THEN excluded.primary_type ELSE primary_type END, secondary_types = CASE WHEN excluded.secondary_types != '' THEN excluded.secondary_types ELSE secondary_types END, release_date = CASE WHEN excluded.release_date != '' THEN excluded.release_date ELSE release_date END, artist_type = CASE WHEN excluded.artist_type != '' THEN excluded.artist_type ELSE artist_type END, country = CASE WHEN excluded.country != '' THEN excluded.country ELSE country END, disambiguation = CASE WHEN excluded.disambiguation != '' THEN excluded.disambiguation ELSE disambiguation END, sort_name = CASE WHEN excluded.sort_name != '' THEN excluded.sort_name ELSE sort_name END, -- Flags and cross-references: non-null wins. in_library = MAX(in_library, excluded.in_library), is_similar = MAX(is_similar, excluded.is_similar), discog_fetched = MAX(discog_fetched, excluded.discog_fetched), local_artist_id = COALESCE(excluded.local_artist_id, local_artist_id), local_release_group_id = COALESCE(excluded.local_release_group_id, local_release_group_id), local_recording_id = COALESCE(excluded.local_recording_id, local_recording_id), schema_version = MAX(schema_version, excluded.schema_version) `, e.EntityType, e.MBID, e.Title, e.ArtistName, e.ArtistMBID, e.Aliases, e.Popularity, e.ListenerCount, e.Duration, e.CAAReleaseMBID, e.ReleaseName, e.PrimaryType, e.SecondaryTypes, e.ReleaseDate, e.ArtistType, e.Country, e.Disambiguation, e.SortName, inLib, isSim, e.LocalArtistID, e.LocalReleaseGroupID, e.LocalRecordingID, discogFetched, currentSchemaVersion, ); err != nil { si.logger.Warn("search index: upsert error", "mbid", e.MBID, "error", err, ) } } if err := tx.Commit(); err != nil { si.logger.Warn("search index: commit error", "error", err) } } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- // getLibraryArtistMBIDs returns MBIDs for all library artists that have one. // Used when Tier 3 was skipped but Tier 4 needs the library MBID list. func (si *SearchIndex) getLibraryArtistMBIDs() []string { rows, err := si.db.QueryContext( "SELECT DISTINCT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''", ) if err != nil { return nil } defer func() { _ = rows.Close() }() var mbids []string for rows.Next() { var mbid string if err := rows.Scan(&mbid); err == nil { mbids = append(mbids, mbid) } } return mbids } // hasMeta reports whether a key exists in explore_index_meta. func (si *SearchIndex) hasMeta(key string) bool { rows, err := si.db.QueryContext( "SELECT 1 FROM explore_index_meta WHERE key = ?", key, ) if err != nil { return false } defer func() { _ = rows.Close() }() return rows.Next() } // PopulateLocalCrossReferences syncs the local library into the search // index: every MB-verified library artist, release group, and recording // is upserted into explore_index, flagged in_library with its local id. // Call after a library scan completes. // // This is the local half of the index — it makes owned content // searchable regardless of the dump's popularity floor, using only data // already in the library tables (no API calls). upsertBatch's conflict // rules mean dump-seeded rows are merely tagged in_library while // below-floor owned entities are inserted fresh (popularity 0, to be // filled later by incremental dumps or a lazy discography fetch). // MBID-less local content is intentionally excluded — the explore index // is MBID-keyed, and unmatched files are served by the library search. func (si *SearchIndex) PopulateLocalCrossReferences() { entries := si.collectLibraryEntities() if len(entries) == 0 { si.logger.Info("library sync: no MB-verified library entities to index") si.setMeta(localXrefReadyKey, "1") return } const batchSize = 1000 for i := 0; i < len(entries); i += batchSize { end := i + batchSize if end > len(entries) { end = len(entries) } si.upsertBatch(entries[i:end]) } si.setMeta(localXrefReadyKey, "1") si.logger.Info("library sync: upserted library entities into index", "count", len(entries)) } // collectLibraryEntities builds index entries for every library artist, // release group, and recording that carries a MusicBrainz ID. Artist // credit strings and the primary artist MBID are resolved from the local // artist_credit tables so no network lookup is needed. func (si *SearchIndex) collectLibraryEntities() []SearchIndexResult { var entries []SearchIndexResult // Artists. artistRows, err := si.db.QueryContext( "SELECT id, name, mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''", ) if err == nil { for artistRows.Next() { var ( id int64 name, mbid string ) if err := artistRows.Scan(&id, &name, &mbid); err != nil { continue } entries = append(entries, SearchIndexResult{ EntityType: "artist", MBID: mbid, Title: name, ArtistName: name, ArtistMBID: mbid, InLibrary: true, LocalArtistID: id, }) } _ = artistRows.Close() } else { si.logger.Warn("library sync: query artists failed", "error", err) } // Release groups. The correlated subquery picks the primary // credited artist's MBID (if it has one). rgRows, err := si.db.QueryContext(` SELECT rg.id, rg.name, rg.mbid, COALESCE(ac.text, ''), COALESCE(( SELECT a.mbid FROM artist_credit_artist aca JOIN artists a ON a.id = aca.artist_id WHERE aca.credit_id = rg.album_artist_credit_id AND a.mbid IS NOT NULL AND a.mbid != '' LIMIT 1 ), '') FROM release_groups rg LEFT JOIN artist_credit ac ON ac.id = rg.album_artist_credit_id WHERE rg.mbid IS NOT NULL AND rg.mbid != '' `) if err == nil { for rgRows.Next() { var ( id int64 name, mbid, credit, artistMB string ) if err := rgRows.Scan(&id, &name, &mbid, &credit, &artistMB); err != nil { continue } entries = append(entries, SearchIndexResult{ EntityType: "release_group", MBID: mbid, Title: name, ArtistName: credit, ArtistMBID: artistMB, InLibrary: true, LocalReleaseGroupID: id, }) } _ = rgRows.Close() } else { si.logger.Warn("library sync: query release groups failed", "error", err) } // Recordings. recRows, err := si.db.QueryContext(` SELECT r.id, r.name, r.mbid, COALESCE(ac.text, ''), COALESCE(( SELECT a.mbid FROM artist_credit_artist aca JOIN artists a ON a.id = aca.artist_id WHERE aca.credit_id = r.artist_credit_id AND a.mbid IS NOT NULL AND a.mbid != '' LIMIT 1 ), '') FROM recordings r LEFT JOIN artist_credit ac ON ac.id = r.artist_credit_id WHERE r.mbid IS NOT NULL AND r.mbid != '' `) if err == nil { for recRows.Next() { var ( id int64 name, mbid, credit, artistMB string ) if err := recRows.Scan(&id, &name, &mbid, &credit, &artistMB); err != nil { continue } entries = append(entries, SearchIndexResult{ EntityType: "recording", MBID: mbid, Title: name, ArtistName: credit, ArtistMBID: artistMB, InLibrary: true, LocalRecordingID: id, }) } _ = recRows.Close() } else { si.logger.Warn("library sync: query recordings failed", "error", err) } return entries } // storeSimilarArtists persists the similar artist relationships // for a source artist into the similar_artist_map table. func (si *SearchIndex) storeSimilarArtists(sourceMBID string, similar []lbSimilarArtistWire) { if len(similar) == 0 { return } tx, err := si.db.BeginTx() if err != nil { return } defer func() { _ = tx.Rollback() }() // Clear existing entries for this source to avoid stale data. _, _ = tx.Exec( "DELETE FROM similar_artist_map WHERE source_artist_mbid = ?", sourceMBID, ) for _, s := range similar { _, _ = tx.Exec(` INSERT OR IGNORE INTO similar_artist_map (source_artist_mbid, similar_artist_mbid, similar_artist_name, score) VALUES (?, ?, ?, ?) `, sourceMBID, s.ArtistMBID, s.Name, s.Score) } _ = tx.Commit() } // PopularityData holds both listen count and listener count for a // single entity. Used by BackfillPopularity and the popularity // pipeline to pass both metrics together. type PopularityData struct { ListenCount int ListenerCount int } // BackfillPopularity writes LB popularity values back to the index // for MBIDs that already exist. Called after LB API responses so // subsequent searches use the index instead of re-fetching from LB. func (si *SearchIndex) BackfillPopularity(updates map[string]PopularityData) { if len(updates) == 0 { return } tx, err := si.db.BeginTx() if err != nil { return } defer func() { _ = tx.Rollback() }() for mbid, data := range updates { if data.ListenCount <= 0 { continue } _, _ = tx.Exec( `UPDATE explore_index SET popularity = CASE WHEN ? > popularity THEN ? ELSE popularity END, listener_count = CASE WHEN ? > listener_count THEN ? ELSE listener_count END WHERE mbid = ?`, data.ListenCount, data.ListenCount, data.ListenerCount, data.ListenerCount, mbid, ) } _ = tx.Commit() } // BackfillPopularitySimple is a convenience wrapper for callers that // only have listen counts (no listener count). func (si *SearchIndex) BackfillPopularitySimple(updates map[string]int) { if len(updates) == 0 { return } full := make(map[string]PopularityData, len(updates)) for mbid, pop := range updates { full[mbid] = PopularityData{ListenCount: pop} } si.BackfillPopularity(full) } // GetSimilarityScores returns the highest similarity score for each // MBID that appears in similar_artist_map as a similar artist. // Returns a map of mbid → max similarity score. func (si *SearchIndex) GetSimilarityScores(mbids []string) map[string]int { if len(mbids) == 0 { return nil } placeholders := make([]string, len(mbids)) args := make([]any, len(mbids)) for i, m := range mbids { placeholders[i] = "?" args[i] = m } query := "SELECT similar_artist_mbid, MAX(score) FROM similar_artist_map WHERE similar_artist_mbid IN (" + strings.Join( placeholders, ",", ) + ") GROUP BY similar_artist_mbid" rows, err := si.db.QueryContext(query, args...) if err != nil { return nil } defer func() { _ = rows.Close() }() result := make(map[string]int, len(mbids)) for rows.Next() { var ( mbid string score int ) if err := rows.Scan(&mbid, &score); err == nil { result[mbid] = score } } return result } // InvalidateDiscographies clears the dump-import completion marker so // the next StartBuild re-runs the full dump import. func (si *SearchIndex) InvalidateDiscographies() { _, _ = si.db.ExecContext( "DELETE FROM explore_index_meta WHERE key = 'dump_import_done'", ) } func (si *SearchIndex) setMeta(key, value string) { if _, err := si.db.ExecContext( "INSERT OR REPLACE INTO explore_index_meta (key, value) VALUES (?, ?)", key, value, ); err != nil { si.logger.Warn("search index: set meta error", "key", key, "error", err) } } // deleteMeta removes a key from explore_index_meta, e.g. to invalidate a // "ready" marker so the next launch re-runs a gated build step. func (si *SearchIndex) deleteMeta(key string) { if _, err := si.db.ExecContext( "DELETE FROM explore_index_meta WHERE key = ?", key, ); err != nil { si.logger.Warn("search index: delete meta error", "key", key, "error", err) } } // MarkReadyIfPopulated sets the index as ready for querying if it // already contains data from a previous build. Called eagerly at // service creation so the index is queryable before StartBuild runs. func (si *SearchIndex) MarkReadyIfPopulated() { rows, err := si.db.QueryContext("SELECT COUNT(*) FROM explore_index") if err != nil { return } var count int if rows.Next() { _ = rows.Scan(&count) } // Release the connection before any further query: under a // single-connection pool (tests) a nested query while these rows are // open would deadlock. _ = rows.Close() if count == 0 { return } // Trust a champion index built in a previous run; otherwise build it // below now that the main index is known-ready. championBuilt := si.hasMeta(championBuiltKey) si.mu.Lock() si.ready = true si.championReady = championBuilt si.mu.Unlock() si.logger.Info("search index: using existing index", "entries", count) if !championBuilt { si.scheduleChampionRebuild() } }