perf: batch popularity lookups in single SQLite query (100+ → 1)
boostWithIndexPopularity was calling GetPopularity() and IsInLibrary() individually for every search result — ~100 separate SQLite queries for a typical search (20 artists × 2 + 20 RGs × 2 + 20 recordings). This took 7.5s on the 'fast path' that was supposed to take ~5ms. Added GetPopularityBatch(mbids) — collects all MBIDs across all entity types and fetches popularity + in_library in a single SELECT ... WHERE mbid IN (...) query. The library bonus (+10M) is applied during the batch scan. Expected Phase 2 improvement: ~7.5s → <10ms.
This commit is contained in:
+30
-21
@@ -933,47 +933,56 @@ var mbSpecialPurposeArtists = map[string]bool{
|
|||||||
// just SQLite lookups. This is the fast path used when the index
|
// just SQLite lookups. This is the fast path used when the index
|
||||||
// is ready.
|
// is ready.
|
||||||
func (e *Service) boostWithIndexPopularity(result *MBSearchResult) {
|
func (e *Service) boostWithIndexPopularity(result *MBSearchResult) {
|
||||||
// Look up popularity for all artist MBIDs.
|
// Collect all MBIDs across all entity types.
|
||||||
// Give a large bonus to library artists so they rank first.
|
allMBIDs := make([]string, 0,
|
||||||
artistPop := make(map[string]int, len(result.Artists))
|
len(result.Artists)+len(result.ReleaseGroups)+len(result.Recordings))
|
||||||
|
|
||||||
for _, a := range result.Artists {
|
for _, a := range result.Artists {
|
||||||
pop := e.index.GetPopularity(a.MBID)
|
if a.MBID != "" {
|
||||||
|
allMBIDs = append(allMBIDs, a.MBID)
|
||||||
// Library artists get a massive popularity bonus.
|
|
||||||
if e.index.IsInLibrary(a.MBID) {
|
|
||||||
pop += 10_000_000 //nolint:mnd
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if pop > 0 {
|
for _, rg := range result.ReleaseGroups {
|
||||||
|
if rg.MBID != "" {
|
||||||
|
allMBIDs = append(allMBIDs, rg.MBID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range result.Recordings {
|
||||||
|
if r.MBID != "" {
|
||||||
|
allMBIDs = append(allMBIDs, r.MBID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single batch query for all popularity + in_library data.
|
||||||
|
popMap := e.index.GetPopularityBatch(allMBIDs)
|
||||||
|
if popMap == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build per-entity maps from the batch result.
|
||||||
|
artistPop := make(map[string]int, len(result.Artists))
|
||||||
|
for _, a := range result.Artists {
|
||||||
|
if pop, ok := popMap[a.MBID]; ok {
|
||||||
artistPop[a.MBID] = pop
|
artistPop[a.MBID] = pop
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rerankArtists(result.Artists, artistPop)
|
rerankArtists(result.Artists, artistPop)
|
||||||
|
|
||||||
// Look up popularity for release groups.
|
|
||||||
rgPop := make(map[string]int, len(result.ReleaseGroups))
|
rgPop := make(map[string]int, len(result.ReleaseGroups))
|
||||||
|
|
||||||
for _, rg := range result.ReleaseGroups {
|
for _, rg := range result.ReleaseGroups {
|
||||||
pop := e.index.GetPopularity(rg.MBID)
|
if pop, ok := popMap[rg.MBID]; ok {
|
||||||
|
|
||||||
if e.index.IsInLibrary(rg.MBID) {
|
|
||||||
pop += 10_000_000 //nolint:mnd
|
|
||||||
}
|
|
||||||
|
|
||||||
if pop > 0 {
|
|
||||||
rgPop[rg.MBID] = pop
|
rgPop[rg.MBID] = pop
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rerankReleaseGroups(result.ReleaseGroups, rgPop)
|
rerankReleaseGroups(result.ReleaseGroups, rgPop)
|
||||||
|
|
||||||
// Look up popularity for recordings.
|
|
||||||
recPop := make(map[string]int, len(result.Recordings))
|
recPop := make(map[string]int, len(result.Recordings))
|
||||||
|
|
||||||
for _, r := range result.Recordings {
|
for _, r := range result.Recordings {
|
||||||
if pop := e.index.GetPopularity(r.MBID); pop > 0 {
|
if pop, ok := popMap[r.MBID]; ok {
|
||||||
recPop[r.MBID] = pop
|
recPop[r.MBID] = pop
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -316,6 +316,53 @@ func (si *SearchIndex) GetPopularity(mbid string) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetPopularityBatch returns popularity (listen count) for multiple
|
||||||
|
// MBIDs in a single query. Returns a map of MBID → popularity.
|
||||||
|
func (si *SearchIndex) GetPopularityBatch(mbids []string) map[string]int {
|
||||||
|
if len(mbids) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
placeholders := make([]string, len(mbids))
|
||||||
|
args := make([]any, len(mbids))
|
||||||
|
|
||||||
|
for i, m := range mbids {
|
||||||
|
placeholders[i] = "?"
|
||||||
|
args[i] = m
|
||||||
|
}
|
||||||
|
|
||||||
|
query := "SELECT mbid, popularity, in_library FROM explore_index WHERE mbid IN (" +
|
||||||
|
strings.Join(placeholders, ",") + ")"
|
||||||
|
|
||||||
|
rows, err := si.db.QueryContext(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
|
result := make(map[string]int, len(mbids))
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var mbid string
|
||||||
|
var pop int
|
||||||
|
var inLib int
|
||||||
|
|
||||||
|
if err := rows.Scan(&mbid, &pop, &inLib); err == nil {
|
||||||
|
existing, ok := result[mbid]
|
||||||
|
if !ok || pop > existing {
|
||||||
|
if inLib == 1 {
|
||||||
|
pop += 10_000_000 //nolint:mnd // library bonus
|
||||||
|
}
|
||||||
|
|
||||||
|
result[mbid] = pop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// IsInLibrary returns whether the given MBID is marked as in the
|
// IsInLibrary returns whether the given MBID is marked as in the
|
||||||
// user's local library in the search index.
|
// user's local library in the search index.
|
||||||
func (si *SearchIndex) IsInLibrary(mbid string) bool {
|
func (si *SearchIndex) IsInLibrary(mbid string) bool {
|
||||||
|
|||||||
Reference in New Issue
Block a user