feat: artist aliases in FTS5 index + BM25 blended scoring
Migration 14: add aliases TEXT column to explore_index, rebuild FTS5 with 3 columns (title, artist_name, aliases), recreate sync triggers. Clears index build timestamps to force alias population on next build. Artist image provider fetches inc=url-rels+aliases (single call, no extra cost). GetAliases() extracts alias names from cached MB rels. indexOneArtist stores aliases as space-separated text after image resolution populates the cache. Search query now uses BM25 blended scoring: ORDER BY bm25(fts, 3.0, 1.0, 0.5) - (ln(popularity+1) * 0.5) Column weights: title=3.0, artist_name=1.0, aliases=0.5 - Title matches score 3x higher than artist name matches - Alias matches are helpful but don't dominate - Popularity is a log-scaled boost, not an override - Exact title match on niche entity beats weak match on mega-popular Enables: 'rhcp' → Red Hot Chili Peppers, 'gnr' → Guns N' Roses, 'sabbath' → Black Sabbath (once index build runs with aliases).
This commit is contained in:
@@ -380,6 +380,16 @@ func runMigrations(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migration 14: add aliases column to explore_index and rebuild
|
||||||
|
// the FTS5 virtual table with 3 searchable columns.
|
||||||
|
if version < 14 { //nolint:mnd
|
||||||
|
if err := migration14ExploreAliases(
|
||||||
|
ctx, db, logger,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1596,6 +1606,106 @@ func migration13MBIDColumns(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// migration14ExploreAliases adds an aliases column to explore_index
|
||||||
|
// and rebuilds the FTS5 virtual table with three searchable columns
|
||||||
|
// (title, artist_name, aliases) for alias-aware search.
|
||||||
|
func migration14ExploreAliases(
|
||||||
|
ctx context.Context,
|
||||||
|
db *sql.DB,
|
||||||
|
logger *slog.Logger,
|
||||||
|
) error {
|
||||||
|
logger.Info("applying migration 14: explore index aliases + FTS5 rebuild")
|
||||||
|
|
||||||
|
// Add aliases column to content table.
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"ALTER TABLE explore_index ADD COLUMN aliases TEXT DEFAULT ''",
|
||||||
|
); err != nil {
|
||||||
|
if !strings.Contains(err.Error(), "duplicate column") {
|
||||||
|
return fmt.Errorf("migration 14: alter explore_index: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop old triggers.
|
||||||
|
for _, name := range []string{
|
||||||
|
"explore_index_ai", "explore_index_ad", "explore_index_au",
|
||||||
|
} {
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"DROP TRIGGER IF EXISTS "+name,
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: drop trigger %s: %w", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop and recreate FTS5 with 3 columns.
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"DROP TABLE IF EXISTS explore_index_fts",
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: drop FTS5: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := db.ExecContext(ctx, `
|
||||||
|
CREATE VIRTUAL TABLE explore_index_fts USING fts5(
|
||||||
|
title, artist_name, aliases,
|
||||||
|
content='explore_index',
|
||||||
|
content_rowid='id'
|
||||||
|
)
|
||||||
|
`); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: create FTS5: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recreate triggers with 3 columns.
|
||||||
|
if _, err := db.ExecContext(ctx, `
|
||||||
|
CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN
|
||||||
|
INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
|
||||||
|
VALUES (new.id, new.title, new.artist_name, new.aliases);
|
||||||
|
END
|
||||||
|
`); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: create insert trigger: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := db.ExecContext(ctx, `
|
||||||
|
CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN
|
||||||
|
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
|
||||||
|
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
|
||||||
|
END
|
||||||
|
`); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: create delete trigger: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := db.ExecContext(ctx, `
|
||||||
|
CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN
|
||||||
|
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
|
||||||
|
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
|
||||||
|
INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
|
||||||
|
VALUES (new.id, new.title, new.artist_name, new.aliases);
|
||||||
|
END
|
||||||
|
`); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: create update trigger: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rebuild FTS5 index from existing content table rows.
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"INSERT INTO explore_index_fts(explore_index_fts) VALUES ('rebuild')",
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("migration 14: rebuild FTS5: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := db.ExecContext(
|
||||||
|
ctx, "PRAGMA user_version = 14",
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("could not set user_version to 14: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear the index build timestamp so the next build populates aliases.
|
||||||
|
_, _ = db.ExecContext(ctx,
|
||||||
|
"DELETE FROM explore_index_meta WHERE key IN ('tier1_built', 'discog_built')",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.Info("migration 14 complete")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// readLibraryDirFromTOML reads the TOML config file and returns
|
// readLibraryDirFromTOML reads the TOML config file and returns
|
||||||
// the Library.DirectoryPath value, or "" if not configured.
|
// the Library.DirectoryPath value, or "" if not configured.
|
||||||
func readLibraryDirFromTOML(logger *slog.Logger) string {
|
func readLibraryDirFromTOML(logger *slog.Logger) string {
|
||||||
|
|||||||
@@ -168,7 +168,7 @@ func (p *ArtistImageProvider) fetchMBRels(artistMBID string) []mbRelation {
|
|||||||
}
|
}
|
||||||
|
|
||||||
url := fmt.Sprintf(
|
url := fmt.Sprintf(
|
||||||
"https://musicbrainz.org/ws/2/artist/%s?fmt=json&inc=url-rels",
|
"https://musicbrainz.org/ws/2/artist/%s?fmt=json&inc=url-rels+aliases",
|
||||||
artistMBID,
|
artistMBID,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -368,6 +368,38 @@ func wikimediaThumbURL(filename string) string {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetAliases returns the artist's aliases as a space-separated
|
||||||
|
// string, extracted from the cached MB rels response. Returns ""
|
||||||
|
// if no aliases are cached.
|
||||||
|
func (p *ArtistImageProvider) GetAliases(artistMBID string) string {
|
||||||
|
cacheKey := "mb:artist-rels:" + artistMBID
|
||||||
|
|
||||||
|
data, ok := p.cache.Get(cacheKey)
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var envelope struct {
|
||||||
|
Aliases []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
} `json:"aliases"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.Unmarshal(data, &envelope); err != nil || len(envelope.Aliases) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
names := make([]string, 0, len(envelope.Aliases))
|
||||||
|
|
||||||
|
for _, a := range envelope.Aliases {
|
||||||
|
if a.Name != "" {
|
||||||
|
names = append(names, a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(names, " ")
|
||||||
|
}
|
||||||
|
|
||||||
func (p *ArtistImageProvider) fetchURL(url string) ([]byte, error) {
|
func (p *ArtistImageProvider) fetchURL(url string) ([]byte, error) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), artistImageTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), artistImageTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ type SearchIndexResult struct {
|
|||||||
ArtistMBID string `json:"artistMbid"`
|
ArtistMBID string `json:"artistMbid"`
|
||||||
Popularity int `json:"popularity"`
|
Popularity int `json:"popularity"`
|
||||||
ExtraJSON string `json:"extraJson,omitempty"`
|
ExtraJSON string `json:"extraJson,omitempty"`
|
||||||
|
Aliases string `json:"aliases,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// lbSitewideArtist is the response shape from the LB sitewide
|
// lbSitewideArtist is the response shape from the LB sitewide
|
||||||
@@ -253,7 +254,7 @@ func (si *SearchIndex) Search(query string, limit int) []SearchIndexResult {
|
|||||||
FROM explore_index i
|
FROM explore_index i
|
||||||
JOIN explore_index_fts f ON f.rowid = i.id
|
JOIN explore_index_fts f ON f.rowid = i.id
|
||||||
WHERE explore_index_fts MATCH ?
|
WHERE explore_index_fts MATCH ?
|
||||||
ORDER BY i.popularity DESC
|
ORDER BY bm25(explore_index_fts, 3.0, 1.0, 0.5) - (ln(i.popularity + 1) * 0.5)
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
`, ftsQuery, limit)
|
`, ftsQuery, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1014,6 +1015,23 @@ func (si *SearchIndex) indexOneArtist(
|
|||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
|
// Extract aliases from the now-cached MB rels (populated by
|
||||||
|
// the image resolution above) and update the artist's index entry.
|
||||||
|
if si.artistImg != nil {
|
||||||
|
aliases := si.artistImg.GetAliases(artist.ArtistMBID)
|
||||||
|
if aliases != "" {
|
||||||
|
si.writeBatch([]SearchIndexResult{{
|
||||||
|
EntityType: "artist",
|
||||||
|
MBID: artist.ArtistMBID,
|
||||||
|
Title: artist.ArtistName,
|
||||||
|
ArtistName: artist.ArtistName,
|
||||||
|
ArtistMBID: artist.ArtistMBID,
|
||||||
|
Popularity: artist.ListenCount,
|
||||||
|
Aliases: aliases,
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Batch write discography results.
|
// Batch write discography results.
|
||||||
all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
|
all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
|
||||||
all = append(all, rgs...)
|
all = append(all, rgs...)
|
||||||
@@ -1216,9 +1234,9 @@ func (si *SearchIndex) writeBatch(entries []SearchIndexResult) {
|
|||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if _, err := tx.Exec(`
|
if _, err := tx.Exec(`
|
||||||
INSERT OR REPLACE INTO explore_index
|
INSERT OR REPLACE INTO explore_index
|
||||||
(entity_type, mbid, title, artist_name, artist_mbid, popularity, extra_json)
|
(entity_type, mbid, title, artist_name, artist_mbid, popularity, extra_json, aliases)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, NULLIF(?, ''))
|
VALUES (?, ?, ?, ?, ?, ?, NULLIF(?, ''), NULLIF(?, ''))
|
||||||
`, e.EntityType, e.MBID, e.Title, e.ArtistName, e.ArtistMBID, e.Popularity, e.ExtraJSON,
|
`, e.EntityType, e.MBID, e.Title, e.ArtistName, e.ArtistMBID, e.Popularity, e.ExtraJSON, e.Aliases,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
si.logger.Warn("search index: insert error",
|
si.logger.Warn("search index: insert error",
|
||||||
"mbid", e.MBID,
|
"mbid", e.MBID,
|
||||||
|
|||||||
Reference in New Issue
Block a user