yellowjacket/backend/explore/indexpatch.go

//go:build indexbuild

package explore

import (
	"context"
	"fmt"
	"net/http"
	"regexp"
	"sort"
	"sync"

	"yellowjacket/backend/jobs"
)

// Helpers used only while building the catalog from the MetaBrainz
// dumps: the similar-artist patch pass and the per-stage error
// reporting that goes with it.  They live behind the `indexbuild` tag
// with the importer they serve, so the app binary carries neither.

const (
	// indexSimilarPerArtist is how many similar artists to store per
	// library artist in similar_artist_map.
	indexSimilarPerArtist = 20

	// similarArtistsBatchSize is the number of seed MBIDs processed in
	// one logging "batch".  The labs multi-seed POST form is broken, so
	// one GET per seed is issued (concurrency bounded by indexerRate);
	// batching here just keeps progress log output bounded.
	similarArtistsBatchSize = 50
)

// setTierError marks a build stage as errored.
func (si *SearchIndex) setTierError(name, errMsg string) {
	si.mu.Lock()

	for i := range si.buildStatus.Tiers {
		if si.buildStatus.Tiers[i].Name == name {
			si.buildStatus.Tiers[i].State = "error"
			si.buildStatus.Tiers[i].Error = errMsg
			si.mu.Unlock()

			si.logIndexJob(jobs.LevelError, name+": "+errMsg)
			si.emitStatus()

			return
		}
	}

	si.mu.Unlock()
}

// fetchSimilarArtistsBatch queries the labs similar-artists endpoint
// for multiple seed MBIDs.  Despite the name, this actually fans
// out one request per seed: the labs API's multi-seed mode is
// broken (results for different seeds get mis-labeled, and some
// seeds return zero), so batching with multiple artist_mbids is
// not viable.  Concurrency is bounded by indexerRate to respect
// the labs rate limit; each call goes through the provided LB
// client's rate limiter and cache.
func (si *SearchIndex) fetchSimilarArtistsBatch(
	ctx context.Context, lb *ListenBrainzClient, seedMBIDs []string,
) map[string][]lbSimilarArtistWire {
	if len(seedMBIDs) == 0 {
		return nil
	}

	var (
		mu      sync.Mutex
		grouped = make(map[string][]lbSimilarArtistWire, len(seedMBIDs))
		wg      sync.WaitGroup
	)

	sem := make(chan struct{}, indexerRate)

	for _, seedMBID := range seedMBIDs {
		if ctx.Err() != nil {
			break
		}

		sem <- struct{}{}

		wg.Add(1)

		go func(seed string) {
			defer func() {
				<-sem
				wg.Done()
			}()

			// Use the LB client's per-seed GET form — goes through
			// the shared rate limiter and cache.  The multi-seed
			// POST form is not viable (see function comment).
			similar, err := lb.SimilarArtists(ctx, seed)
			if err != nil || len(similar) == 0 {
				return
			}

			// Convert to the internal wire type used by the caller
			// and trim to indexSimilarPerArtist.
			if len(similar) > indexSimilarPerArtist {
				similar = similar[:indexSimilarPerArtist]
			}

			results := make([]lbSimilarArtistWire, len(similar))
			for i, s := range similar {
				results[i] = lbSimilarArtistWire{
					ArtistMBID:    s.ArtistMBID,
					Name:          s.Name,
					Score:         int(s.Score),
					ReferenceMBID: seed,
				}
			}

			mu.Lock()
			grouped[seed] = results
			mu.Unlock()
		}(seedMBID)
	}

	wg.Wait()

	return grouped
}

// chunkStrings splits a slice into chunks of at most size n.
func chunkStrings(s []string, n int) [][]string {
	var chunks [][]string

	for i := 0; i < len(s); i += n {
		end := i + n
		if end > len(s) {
			end = len(s)
		}

		chunks = append(chunks, s[i:end])
	}

	return chunks
}

// getLibraryArtistMBIDs returns MBIDs for all library artists that have one.
// Used when Tier 3 was skipped but Tier 4 needs the library MBID list.
func (si *SearchIndex) getLibraryArtistMBIDs() []string {
	rows, err := si.db.QueryContext(
		"SELECT DISTINCT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''",
	)
	if err != nil {
		return nil
	}

	defer func() { _ = rows.Close() }()

	var mbids []string

	for rows.Next() {
		var mbid string
		if err := rows.Scan(&mbid); err == nil {
			mbids = append(mbids, mbid)
		}
	}

	return mbids
}

// discoverDumpFile walks a dump base directory, finds subdirectories
// matching dirRe (newest first, lexicographically — MetaBrainz dump
// directory names embed sortable timestamps), and returns the full URL
// of the first file inside matching fileRe.  Directories that don't
// contain a matching file (e.g. partial uploads) are skipped.
func discoverDumpFile(
	ctx context.Context,
	client *http.Client,
	baseURL string,
	dirRe, fileRe *regexp.Regexp,
) (string, error) {
	hrefs, err := listHrefs(ctx, client, baseURL)
	if err != nil {
		return "", err
	}

	var dirs []string

	for _, h := range hrefs {
		trimmed := trimTrailingSlash(h)
		if dirRe.MatchString(trimmed) {
			dirs = append(dirs, trimmed)
		}
	}

	if len(dirs) == 0 {
		return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL)
	}

	sort.Sort(sort.Reverse(sort.StringSlice(dirs)))

	for _, dir := range dirs {
		dirURL := baseURL + dir + "/"

		files, err := listHrefs(ctx, client, dirURL)
		if err != nil {
			continue
		}

		for _, f := range files {
			if fileRe.MatchString(f) {
				return dirURL + f, nil
			}
		}
	}

	return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL)
}