perf(library): send the track list as a dictionary-encoded column table

GetTracks answered with one JSON object per track: 20.5 MB at 26k
tracks, ~350 bytes a row of repeated key names, four cover URLs that
are identical across an album, and the album, artist and genre strings
repeated on every track. Encoding it was ~170 MB of transient Go
allocation, and parsing it the WebView's memory peak.

GetTrackTable replaces it: one array per column, every repeated
string stored once and sent as an index, genre lists interned, and only
the fields the Tracks view reads. LastPlayed and the three larger cover
tiers are left to the details dialog, which now fetches whole tracks by
path. Each row still goes through trackFromRow, so this is an encoding
of the one projection, not a second one. 167 bytes a track against 929
in the test library; a Go test holds it under a quarter of the object
encoding, and a Vitest test reads the Go columns and the generated
Track interface and fails if they drift.

An empty library is now an empty table rather than the "no tracks in
library" error the old binding returned.

Closes #281

backend/library/query.go

@@ -2,7 +2,6 @@ package library
 
 import (
 	"database/sql"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -18,8 +17,6 @@ import (
 // searchTrackLimit bounds an FTS search's result set.
 const searchTrackLimit = 500
 
-var errNoTracksInLibrary = errors.New("no tracks in library")
-
 // Track is one audio file with everything a list needs to draw it.
 type Track struct {
 	TrackName        string
@@ -169,30 +166,6 @@ func (l *Library) GetTrackMBIDs(filePath string) TrackMBIDs {
 	}
 }
 
-// GetTracks returns every track in a library, or in all of them when
-// libraryID is 0.
-//
-// The library id is a parameter rather than a second method because the
-// two used to be separate queries, separate bindings and a branch at
-// every call site - and the scoped form costs nothing (measured: 23 ms
-// against 21 ms over 26k rows).
-func (l *Library) GetTracks(libraryID int64) ([]Track, error) {
-	rows, err := l.db.ReadQueries.GetTracks(l.ctx, libraryID)
-	if err != nil {
-		l.logger.Error("could not retrieve audio files", "error", err)
-
-		return nil, fmt.Errorf("could not get tracks: %w", err)
-	}
-
-	l.logger.Info("audio file list", "count", len(rows), "libraryID", libraryID)
-
-	if len(rows) == 0 {
-		return nil, errNoTracksInLibrary
-	}
-
-	return tracksFromRows(rows), nil
-}
-
 // pathLookupChunk bounds the paths bound into one IN (...) query, well
 // under SQLite's bind-variable limit.
 const pathLookupChunk = 500

backend/library/scan_fixtures_test.go

@@ -58,15 +58,15 @@ func TestScan_FixtureLibraryLeavesNothingBehind(t *testing.T) {
 		t.Skip("fixture library is empty; run make testdata")
 	}
 
-	tracks, err := lib.GetTracks(0)
+	table, err := lib.GetTrackTable(0)
 	if err != nil {
-		t.Fatalf("GetTracks: %v", err)
+		t.Fatalf("GetTrackTable: %v", err)
 	}
 
 	// One track per file: the projection cannot multiply rows, because
 	// there is no join table left to multiply them.
-	if int64(len(tracks)) != files {
-		t.Errorf("GetTracks returned %d rows for %d files", len(tracks), files)
+	if int64(len(table.FilePath)) != files {
+		t.Errorf("GetTrackTable returned %d rows for %d files", len(table.FilePath), files)
 	}
 
 	// Nothing shared outlives what refers to it.

backend/library/tracktable.go

@@ -0,0 +1,190 @@
+package library
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// TrackTable is every track in a library as the Tracks view uses it:
+// one array per column, and every repeated string stored once (#281).
+//
+// GetTracks used to answer with one object per track, which at 26 138
+// tracks was 20.5 MB of JSON — ~350 bytes a row of key names, four
+// cover URLs identical across an album, and artist, album and genre
+// strings repeated on every track of the album.  Encoding it cost the
+// backend ~170 MB of transient allocation and parsing it was the
+// WebView's peak.  Here the keys appear once, a repeated string is a
+// small integer, and the columns the Tracks view does not read are not
+// sent at all: LastPlayed and the three larger cover tiers belong to
+// the details dialog, which fetches whole tracks by path.
+//
+// The projection is still trackFromRow's — each row goes through it —
+// so this is an encoding of a Track, never a second description of
+// one.  frontend/src/utils/track-table.ts is the only decoder.
+type TrackTable struct {
+	// Strings holds every distinct string value; a string column holds
+	// indexes into it.  Index 0 is always "".
+	Strings []string `json:"strings"`
+	// GenreSets holds every distinct genre list, as indexes into
+	// Strings; Genre holds an index into it per track.
+	GenreSets [][]uint32 `json:"genreSets"`
+
+	FilePath         []string `json:"filePath"`
+	TrackName        []uint32 `json:"trackName"`
+	ArtistName       []uint32 `json:"artistName"`
+	Album            []uint32 `json:"album"`
+	Composer         []uint32 `json:"composer"`
+	FileType         []uint32 `json:"fileType"`
+	Genre            []uint32 `json:"genre"`
+	ArtistMBID       []uint32 `json:"artistMbid"`
+	ReleaseGroupMBID []uint32 `json:"releaseGroupMbid"`
+	RecordingMBID    []uint32 `json:"recordingMbid"`
+	CoverArtSmall    []uint32 `json:"coverArtSmall"`
+
+	// LengthMs is Track.TrackLength as the number it encodes.
+	LengthMs    []int64 `json:"lengthMs"`
+	TrackNumber []int64 `json:"trackNumber"`
+	DiscNumber  []int64 `json:"discNumber"`
+	Year        []int64 `json:"year"`
+	SampleRate  []int64 `json:"sampleRate"`
+	BitDepth    []int64 `json:"bitDepth"`
+	Channels    []int64 `json:"channels"`
+	Bitrate     []int64 `json:"bitrate"`
+	FileSize    []int64 `json:"fileSize"`
+	PlayCount   []int64 `json:"playCount"`
+}
+
+// trackTableBuilder interns strings and genre lists while rows are
+// appended.
+type trackTableBuilder struct {
+	table   TrackTable
+	strings map[string]uint32
+	genres  map[string]uint32
+}
+
+func newTrackTableBuilder(capacity int) *trackTableBuilder {
+	b := &trackTableBuilder{
+		strings: map[string]uint32{"": 0},
+		genres:  map[string]uint32{},
+	}
+
+	t := &b.table
+	t.Strings = []string{""}
+	t.FilePath = make([]string, 0, capacity)
+
+	for _, col := range b.stringColumns() {
+		*col = make([]uint32, 0, capacity)
+	}
+
+	for _, col := range b.intColumns() {
+		*col = make([]int64, 0, capacity)
+	}
+
+	t.Genre = make([]uint32, 0, capacity)
+
+	return b
+}
+
+// stringColumns are the interned columns, in one place so the builder
+// cannot allocate one and forget to fill it.
+func (b *trackTableBuilder) stringColumns() []*[]uint32 {
+	t := &b.table
+
+	return []*[]uint32{
+		&t.TrackName, &t.ArtistName, &t.Album, &t.Composer, &t.FileType,
+		&t.ArtistMBID, &t.ReleaseGroupMBID, &t.RecordingMBID, &t.CoverArtSmall,
+	}
+}
+
+func (b *trackTableBuilder) intColumns() []*[]int64 {
+	t := &b.table
+
+	return []*[]int64{
+		&t.LengthMs, &t.TrackNumber, &t.DiscNumber, &t.Year, &t.SampleRate,
+		&t.BitDepth, &t.Channels, &t.Bitrate, &t.FileSize, &t.PlayCount,
+	}
+}
+
+func (b *trackTableBuilder) intern(s string) uint32 {
+	if i, ok := b.strings[s]; ok {
+		return i
+	}
+
+	i := uint32(len(b.table.Strings)) //nolint:gosec // bounded by the row count
+
+	b.table.Strings = append(b.table.Strings, s)
+	b.strings[s] = i
+
+	return i
+}
+
+func (b *trackTableBuilder) internGenres(genres []string) uint32 {
+	key := strings.Join(genres, genreDelimiter)
+
+	if i, ok := b.genres[key]; ok {
+		return i
+	}
+
+	set := make([]uint32, len(genres))
+	for j, g := range genres {
+		set[j] = b.intern(g)
+	}
+
+	i := uint32(len(b.table.GenreSets)) //nolint:gosec // bounded by the row count
+
+	b.table.GenreSets = append(b.table.GenreSets, set)
+	b.genres[key] = i
+
+	return i
+}
+
+func (b *trackTableBuilder) add(tr Track) error {
+	lengthMs, err := strconv.ParseInt(tr.TrackLength, 10, 64)
+	if err != nil {
+		return fmt.Errorf("track %q length %q: %w", tr.FilePath, tr.TrackLength, err)
+	}
+
+	t := &b.table
+	t.FilePath = append(t.FilePath, tr.FilePath)
+
+	strs := []string{
+		tr.TrackName, tr.ArtistName, tr.Album, tr.Composer, tr.FileType,
+		tr.ArtistMBID, tr.ReleaseGroupMBID, tr.RecordingMBID, tr.CoverArtSmall,
+	}
+	for i, col := range b.stringColumns() {
+		*col = append(*col, b.intern(strs[i]))
+	}
+
+	ints := []int64{
+		lengthMs, tr.TrackNumber, tr.DiscNumber, tr.Year, tr.SampleRate,
+		tr.BitDepth, tr.Channels, tr.Bitrate, tr.FileSize, tr.PlayCount,
+	}
+	for i, col := range b.intColumns() {
+		*col = append(*col, ints[i])
+	}
+
+	t.Genre = append(t.Genre, b.internGenres(tr.Genre))
+
+	return nil
+}
+
+// GetTrackTable returns every track in a library, or in all of them
+// when libraryID is 0, as a TrackTable.  An empty library is an empty
+// table, not an error.
+func (l *Library) GetTrackTable(libraryID int64) (TrackTable, error) {
+	rows, err := l.db.ReadQueries.GetTracks(l.ctx, libraryID)
+	if err != nil {
+		return TrackTable{}, fmt.Errorf("could not get tracks: %w", err)
+	}
+
+	b := newTrackTableBuilder(len(rows))
+
+	for i := range rows {
+		if err := b.add(trackFromRow(rows[i])); err != nil {
+			return TrackTable{}, err
+		}
+	}
+
+	return b.table, nil
+}

backend/library/tracktable_test.go

@@ -0,0 +1,198 @@
+package library
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"testing"
+
+	"yellowjacket/backend/database"
+)
+
+// seedTableLibrary seeds n albums of perAlbum tracks each, with a cover
+// on every album, so the table has the repetition it exists to remove.
+func seedTableLibrary(t *testing.T, lib *Library, albums, perAlbum int) {
+	t.Helper()
+
+	for a := range albums {
+		for n := range perAlbum {
+			database.InsertTestTrack(t, lib.db, database.TestTrack{
+				FilePath: fmt.Sprintf(
+					"/music/artist-%d/album-%d/%02d - Some Track Title.flac", a%7, a, n+1,
+				),
+				Title:         fmt.Sprintf("Some Track Title %d", n+1),
+				Artist:        fmt.Sprintf("Artist %d", a%7),
+				ArtistMBID:    fmt.Sprintf("0b7a8d2e-0000-4000-8000-%012d", a%7),
+				Album:         fmt.Sprintf("Album Name %d", a),
+				AlbumMBID:     fmt.Sprintf("1c7a8d2e-0000-4000-8000-%012d", a),
+				RecordingMBID: fmt.Sprintf("2d7a8d2e-0000-4000-8000-%012d", a*100+n),
+				Genres:        []string{"Ambient", fmt.Sprintf("Genre %d", a%3)},
+				TrackNumber:   int64(n + 1),
+				DiscNumber:    1,
+				Year:          2000 + int64(a%20),
+				LengthMs:      200_000 + int64(n),
+				PlayCount:     int64(n),
+			})
+		}
+
+		res, err := lib.db.ExecContext(
+			"INSERT INTO cover_art (file_path, mime_type) VALUES (?, 'image/jpeg')",
+			fmt.Sprintf("/data/covers/%064d.jpg", a),
+		)
+		if err != nil {
+			t.Fatalf("seed cover: %v", err)
+		}
+
+		coverID, _ := res.LastInsertId()
+
+		if _, err := lib.db.ExecContext(
+			"UPDATE albums SET cover_art_id = ? WHERE name = ?",
+			coverID, fmt.Sprintf("Album Name %d", a),
+		); err != nil {
+			t.Fatalf("seed album cover: %v", err)
+		}
+	}
+}
+
+// decodeTable is the Go mirror of frontend/src/utils/track-table.ts,
+// for asserting the encoding loses nothing it claims to carry.
+func decodeTable(t *testing.T, tbl TrackTable) []Track {
+	t.Helper()
+
+	str := func(col []uint32, i int) string { return tbl.Strings[col[i]] }
+	tracks := make([]Track, len(tbl.FilePath))
+
+	for i := range tracks {
+		var genres []string
+		for _, g := range tbl.GenreSets[tbl.Genre[i]] {
+			genres = append(genres, tbl.Strings[g])
+		}
+
+		tracks[i] = Track{
+			FilePath:         tbl.FilePath[i],
+			TrackName:        str(tbl.TrackName, i),
+			ArtistName:       str(tbl.ArtistName, i),
+			Album:            str(tbl.Album, i),
+			Composer:         str(tbl.Composer, i),
+			FileType:         str(tbl.FileType, i),
+			ArtistMBID:       str(tbl.ArtistMBID, i),
+			ReleaseGroupMBID: str(tbl.ReleaseGroupMBID, i),
+			RecordingMBID:    str(tbl.RecordingMBID, i),
+			CoverArtSmall:    str(tbl.CoverArtSmall, i),
+			Genre:            genres,
+			TrackLength:      strconv.FormatInt(tbl.LengthMs[i], 10),
+			TrackNumber:      tbl.TrackNumber[i],
+			DiscNumber:       tbl.DiscNumber[i],
+			Year:             tbl.Year[i],
+			SampleRate:       tbl.SampleRate[i],
+			BitDepth:         tbl.BitDepth[i],
+			Channels:         tbl.Channels[i],
+			Bitrate:          tbl.Bitrate[i],
+			FileSize:         tbl.FileSize[i],
+			PlayCount:        tbl.PlayCount[i],
+		}
+	}
+
+	return tracks
+}
+
+// #281: the table is an encoding of trackFromRow's Track, minus the
+// fields the Tracks view does not read.  Decoding it must give back
+// exactly that, row for row.
+func TestTrackTableRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	lib, _ := setupTestLibrary(t)
+	seedTableLibrary(t, lib, 6, 4)
+
+	tbl, err := lib.GetTrackTable(0)
+	if err != nil {
+		t.Fatalf("GetTrackTable: %v", err)
+	}
+
+	rows, err := lib.db.ReadQueries.GetTracks(lib.ctx, 0)
+	if err != nil {
+		t.Fatalf("GetTracks: %v", err)
+	}
+
+	if len(rows) != 24 || len(tbl.FilePath) != len(rows) {
+		t.Fatalf("table has %d rows, query %d, want 24", len(tbl.FilePath), len(rows))
+	}
+
+	got := decodeTable(t, tbl)
+
+	for i, row := range rows {
+		want := trackFromRow(row)
+		// Not carried: the details dialog fetches these by path.
+		want.LastPlayed, want.CoverArtPath, want.CoverArtMedium, want.CoverArtLarge = "", "", "", ""
+
+		if fmt.Sprintf("%+v", got[i]) != fmt.Sprintf("%+v", want) {
+			t.Fatalf("row %d:\n got %+v\nwant %+v", i, got[i], want)
+		}
+	}
+
+	// It was the cover, genre and name repetition that was paid for; a
+	// table that interned nothing would still round-trip.
+	if len(tbl.GenreSets) != 3 {
+		t.Errorf("genre sets = %d, want 3 distinct lists", len(tbl.GenreSets))
+	}
+
+	if a, b := tbl.CoverArtSmall[0], tbl.CoverArtSmall[1]; a != b || tbl.Strings[a] == "" {
+		t.Errorf("two tracks of one album hold cover indexes %d and %d", a, b)
+	}
+}
+
+// The point of the table, pinned: the same rows encode to a fraction of
+// the object-per-track JSON they replace.  A new column has to fit
+// under this or raise it on purpose.
+func TestTrackTableSizeBudget(t *testing.T) {
+	t.Parallel()
+
+	lib, _ := setupTestLibrary(t)
+	seedTableLibrary(t, lib, 40, 10)
+
+	tbl, err := lib.GetTrackTable(0)
+	if err != nil {
+		t.Fatalf("GetTrackTable: %v", err)
+	}
+
+	rows, err := lib.db.ReadQueries.GetTracks(lib.ctx, 0)
+	if err != nil {
+		t.Fatalf("GetTracks: %v", err)
+	}
+
+	asTable, err := json.Marshal(tbl)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	asObjects, err := json.Marshal(tracksFromRows(rows))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	perTrack := len(asTable) / len(rows)
+	ratio := float64(len(asTable)) / float64(len(asObjects))
+
+	t.Logf("table %d B (%d B/track), objects %d B, ratio %.2f",
+		len(asTable), perTrack, len(asObjects), ratio)
+
+	if ratio > 0.25 {
+		t.Errorf("table is %.0f%% of the object encoding, budget 25%%", ratio*100)
+	}
+}
+
+func TestTrackTableEmptyLibraryIsAnAnswer(t *testing.T) {
+	t.Parallel()
+
+	lib, _ := setupTestLibrary(t)
+
+	tbl, err := lib.GetTrackTable(0)
+	if err != nil {
+		t.Fatalf("an empty library is an empty table, not an error: %v", err)
+	}
+
+	if len(tbl.FilePath) != 0 || len(tbl.Strings) != 1 {
+		t.Errorf("empty table = %+v", tbl)
+	}
+}