add dev-only profiling with pprof, runtime/trace, and operation timing

Wire up Go's standard profiling toolkit so it's automatically available
in dev builds and completely absent from production. The profiling
package uses build tags (dev/!dev) to eliminate all pprof, trace, and
timing code from release binaries with zero new dependencies.

- backend/profiling: pprof HTTP server on :6060, /debug/trace endpoint,
  block/mutex profiling, and TimeOp helper for structured operation timing
- scripts/profile.sh: interactive menu-driven script that auto-selects
  free ports (8080-8089) so multiple profiles can be open simultaneously
- Instrumented key operations: app init, database init, player load/restore,
  queue set/restore
- Makefile targets: profile, profile-cpu, profile-heap, profile-trace
- .gitignore: exclude trace-*.out and *.pprof artifacts
This commit is contained in:
2026-02-20 18:27:25 -06:00
parent 402789e763
commit 2a3a79652c
13 changed files with 629 additions and 0 deletions
+4
View File
@@ -5,3 +5,7 @@ test_data
test.db
.aider*
lefthook-local.yml
# Profiling artifacts
trace-*.out
*.pprof
+13
View File
@@ -36,3 +36,16 @@ install: ## Install all development dependencies (Go tools, frontend packages)
setup: install ## Install dependencies and set up git hooks
go tool lefthook install
# Profiling (dev builds only — pprof server on :6060 starts automatically)
profile: ## Open interactive profiling menu (CPU, heap, trace, etc.)
@./scripts/profile.sh
profile-cpu: ## Capture CPU profile and open flame graph in browser
@./scripts/profile.sh cpu
profile-heap: ## Capture heap profile and open in browser
@./scripts/profile.sh heap
profile-trace: ## Capture execution trace and open trace viewer
@./scripts/profile.sh trace
+5
View File
@@ -19,6 +19,7 @@ import (
"yellowjacket/backend/library"
"yellowjacket/backend/player"
"yellowjacket/backend/playlist"
"yellowjacket/backend/profiling"
"yellowjacket/backend/queue"
)
@@ -43,6 +44,8 @@ func NewYellowJacketApp(
logger *slog.Logger,
assetHandler *assets.Handler,
) (*YellowJacketApp, error) {
defer profiling.TimeOp(logger, "app.NewYellowJacketApp")()
// initialize anything that does not need access to the wails runtime here
yjApp := &YellowJacketApp{
logger: logger,
@@ -118,6 +121,8 @@ var startupErr error
// OnStartup initializes components that require the Wails runtime context.
func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
defer profiling.TimeOp(yj.logger, "app.OnStartup")()
// initialize anything that needs to use the wails runtime AFTER its been initialized
// you CANNOT use the wails runtime during this function
yj.appContext = ctx
+3
View File
@@ -13,6 +13,7 @@ import (
_ "modernc.org/sqlite" // Register sqlite driver.
"yellowjacket/backend/database/sql/sqlcgen"
"yellowjacket/backend/profiling"
"yellowjacket/backend/system"
)
@@ -31,6 +32,8 @@ type DB struct {
// NewDB opens the database and applies schema migrations.
func NewDB(logger *slog.Logger) (*DB, error) {
defer profiling.TimeOp(logger, "database.NewDB")()
dbCtx := context.Background()
userDataDir, err := system.GetUserDataDirPath()
+7
View File
@@ -22,6 +22,7 @@ import (
"yellowjacket/backend/events"
"yellowjacket/backend/library"
"yellowjacket/backend/metadata"
"yellowjacket/backend/profiling"
)
// Player handles audio playback and state management.
@@ -64,6 +65,8 @@ var speakerSampleRate = beep.SampleRate(44100)
// NewPlayer creates a player and initializes the audio speaker.
func NewPlayer(ctx context.Context, logger *slog.Logger, db *database.DB) (*Player, error) {
defer profiling.TimeOp(logger, "player.NewPlayer")()
player := &Player{
ctx: ctx,
logger: logger,
@@ -313,6 +316,8 @@ func (p *Player) startPaused() {
// LoadFile opens and decodes an audio file for playback.
func (p *Player) LoadFile(filePath string) error {
defer profiling.TimeOp(p.logger, "player.LoadFile")()
// opening file
f, err := os.Open(filePath)
if err != nil {
@@ -724,6 +729,8 @@ func (p *Player) saveState() {
// RestoreState loads the persisted player state from the database.
func (p *Player) RestoreState() {
defer profiling.TimeOp(p.logger, "player.RestoreState")()
if p.db == nil {
p.logger.Warn("No database available, cannot restore player state")
+56
View File
@@ -0,0 +1,56 @@
// Package profiling provides dev-only performance profiling via pprof and runtime/trace.
//
// In dev builds (build tag "dev"), Start launches an HTTP server on localhost:6060
// exposing the standard pprof endpoints and a /debug/trace endpoint for capturing
// execution traces. It also enables block and mutex profiling at reasonable sampling
// rates.
//
// In production builds, all exported functions are no-ops and the pprof/trace
// imports are excluded from the binary entirely.
//
// # Quick start
//
// Run the app in dev mode (pprof starts automatically):
//
// make dev
//
// Then, in a separate terminal, use the interactive profiling helper:
//
// ./scripts/profile.sh
//
// The script provides a menu-driven interface that opens results in your
// browser as flame graphs. No pprof knowledge required. You can also
// invoke it directly:
//
// ./scripts/profile.sh cpu # CPU profile
// ./scripts/profile.sh heap # Heap (memory) profile
// ./scripts/profile.sh allocs # Allocation profile
// ./scripts/profile.sh goroutine # Goroutine dump
// ./scripts/profile.sh block # Block (sync) profile
// ./scripts/profile.sh mutex # Mutex contention profile
// ./scripts/profile.sh trace # Execution trace
// ./scripts/profile.sh health # Quick runtime health check
//
// # Manual usage
//
// If you prefer the CLI directly:
//
// go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30 # CPU
// go tool pprof http://localhost:6060/debug/pprof/heap # Memory
// go tool pprof http://localhost:6060/debug/pprof/goroutine # Goroutines
// curl -o trace.out http://localhost:6060/debug/trace?seconds=5 # Trace
// go tool trace trace.out
//
// # Programmatic usage
//
// stop := profiling.Start(logger)
// defer stop()
//
// # Operation timing
//
// Use TimeOp to log the duration of any operation in dev builds:
//
// defer profiling.TimeOp(logger, "player.LoadFile")()
//
// In production builds TimeOp is a no-op with zero overhead.
package profiling
+159
View File
@@ -0,0 +1,159 @@
//go:build dev
package profiling
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"net/http/pprof"
"runtime"
"runtime/trace"
"strconv"
"time"
)
const (
// pprofAddr is the address the pprof HTTP server listens on.
pprofAddr = "localhost:6060"
// defaultTraceSecs is the default trace capture duration.
defaultTraceSecs = 5
// blockProfileRate controls the fraction of goroutine blocking
// events reported. 1 = every event (most detailed, slight overhead).
blockProfileRate = 1
// mutexProfileFraction controls the fraction of mutex contention
// events reported. 5 = 1/5 of events.
mutexProfileFraction = 5
// serverShutdownTimeout is the maximum time to wait for the
// pprof server to drain connections on shutdown.
serverShutdownTimeout = 5 * time.Second
)
// Start launches the pprof HTTP server and enables block/mutex profiling.
// It returns a stop function that gracefully shuts down the server.
func Start(logger *slog.Logger) func() {
plog := logger.WithGroup("profiling")
// Enable block and mutex profiling so /debug/pprof/block and
// /debug/pprof/mutex return useful data.
runtime.SetBlockProfileRate(blockProfileRate)
runtime.SetMutexProfileFraction(mutexProfileFraction)
mux := http.NewServeMux()
// Register the standard pprof handlers.
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
// Custom endpoint: capture a runtime/trace for a configurable
// duration and stream it back. Usage:
// curl -o trace.out http://localhost:6060/debug/trace?seconds=5
// go tool trace trace.out
mux.HandleFunc("/debug/trace", traceHandler(plog))
srv := &http.Server{
Addr: pprofAddr,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
// Use a listener so we can log the actual bound address.
ln, err := net.Listen("tcp", pprofAddr)
if err != nil {
plog.Error(
"Failed to start pprof server",
"addr", pprofAddr, "err", err,
)
return func() {}
}
plog.Info(
fmt.Sprintf(
"pprof server listening on http://%s/debug/pprof/",
ln.Addr().String(),
),
)
go func() {
if serveErr := srv.Serve(ln); serveErr != nil &&
!errors.Is(serveErr, http.ErrServerClosed) {
plog.Error("pprof server error", "err", serveErr)
}
}()
return func() {
plog.Info("Shutting down pprof server")
ctx, cancel := context.WithTimeout(
context.Background(), serverShutdownTimeout,
)
defer cancel()
if shutErr := srv.Shutdown(ctx); shutErr != nil {
plog.Error(
"pprof server shutdown error",
"err", shutErr,
)
}
// Disable block/mutex profiling.
runtime.SetBlockProfileRate(0)
runtime.SetMutexProfileFraction(0)
}
}
// traceHandler returns an HTTP handler that captures a runtime/trace
// for the requested number of seconds (default 5).
func traceHandler(logger *slog.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
secs := defaultTraceSecs
if s := r.URL.Query().Get("seconds"); s != "" {
if v, err := strconv.Atoi(s); err == nil && v > 0 {
secs = v
}
}
logger.Info(
"Starting trace capture",
"seconds", secs,
)
w.Header().Set(
"Content-Type", "application/octet-stream",
)
w.Header().Set(
"Content-Disposition",
"attachment; filename=trace.out",
)
if err := trace.Start(w); err != nil {
http.Error(
w,
fmt.Sprintf("trace already in progress: %v", err),
http.StatusConflict,
)
return
}
time.Sleep(time.Duration(secs) * time.Second)
trace.Stop()
logger.Info(
"Trace capture complete",
"seconds", secs,
)
}
}
+11
View File
@@ -0,0 +1,11 @@
//go:build !dev
package profiling
import "log/slog"
// Start is a no-op in production builds. The pprof and runtime/trace
// imports are excluded entirely, adding zero overhead to the binary.
func Start(_ *slog.Logger) func() {
return func() {}
}
+29
View File
@@ -0,0 +1,29 @@
//go:build dev
package profiling
import (
"log/slog"
"time"
)
// TimeOp starts a timer and returns a function that, when called, logs the
// elapsed duration. Intended for use with defer:
//
// defer profiling.TimeOp(logger, "database.Init")()
//
// The extra () is required — defer evaluates the outer call immediately
// (capturing the start time) and defers the returned closure.
func TimeOp(logger *slog.Logger, operation string) func() {
start := time.Now()
logger.Debug("operation started", "op", operation)
return func() {
logger.Info(
"operation completed",
"op", operation,
"duration", time.Since(start),
)
}
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !dev
package profiling
import "log/slog"
func noop() {}
// TimeOp is a no-op in production builds. The compiler will inline
// and eliminate this entirely.
func TimeOp(_ *slog.Logger, _ string) func() {
return noop
}
+5
View File
@@ -19,6 +19,7 @@ import (
"yellowjacket/backend/database"
"yellowjacket/backend/database/sql/sqlcgen"
"yellowjacket/backend/events"
"yellowjacket/backend/profiling"
)
// RepeatMode represents the queue repeat behavior.
@@ -611,6 +612,8 @@ func (q *Queue) handlePlayTracksNext(data ...any) {
// resolved in the background. A generation counter ensures stale
// background work is discarded if SetQueue is called again.
func (q *Queue) SetQueue(filePaths []string, startIndex int) {
defer profiling.TimeOp(q.logger, "queue.SetQueue")()
gen := q.setQueueGen.Add(1)
if startIndex < 0 || startIndex >= len(filePaths) {
@@ -1592,6 +1595,8 @@ func (q *Queue) SaveState() {
// RestoreState loads the queue state from the database.
func (q *Queue) RestoreState() {
defer profiling.TimeOp(q.logger, "queue.RestoreState")()
q.mu.Lock()
defer q.mu.Unlock()
+10
View File
@@ -14,6 +14,7 @@ import (
"yellowjacket/backend"
"yellowjacket/backend/assets"
"yellowjacket/backend/logging"
"yellowjacket/backend/profiling"
"yellowjacket/internal/dev"
)
@@ -44,16 +45,22 @@ func main() {
slog.SetDefault(sLogger)
sLogger.Info("starting yellowjacket", "version", version, "commit", commit)
// Start profiling server (pprof + trace). In production builds this
// is a no-op — the compiler eliminates all profiling code.
stopProfiler := profiling.Start(sLogger)
// create asset handler
assetHandler, err := assets.NewAssetHandler(sLogger, frontendDistAssets)
if err != nil {
sLogger.Error("could not create asset handler", "err", err.Error())
stopProfiler()
os.Exit(1)
}
yjApp, err := backend.NewYellowJacketApp(sLogger, assetHandler)
if err != nil {
sLogger.Error("problem initializing yellowjacket", "err", err.Error())
stopProfiler()
os.Exit(1)
}
@@ -83,6 +90,9 @@ func main() {
WebviewGpuPolicy: linux.WebviewGpuPolicyAlways,
},
})
stopProfiler()
if err != nil {
sLogger.Error("application error", "err", err.Error())
os.Exit(1)
+314
View File
@@ -0,0 +1,314 @@
#!/usr/bin/env bash
#
# profile.sh — Interactive profiling helper for yellowjacket.
#
# Prerequisites:
# - The app must be running via `make dev` (pprof server on :6060).
# - Go toolchain must be installed (for `go tool pprof` / `go tool trace`).
# - `curl` must be available (for trace capture).
#
# Usage:
# ./scripts/profile.sh # Interactive menu
# ./scripts/profile.sh cpu # Skip menu, run CPU profile directly
# ./scripts/profile.sh heap # Skip menu, run heap profile directly
# ./scripts/profile.sh trace # Skip menu, capture execution trace
#
set -euo pipefail
PPROF_BASE="http://localhost:6060"
PPROF_URL="${PPROF_BASE}/debug/pprof"
TRACE_URL="${PPROF_BASE}/debug/trace"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
DIM='\033[2m'
RESET='\033[0m'
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
print_header() {
echo ""
echo -e "${BOLD}Yellowjacket Profiler${RESET}"
echo -e "${DIM}────────────────────────────────────────${RESET}"
echo ""
}
check_server() {
if ! curl -s --max-time 2 "${PPROF_URL}/" > /dev/null 2>&1; then
echo -e "${RED}Error: pprof server not reachable at ${PPROF_BASE}${RESET}"
echo ""
echo " Make sure the app is running with: make dev"
echo " The pprof server starts automatically in dev builds."
echo ""
exit 1
fi
}
# prompt_duration asks the user for a duration in seconds.
# $1 = prompt label, $2 = default value.
prompt_duration() {
local label="$1"
local default="$2"
read -rp " ${label} [${default}s]: " input
echo "${input:-$default}"
}
# WEB_PORT_MIN and WEB_PORT_MAX define the range of ports the pprof web
# UI will try when opening a browser. If a port is busy it moves to the
# next one automatically.
WEB_PORT_MIN=8080
WEB_PORT_MAX=8089
# find_free_port echoes the first available port in the range, or returns 1.
find_free_port() {
for port in $(seq "${WEB_PORT_MIN}" "${WEB_PORT_MAX}"); do
if ! ss -tlnp 2>/dev/null | grep -q ":${port} " &&
! lsof -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1; then
echo "${port}"
return 0
fi
done
return 1
}
# pprof_web opens a pprof profile in the browser. It finds a free port
# automatically so multiple profiles can be open at once.
# $1 = the pprof endpoint URL (e.g. http://…/profile?seconds=30).
pprof_web() {
local url="$1"
local port
port=$(find_free_port) || {
echo -e "${RED}No free port found in range ${WEB_PORT_MIN}-${WEB_PORT_MAX}.${RESET}"
echo -e "${DIM} Close an existing pprof browser tab and try again.${RESET}"
return 1
}
echo -e "${DIM} Opening browser UI on port ${port}...${RESET}"
go tool pprof -http=":${port}" "${url}"
}
# ---------------------------------------------------------------------------
# Profile commands
# ---------------------------------------------------------------------------
do_cpu() {
local secs
secs=$(prompt_duration "Capture duration" "30")
echo ""
echo -e "${CYAN}Capturing CPU profile for ${secs}s...${RESET}"
echo -e "${DIM} While this runs, use the app normally to generate load.${RESET}"
echo ""
pprof_web "${PPROF_URL}/profile?seconds=${secs}"
}
do_heap() {
echo ""
echo -e "${CYAN}Capturing heap profile...${RESET}"
echo ""
pprof_web "${PPROF_URL}/heap"
}
do_allocs() {
echo ""
echo -e "${CYAN}Capturing allocation profile...${RESET}"
echo -e "${DIM} Shows where memory allocations happen (even if already freed).${RESET}"
echo ""
pprof_web "${PPROF_URL}/allocs"
}
do_goroutine() {
echo ""
echo -e "${CYAN}Capturing goroutine dump...${RESET}"
echo -e "${DIM} Shows all goroutines and what they are currently doing.${RESET}"
echo ""
pprof_web "${PPROF_URL}/goroutine"
}
do_block() {
echo ""
echo -e "${CYAN}Capturing block profile...${RESET}"
echo -e "${DIM} Shows where goroutines block waiting on synchronization"
echo -e " primitives (mutexes, channels, select).${RESET}"
echo ""
pprof_web "${PPROF_URL}/block"
}
do_mutex() {
echo ""
echo -e "${CYAN}Capturing mutex contention profile...${RESET}"
echo -e "${DIM} Shows where goroutines contend on mutexes.${RESET}"
echo ""
pprof_web "${PPROF_URL}/mutex"
}
do_trace() {
local secs
secs=$(prompt_duration "Capture duration" "5")
local outfile="trace-$(date +%Y%m%d-%H%M%S).out"
echo ""
echo -e "${CYAN}Capturing execution trace for ${secs}s...${RESET}"
echo -e "${DIM} This records goroutine scheduling, GC pauses, syscalls,"
echo -e " and network activity at microsecond resolution.${RESET}"
echo ""
curl -s -o "${outfile}" "${TRACE_URL}?seconds=${secs}"
echo -e "${GREEN}Trace saved to ${outfile}${RESET}"
echo -e "Opening trace viewer in browser..."
echo ""
go tool trace "${outfile}"
}
do_health() {
echo ""
echo -e "${CYAN}Runtime health check${RESET}"
echo -e "${DIM}────────────────────────────────────────${RESET}"
# Goroutine count
local goroutines
goroutines=$(curl -s "${PPROF_URL}/goroutine?debug=0" | head -c 500 | wc -l)
echo -e " Goroutines: $(curl -s "${PPROF_URL}/goroutine?debug=1" | head -1 | grep -oP '\d+')"
# Heap stats via /debug/pprof/heap?debug=1
local heap_info
heap_info=$(curl -s "${PPROF_URL}/heap?debug=1" | head -20)
local heap_inuse
heap_inuse=$(echo "${heap_info}" | grep -oP '# Heap = \K\d+' || echo "unknown")
if [ "${heap_inuse}" != "unknown" ]; then
local heap_mb
heap_mb=$(echo "scale=1; ${heap_inuse} / 1048576" | bc 2>/dev/null || echo "${heap_inuse} bytes")
echo -e " Heap in use: ${heap_mb} MB"
fi
local heap_sys
heap_sys=$(echo "${heap_info}" | grep -oP 'HeapSys = \K\d+' || echo "")
if [ -n "${heap_sys}" ]; then
local sys_mb
sys_mb=$(echo "scale=1; ${heap_sys} / 1048576" | bc 2>/dev/null || echo "${heap_sys} bytes")
echo -e " Heap reserved: ${sys_mb} MB"
fi
local num_gc
num_gc=$(echo "${heap_info}" | grep -oP 'NumGC = \K\d+' || echo "unknown")
echo -e " GC cycles: ${num_gc}"
echo ""
echo -e "${DIM} For detailed runtime stats, visit:"
echo -e " ${PPROF_URL}/heap?debug=1${RESET}"
echo ""
}
# ---------------------------------------------------------------------------
# Menu
# ---------------------------------------------------------------------------
show_menu() {
echo -e " ${BOLD}What would you like to profile?${RESET}"
echo ""
echo -e " ${GREEN}1)${RESET} CPU profile ${DIM}Find slow functions (flame graph in browser)${RESET}"
echo -e " ${GREEN}2)${RESET} Heap profile ${DIM}See current memory usage by location${RESET}"
echo -e " ${GREEN}3)${RESET} Allocation profile ${DIM}Find where allocations happen (even freed ones)${RESET}"
echo -e " ${GREEN}4)${RESET} Goroutine dump ${DIM}See all goroutines and what they're doing${RESET}"
echo -e " ${GREEN}5)${RESET} Block profile ${DIM}Find where goroutines block on sync primitives${RESET}"
echo -e " ${GREEN}6)${RESET} Mutex profile ${DIM}Find mutex contention hotspots${RESET}"
echo -e " ${GREEN}7)${RESET} Execution trace ${DIM}Detailed timeline: scheduling, GC, syscalls${RESET}"
echo -e " ${GREEN}8)${RESET} Quick health check ${DIM}Goroutine count, heap size, GC stats${RESET}"
echo ""
echo -e " ${GREEN}q)${RESET} Quit"
echo ""
read -rp " Choose [1-8, q]: " choice
echo ""
case "${choice}" in
1|cpu) do_cpu || true ;;
2|heap) do_heap || true ;;
3|allocs) do_allocs || true ;;
4|goroutine) do_goroutine || true ;;
5|block) do_block || true ;;
6|mutex) do_mutex || true ;;
7|trace) do_trace || true ;;
8|health) do_health || true ;;
q|Q|quit) echo "Bye."; exit 0 ;;
*) echo -e "${RED}Invalid choice: ${choice}${RESET}"; echo "" ;;
esac
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
main() {
# Direct invocation: ./scripts/profile.sh cpu
if [ $# -gt 0 ]; then
case "$1" in
help|-h|--help)
echo "Usage: $0 [cpu|heap|allocs|goroutine|block|mutex|trace|health]"
echo ""
echo "Run without arguments for an interactive menu."
echo ""
echo "Commands:"
echo " cpu CPU profile — find slow functions (opens flame graph)"
echo " heap Heap profile — see current memory usage by location"
echo " allocs Allocation profile — find where allocations happen"
echo " goroutine Goroutine dump — see all goroutines and their state"
echo " block Block profile — find sync primitive bottlenecks"
echo " mutex Mutex profile — find mutex contention hotspots"
echo " trace Execution trace — detailed scheduling/GC/syscall timeline"
echo " health Quick health check — goroutine count, heap, GC stats"
exit 0
;;
esac
check_server
case "$1" in
cpu) do_cpu ;;
heap) do_heap ;;
allocs) do_allocs ;;
goroutine) do_goroutine ;;
block) do_block ;;
mutex) do_mutex ;;
trace) do_trace ;;
health) do_health ;;
*)
echo -e "${RED}Unknown command: $1${RESET}"
echo "Usage: $0 [cpu|heap|allocs|goroutine|block|mutex|trace|health]"
exit 1
;;
esac
exit 0
fi
# Interactive mode
print_header
check_server
echo -e " ${GREEN}Connected to pprof server at ${PPROF_BASE}${RESET}"
echo ""
while true; do
show_menu
done
}
main "$@"