feat(jobs): surface background jobs with progress, logs and controls
Add a central job registry that library scans and search index builds report into, so background work is visible instead of buried in the settings page. - backend/jobs: registry with per-job ring-buffer logs, capability-driven controls, and one coalesced JobsChanged snapshot at 4Hz - pause survives restart via a job_state table; a paused scan is adopted back on launch and skipped by the soft scan - top-bar indicator, popover, details drawer and a Jobs page replacing the config page's scan UI; per-library start/stop retained - scan timing breakdown moves into the job log, Full rescan to the Jobs page; delete the orphaned library-manager component Also add cmd/indexbuild and cmd/indexexport so the explore index can be built once centrally rather than by every install, which today streams ~205GB from the ListenBrainz spark dump on first run. indexbuild picks build/refresh/rebuild from index state; the Gitea workflow runs it on push, weekly, or manually and publishes only when content changed. fresh-install no longer defaults YJ_HOME under /tmp: it is tmpfs on most distros, and the import needs ~6GB of real disk. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,131 @@
|
||||
name: Search index maintenance
|
||||
|
||||
# indexbuild decides what to do from the index's own state, so every
|
||||
# trigger below runs the same command:
|
||||
#
|
||||
# no completed import -> build (first run, or resume a partial one)
|
||||
# import older than 3mo -> rebuild (re-import from the newest dump)
|
||||
# otherwise -> refresh (fold in new incremental listens)
|
||||
#
|
||||
# A refresh is cheap and no-ops when nothing new has been published, so
|
||||
# running it on every push to main is safe.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Weekly update pass. The 3-month rebuild is triggered by the same
|
||||
# command when it notices the import has aged out.
|
||||
- cron: '0 4 * * 1'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: 'auto | build | refresh | rebuild'
|
||||
required: false
|
||||
default: 'auto'
|
||||
budget:
|
||||
description: 'Max build time this run'
|
||||
required: false
|
||||
default: '3h'
|
||||
artists:
|
||||
description: 'Top artists in the core artifact'
|
||||
required: false
|
||||
default: '50000'
|
||||
|
||||
# Runs share one persistent working directory, so they must not overlap.
|
||||
# A push landing mid-build waits rather than corrupting the checkpoint.
|
||||
concurrency:
|
||||
group: search-index
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
maintain-index:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
# CGO is not needed: the project uses the pure-Go modernc sqlite
|
||||
# driver, and neither command imports the Wails app.
|
||||
image: golang:1.25
|
||||
# This host path must exist on the runner and be listed verbatim in
|
||||
# act_runner's container.valid_volumes. It holds explore-staging/
|
||||
# (counts.bin + state.json) and yj.db — the checkpoint that makes
|
||||
# resuming possible. Losing it means re-downloading ~205GB.
|
||||
volumes:
|
||||
- /srv/yellowjacket/index-cache:/cache
|
||||
env:
|
||||
YJ_HOME: /cache
|
||||
CGO_ENABLED: '0'
|
||||
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
MODE: ${{ inputs.mode || 'auto' }}
|
||||
BUDGET: ${{ inputs.budget || '3h' }}
|
||||
ARTISTS: ${{ inputs.artists || '50000' }}
|
||||
steps:
|
||||
- name: Check out
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Verify the cache volume
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p /cache
|
||||
# A RAM-backed cache would defeat the point: the checkpoint has
|
||||
# to outlive the job, and the import wants real disk headroom.
|
||||
fstype=$(stat -f -c %T /cache || echo unknown)
|
||||
echo "cache fstype: $fstype"
|
||||
case "$fstype" in
|
||||
tmpfs|ramfs)
|
||||
echo "::error::/cache is RAM-backed; use a disk-backed host path."
|
||||
exit 1 ;;
|
||||
esac
|
||||
df -h /cache
|
||||
|
||||
- name: Build tools
|
||||
run: go build -o /usr/local/bin/ ./cmd/indexbuild ./cmd/indexexport
|
||||
|
||||
- name: Maintain index
|
||||
id: maintain
|
||||
run: |
|
||||
set +e
|
||||
indexbuild -mode "$MODE" -budget "$BUDGET"
|
||||
code=$?
|
||||
set -e
|
||||
case "$code" in
|
||||
0) ;;
|
||||
3) echo "::notice::Build checkpointed with work remaining — rerun to continue." ;;
|
||||
*) exit "$code" ;;
|
||||
esac
|
||||
|
||||
# Publishing only on `changed` keeps identical artifacts from
|
||||
# accumulating when a refresh finds nothing new.
|
||||
- name: Export core artifact
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
command -v zstd >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq zstd; }
|
||||
indexexport -o /tmp/core-index.db -artists "$ARTISTS"
|
||||
zstd -19 -T0 -q -f /tmp/core-index.db -o /tmp/core-index.db.zst
|
||||
sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256
|
||||
ls -lh /tmp/core-index.db.zst
|
||||
|
||||
- name: Publish to the Gitea package registry
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
version="$(date -u +%Y%m%d)"
|
||||
base="${SERVER_URL}/api/packages/${OWNER}/generic/yellowjacket-core-index/${version}"
|
||||
for f in core-index.db.zst core-index.db.zst.sha256; do
|
||||
echo "Uploading $f -> $version"
|
||||
curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \
|
||||
--upload-file "/tmp/$f" "${base}/${f}"
|
||||
done
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "complete=${{ steps.maintain.outputs.complete }}"
|
||||
echo "changed=${{ steps.maintain.outputs.changed }}"
|
||||
if [ "${{ steps.maintain.outputs.complete }}" != "true" ]; then
|
||||
echo "Build incomplete — rerun to continue from the checkpoint."
|
||||
echo "Progress lives in /cache/data/explore-staging."
|
||||
elif [ "${{ steps.maintain.outputs.changed }}" != "true" ]; then
|
||||
echo "Nothing new to publish."
|
||||
fi
|
||||
Reference in New Issue
Block a user