yellowjacket/.gitea/workflows/index-artifact.yml

name: Search index maintenance

# indexbuild decides what to do from the index's own state, so every
# trigger below runs the same command:
#
#   no completed import   -> build   (first run, or resume a partial one)
#   import older than 6mo -> rebuild (re-import from the newest dump)
#   otherwise             -> refresh (fold in new incremental listens)
#
# A refresh is cheap and no-ops when nothing new has been published, so
# running it on every push to main is safe.
on:
  push:
    branches: [main]
  schedule:
    # Weekly update pass. The 6-month rebuild is triggered by the same
    # command when it notices the import has aged out.
    - cron: '0 4 * * 1'
  workflow_dispatch:
    inputs:
      mode:
        description: 'auto | build | refresh | rebuild'
        required: false
        default: 'auto'
      budget:
        description: 'Max build time this run'
        required: false
        default: '3h'
      artists:
        description: 'Top artists in the core artifact'
        required: false
        default: '50000'

# Runs share one persistent working directory, so they must not overlap.
# A push landing mid-build waits rather than corrupting the checkpoint.
concurrency:
  group: search-index
  cancel-in-progress: false

jobs:
  maintain-index:
    runs-on: ubuntu-latest
    container:
      # CGO is not needed: the project uses the pure-Go modernc sqlite
      # driver, and neither command imports the Wails app.
      image: golang:1.25
      # This host path must exist on the runner and be listed verbatim in
      # act_runner's container.valid_volumes. It holds explore-staging/
      # (counts.bin + state.json) and yj.db — the checkpoint that makes
      # resuming possible. Losing it means re-downloading ~205GB.
      volumes:
        - /srv/yellowjacket/index-cache:/cache
    env:
      YJ_HOME: /cache
      CGO_ENABLED: '0'
      PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
      SERVER_URL: ${{ github.server_url }}
      OWNER: ${{ github.repository_owner }}
      REPO: ${{ github.repository }}
      SHA: ${{ github.sha }}
      MODE: ${{ inputs.mode || 'auto' }}
      BUDGET: ${{ inputs.budget || '3h' }}
      ARTISTS: ${{ inputs.artists || '50000' }}
    steps:
      # Cloned by hand rather than with actions/checkout: that is a JS
      # action and needs node inside the job container, which the golang
      # image does not carry. Same approach as arch-package.yml.
      - name: Clone repo at this commit
        run: |
          set -eu
          git clone --quiet \
            "https://x-access-token:${PACKAGE_TOKEN}@${SERVER_URL#https://}/${REPO}.git" \
            /src
          git -C /src checkout --quiet --detach "$SHA"
          git -C /src log --oneline -1

      - name: Verify the cache volume
        run: |
          set -eu
          mkdir -p /cache
          # A RAM-backed cache would defeat the point: the checkpoint has
          # to outlive the job, and the import wants real disk headroom.
          fstype=$(stat -f -c %T /cache || echo unknown)
          echo "cache fstype: $fstype"
          case "$fstype" in
            tmpfs|ramfs)
              echo "::error::/cache is RAM-backed; use a disk-backed host path."
              exit 1 ;;
          esac
          df -h /cache

      - name: Build tools
        working-directory: /src
        # The dump importer is behind the `indexbuild` tag so it is not
        # linked into the app binary; cmd/indexbuild carries the same tag
        # and will not build without it.
        run: |
          go build -tags indexbuild -o /usr/local/bin/ ./cmd/indexbuild
          go build -o /usr/local/bin/ ./cmd/indexexport

      - name: Maintain index
        id: maintain
        run: |
          set +e
          indexbuild -mode "$MODE" -budget "$BUDGET"
          code=$?
          set -e
          case "$code" in
            0) ;;
            3) echo "::notice::Build checkpointed with work remaining — rerun to continue." ;;
            *) exit "$code" ;;
          esac

      # Publishing only on `changed` keeps identical artifacts from
      # accumulating when a refresh finds nothing new.
      - name: Export core artifact
        if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
        run: |
          set -eu
          command -v zstd >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq zstd; }
          indexexport -o /tmp/core-index.db -artists "$ARTISTS"
          zstd -19 -T0 -q -f /tmp/core-index.db -o /tmp/core-index.db.zst
          sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256
          ls -lh /tmp/core-index.db.zst

      - name: Publish to the Gitea package registry
        if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
        run: |
          set -eu
          pkg="${SERVER_URL}/api/packages/${OWNER}/generic/yellowjacket-core-index"

          # Published twice: under a dated version for history, and under
          # the fixed "latest" version the client fetches. Clients cannot
          # discover the newest dated version on their own — the package
          # listing API requires a token, while a plain file GET does not
          # — so "latest" is what makes an anonymous first run possible.
          #
          # A generic package rejects re-uploading a filename that already
          # exists, so "latest" is deleted before being rewritten. It is
          # absent on the very first publish, hence the tolerated 404.
          curl --silent --show-error --user "${OWNER}:${PACKAGE_TOKEN}" \
            --request DELETE "${pkg}/latest" || true

          for version in "$(date -u +%Y%m%d)" latest; do
            for f in core-index.db.zst core-index.db.zst.sha256; do
              echo "Uploading $f -> $version"
              curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \
                --upload-file "/tmp/$f" "${pkg}/${version}/${f}"
            done
          done

      - name: Summary
        if: always()
        run: |
          echo "complete=${{ steps.maintain.outputs.complete }}"
          echo "changed=${{ steps.maintain.outputs.changed }}"
          if [ "${{ steps.maintain.outputs.complete }}" != "true" ]; then
            echo "Build incomplete — rerun to continue from the checkpoint."
            echo "Progress lives in /cache/data/explore-staging."
          elif [ "${{ steps.maintain.outputs.changed }}" != "true" ]; then
            echo "Nothing new to publish."
          fi