ci: patches job, candidate_id and replace for the hatchet pipeline

.github/workflows/release-atomic.yml

@@ -1,4 +1,5 @@
 name: Release (Atomic)
+run-name: "Release (Atomic) ${{ inputs.upstream_tag }} [cid:${{ inputs.candidate_id }}]"
 
 # Builds every stable-diffusion.cpp engine Atomic Chat ships, from an unmodified upstream tag plus
 # the patch series in atomic/patches, and publishes them as one release of that tag here:
@@ -14,6 +15,10 @@ name: Release (Atomic)
 # Asset names follow upstream's pattern (sd-master-<sha7>-bin-...), so atomic-chat-conf's
 # mirror-sdcpp.yml matches them by pattern. `targets` rebuilds a subset while fixing one; a release
 # needs `all`. See atomic/README.md.
+#
+# The hatchet pipeline (AtomicBot-ai/atomic-hatchet-pipeline) dispatches this on upstream-sync/<tag>
+# with publish and replace, finds the run by [cid:...] in its title, and tells a patch conflict from
+# a broken build by which job failed: the one named `patches` or another.
 
 on:
   workflow_dispatch:
@@ -28,10 +33,20 @@ on:
         default: all
         type: string
       publish:
-        description: "Create or update the release of that tag in this repository"
+        description: "Create the release of that tag in this repository"
         required: false
         default: false
         type: boolean
+      replace:
+        description: "Replace that release's archives if it already exists; without it publishing fails"
+        required: false
+        default: false
+        type: boolean
+      candidate_id:
+        description: "Id the caller finds this run by in its title"
+        required: false
+        default: manual
+        type: string
 
 concurrency:
   group: release-atomic-${{ inputs.upstream_tag }}-${{ inputs.targets }}
@@ -71,6 +86,25 @@ jobs:
       smoke_linux: ${{ steps.plan.outputs.smoke_linux }}
       smoke_windows: ${{ steps.plan.outputs.smoke_windows }}
     steps:
+      # The pipeline matches [cid:<id>] in the run title literally: no spaces or brackets in it.
+      # An existing release without `replace` would fail only in the release job, hours later.
+      - name: Check inputs
+        env:
+          CANDIDATE_ID: ${{ inputs.candidate_id }}
+          PUBLISH: ${{ inputs.publish }}
+          REPLACE: ${{ inputs.replace }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          if ! [[ "$CANDIDATE_ID" =~ ^[A-Za-z0-9._:-]{1,80}$ ]]; then
+            echo "::error::candidate_id must match ^[A-Za-z0-9._:-]{1,80}\$"
+            exit 1
+          fi
+          if [ "$PUBLISH" = true ] && [ "$REPLACE" != true ] && gh release view "$TAG" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            echo "::error::release $TAG already exists here; run with replace to overwrite its archives"
+            exit 1
+          fi
+
       - name: Resolve the upstream tag
         id: tag
         run: |
@@ -151,9 +185,33 @@ jobs:
                   print(f"{k}={v}")
           EOF
 
+  # The series must apply to the tag before any runner spends hours on it. The hatchet pipeline reads
+  # a failure of the job named `patches` as a patch conflict (its agent fixes the series in this
+  # branch) and any other failure as a broken build, so keep the id and the name, and keep every
+  # build job behind it. prepare only checks the inputs.
+  patches:
+    name: patches
+    needs: prepare
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Clone upstream tag
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.UPSTREAM_REPO }}
+          ref: ${{ needs.prepare.outputs.sha }}
+          submodules: recursive
+      - name: Clone Atomic build files
+        uses: actions/checkout@v4
+        with:
+          path: _atomic
+          sparse-checkout: atomic
+      - name: Check the patch series
+        run: bash _atomic/atomic/ci/apply-patches.sh --check
+
   # ----------------------------------------------------------------------------------- macOS
   macos-arm64:
-    needs: prepare
+    needs: [prepare, patches]
     if: contains(needs.prepare.outputs.selected, ',macos-arm64,')
     runs-on: macos-latest
     timeout-minutes: 120
@@ -218,7 +276,7 @@ jobs:
   # ----------------------------------------------------------------------------------- Linux
   linux:
     name: ${{ matrix.id }}
-    needs: prepare
+    needs: [prepare, patches]
     if: needs.prepare.outputs.linux != '[]'
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 300
@@ -349,7 +407,7 @@ jobs:
   # Upstream's ROCm job (TheRock wheels). The ROCm runtime is not bundled, as upstream: users
   # install it; the archive is made flat and symlink-free like every other Linux one here.
   linux-rocm-x64:
-    needs: prepare
+    needs: [prepare, patches]
     if: contains(needs.prepare.outputs.selected, ',linux-rocm-x64,')
     runs-on: ubuntu-24.04
     timeout-minutes: 300
@@ -453,7 +511,7 @@ jobs:
   # ------------------------------------------------------------------------- Windows x64 (MSVC)
   windows-x64:
     name: ${{ matrix.id }}
-    needs: prepare
+    needs: [prepare, patches]
     if: needs.prepare.outputs.windows_x64 != '[]'
     runs-on: windows-2022
     timeout-minutes: 330
@@ -586,7 +644,7 @@ jobs:
   # Upstream's Windows ROCm job (TheRock wheels). amdhip64 comes with the AMD driver, which no
   # runner has, so unresolved imports are reported, not fatal.
   windows-rocm-x64:
-    needs: prepare
+    needs: [prepare, patches]
     if: contains(needs.prepare.outputs.selected, ',win-rocm-x64,')
     runs-on: windows-2022
     timeout-minutes: 330
@@ -698,7 +756,7 @@ jobs:
   # (llama.cpp's toolchain); -march=armv8.7-a covers Snapdragon X and N1X. OpenMP is off: there is
   # no arm64 libomp on the x64 host. The DLL CRT is explicit: ggml-cuda.dll (MSVC) shares its heap.
   windows-arm64-cpu:
-    needs: prepare
+    needs: [prepare, patches]
     if: contains(needs.prepare.outputs.selected, ',win-cpu-arm64,')
     runs-on: windows-2022
     timeout-minutes: 120
@@ -775,7 +833,7 @@ jobs:
   # a standalone ggml would disagree on the ggml_tensor layout); it is dropped into the clang tree.
   # The clang stable-diffusion.dll lacks SD_USE_CUDA: no head padding for d_head < 64 (speed only).
   windows-arm64-cuda13:
-    needs: [prepare, windows-arm64-cpu]
+    needs: [prepare, patches, windows-arm64-cpu]
     if: contains(needs.prepare.outputs.selected, ',win-cuda13-arm64,')
     runs-on: windows-2022
     timeout-minutes: 240
@@ -919,9 +977,11 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
+  # A subset build skips some Windows jobs, so skipped ones are let through; `patches` is listed so
+  # that its failure, which skips all of them, is not.
   smoke-windows:
     name: smoke ${{ matrix.id }}
-    needs: [prepare, windows-x64, windows-arm64-cpu, windows-arm64-cuda13]
+    needs: [prepare, patches, windows-x64, windows-arm64-cpu, windows-arm64-cuda13]
     if: >-
       always() && needs.prepare.outputs.smoke_windows != '[]' &&
       !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
@@ -1088,13 +1148,26 @@ jobs:
           } > notes.md
           cat notes.md
 
+      # Asset names carry the runner's macOS and the ROCm version, so a rebuild can name an archive
+      # differently; the old one is removed, since mirror-sdcpp.yml refuses two matches of a pattern.
       - name: Publish
         env:
           GH_TOKEN: ${{ github.token }}
+          REPLACE: ${{ inputs.replace }}
         run: |
           set -euo pipefail
           if gh release view "$TAG" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            if [ "$REPLACE" != true ]; then
+              echo "::error::release $TAG already exists here; run with replace to overwrite its archives"
+              exit 1
+            fi
             gh release upload "$TAG" dist/* --clobber -R "$GITHUB_REPOSITORY"
+            for name in $(gh release view "$TAG" -R "$GITHUB_REPOSITORY" --json assets --jq '.assets[].name'); do
+              if [ ! -e "dist/$name" ]; then
+                echo "removing $name, not in this build"
+                gh release delete-asset "$TAG" "$name" -y -R "$GITHUB_REPOSITORY"
+              fi
+            done
             gh release edit "$TAG" --notes-file notes.md --title "stable-diffusion.cpp $TAG (Atomic)" -R "$GITHUB_REPOSITORY"
           else
             gh release create "$TAG" dist/* --target "$SHA" -R "$GITHUB_REPOSITORY" \

atomic/README.md

@@ -16,21 +16,67 @@ They are consumed through `atomic-chat-conf/backends/sdcpp-manifest.json`; the c
 
 | Branch   | What it is                                                                 |
 | -------- | -------------------------------------------------------------------------- |
-| `master` | A fast-forward mirror of upstream `master`. Never commit here.             |
-| `atomic` | The default branch: upstream `master` plus **only added files** (`atomic/`, `.github/workflows/release-atomic.yml`). |
+| `master` | A fast-forward mirror of upstream `master`, moved by the pipeline through GitHub's merge-upstream API. Never commit here. |
+| `atomic` | The default branch: upstream at the last synced tag plus **only added files** (`atomic/`, `.github/workflows/release-atomic.yml`). |
+| `upstream-sync/<tag>` | One per upstream tag, made by the pipeline: `atomic` with the tag merged in. Built, tested, then merged into `atomic`. |
 
 Source code is never changed on a branch. Every build starts from an **upstream tag** and applies the
 patch series in [`atomic/patches/`](patches/). That keeps rebasing trivial and makes the release
 notes say exactly what differs from upstream.
 
+## How the pipeline updates this fork
+
+[atomic-hatchet-pipeline](https://github.com/AtomicBot-ai/atomic-hatchet-pipeline) (engine `sd-cpp`)
+watches leejet's tags. For a new `master-N-sha7` it:
+
+1. fast-forwards `master` to upstream with the merge-upstream API;
+2. creates `upstream-sync/<tag>` from `atomic` and merges the tag's commit into it (merges API);
+3. runs **Release (Atomic)** on that branch with `publish` and `replace`. The build is still the
+   upstream tag plus this branch's `atomic/`, so the branch is where every fix goes;
+4. if the `patches` job failed, its agent fixes `atomic/patches/` and pushes to the branch. Any other
+   failure is retried once, then given to the agent too. The agent gets one attempt; after it the
+   pipeline stops and asks for a human in Telegram;
+5. runs Atomic Chat core's tests on the release, and on green merges the PR `upstream-sync/<tag>` →
+   `atomic` (a merge commit). atomic-chat-conf then signs the archives and rolls them out as
+   `<tag>-a<rev>`, where `<rev>` is the first seven characters of the commit the build ran at.
+
+The agent's changes arrive as bot commits on `upstream-sync/<tag>`, so its PR shows them.
+
+Upstream's `build.yml`, `stale-prs.yml` and `close-inactive-issues.yml` come with every sync and must
+stay disabled in this fork's Actions: on a push to `master`, `build.yml` publishes its own release
+under the same tag (and it runs on the pipeline's PRs too), and the stale bots close PRs.
+
+### Fixing a sync by hand
+
+Work in the sync branch, never in `atomic` or `master`:
+
+```bash
+git fetch origin
+git switch upstream-sync/master-975-95b3090
+git submodule update --init --recursive
+bash atomic/ci/apply-patches.sh --check   # what the patches job saw, changes nothing
+```
+
+The branch holds upstream's tree at the tag, so a patch is reworked right there: apply it (e.g.
+`git apply --3way`), resolve, write the diff of the touched files back into the patch keeping its
+header, and restore the files. Commit only `atomic/` (or `release-atomic.yml`: a dispatch on the
+branch runs the branch's copy of the workflow), push, and re-run the pipeline for the same tag
+(atomic-hatchet-pipeline's `docs/engine-updater.md`). It builds the branch as it is now.
+
 ## Building a release
 
 Actions → **Release (Atomic)**:
 - `upstream_tag`: the tag Atomic Chat's manifest pins, e.g. `master-883-137f740`;
 - `targets`: `all`, or a comma-separated list of backend ids to rebuild while fixing one;
-- `publish`: creates the release of the tag here, and needs `all`.
+- `publish`: creates the release of the tag here, and needs `all`. If that release exists, the run
+  fails at once, unless
+- `replace` is on: the build's archives overwrite the release's, and archives this build no longer
+  names (a newer runner macOS, another ROCm) are removed;
+- `candidate_id`: goes into the run title as `[cid:<id>]`, which is how the pipeline finds its run;
+  `manual` by default.
 
-Then run `mirror-sdcpp.yml` in atomic-chat-conf for the same tag.
+The first job, `patches`, runs `atomic/ci/apply-patches.sh --check` on the tag; nothing is built if
+a patch does not apply. Then run `mirror-sdcpp.yml` in atomic-chat-conf for the same tag.
 
 Every archive except ROCm is smoke-tested on a machine without a GPU: it must start, list its devices,
 and generate an image through sd-server's HTTP API. On Linux this runs in a bare `python:3.12-slim`

atomic/ci/apply-patches.sh

@@ -3,10 +3,63 @@
 #
 # A patch named `ggml--<name>.patch` is applied inside the ggml submodule (git apply cannot
 # reach into a submodule from the superproject); every other patch at the repository root.
+#
+# --check changes nothing. Each patch goes through `git apply --check` against a scratch index that
+# already holds the patches before it, as the real run would apply them; every one that does not
+# apply is reported with git's reason, and the exit status is 1 if any did not. Release (Atomic)'s
+# `patches` job runs this, and the hatchet pipeline reads a failure there as a patch conflict.
 set -euo pipefail
 shopt -s nullglob
 
+check=0
+case "${1:-}" in
+  --check) check=1 ;;
+  "") ;;
+  *) echo "usage: $0 [--check]" >&2; exit 2 ;;
+esac
+
 dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../patches" && pwd)"
+
+if [ "$check" = 1 ]; then
+  scratch="$(mktemp -d)"
+  trap 'rm -rf "$scratch"' EXIT
+  total=0
+  failed=""
+  for patch in "$dir"/*.patch; do
+    name="$(basename "$patch")"
+    case "$name" in
+      ggml--*) target=ggml index="$scratch/ggml.index" ;;
+      *) target=. index="$scratch/root.index" ;;
+    esac
+    [ -e "$index" ] || GIT_INDEX_FILE="$index" git -C "$target" read-tree HEAD
+    total=$((total + 1))
+    if out="$(GIT_INDEX_FILE="$index" git -C "$target" apply --check --cached --verbose --whitespace=nowarn "$patch" 2>&1)"; then
+      echo "::group::$name applies in $target"
+      echo "$out"
+      echo "::endgroup::"
+      # Into the scratch index only, so the next patch is checked on top of this one.
+      GIT_INDEX_FILE="$index" git -C "$target" apply --cached --whitespace=nowarn "$patch"
+      continue
+    fi
+    # Outside a group, so git's reason is not folded away in the job log.
+    echo "$name does not apply in $target:"
+    echo "$out"
+    failed="$failed $name"
+    # A patch that landed upstream no longer applies but reverses cleanly: it is to be deleted.
+    if GIT_INDEX_FILE="$index" git -C "$target" apply --check --cached --reverse --whitespace=nowarn "$patch" 2>/dev/null; then
+      echo "::error title=Patch already upstream::$name is already in the tree (it reverses cleanly): delete it"
+    else
+      echo "::error title=Patch does not apply::$name does not apply in $target"
+    fi
+  done
+  if [ -n "$failed" ]; then
+    echo "$(echo "$failed" | wc -w | tr -d ' ') of $total patch(es) do not apply:$failed"
+    exit 1
+  fi
+  echo "all $total patch(es) apply"
+  exit 0
+fi
+
 applied=0
 for patch in "$dir"/*.patch; do
   name="$(basename "$patch")"

atomic/patches/README.md

@@ -5,6 +5,10 @@ order. Name them `NNNN-short-description.patch`; prefix with `ggml--` for a patc
 (applied with `git -C ggml apply`). Keep each patch small, explain why in its header, and offer it
 upstream: a patch that lands upstream is deleted here on the next tag.
 
+`apply-patches.sh --check`, run from the root of a checkout of the tag, changes nothing: it lists every
+patch that does not apply on top of the ones before it, with git's reason, and flags the ones that
+reverse cleanly as already upstream.
+
 Current series (build system only; no source code differs from the upstream tag):
 
 | Patch | Why |