Sync from deal-sage/deal-sage@83bb6f4c19

README.md

@@ -22,10 +22,36 @@ to every app on its next push, instead of being patched repo by repo.
 |---|---|---|
 | Tooling | `scripts/ensure-tools.sh` | curl + jq; AWS CLI only for the S3 upload path |
 | Manifest | `scripts/manifest.sh` | validates `.dealsage/app.json`, exports `APP_ID`, `CHECKS_JSON` |
-| Install | (action input) | `npm install --no-audit --no-fund` by default |
+| Build record | `scripts/start-build.sh` | posts the run to `record-app-build` (commit, branch, author); every later exit closes it with an outcome, so failed runs show in the app hub |
+| Set up Node.js | `actions/setup-node@v4` | `node-version` input |
+| Install | `scripts/install.sh` | runs the `install-command` input (`npm install --no-audit --no-fund` by default); a failure closes the run as `build_failed` with the output's tail |
 | Checks | `scripts/run-checks.sh` | runs each declared check, posts every result to `record-app-check`; a failed **required** check stops here |
 | Build | `scripts/build.sh` | `npm run build` with `VITE_SUPABASE_URL/ANON_KEY` from org variables; recorded as the `build` check |
 | Publish | `scripts/publish.sh` | allocate → upload → promote request; DealSage applies the app's publish policy |
+| Close (no publish) | `scripts/finish-build.sh` | with `skip-publish`, records the run as `checked` |
+| Report the steps | `scripts/finish-steps.sh` | `if: always()`: settles the step list and closes a run no script closed (see below) |
+
+### Steps on the build page
+
+Gitea shows this whole action as a single step of the job, so the scripts
+report their own steps to `record-app-build` (action `steps`) and the app hub's
+build page lists them while the run is going: status, duration, a one-line
+summary and, for a step that failed, the tail of its output. The step list is
+`PIPELINE_STEPS` in `scripts/lib.sh`, in `action.yml` order (keep the two in
+step); each script marks its step running when it starts and passed, failed or
+skipped when it ends (`step_start` / `step_done`; `fail_build` and `die` fail
+the running step). Set up Node.js is a stock action, so the step before it
+starts it and the step after it ends it.
+
+State lives in a file under `$RUNNER_TEMP` that every step of the job shares,
+so a step that dies still leaves what it wrote. The last step,
+`finish-steps.sh`, runs whatever happened: a step still running failed, steps
+that never started were skipped, and if nothing closed the build record (Set
+up Node.js failed, a script crashed) it closes the run as `failed`, naming the
+step, instead of leaving it "running" until the hub calls it lost. Reporting is
+best effort: posts time out after 20 s, and after the first refused post the
+run stops sending steps; the build itself never waits on or fails because of
+them.
 
 The action never decides whether a build goes live. `record-app-version`'s
 `promote` action does, from the app's `publish_policy` and default branch, and

VERSION

@@ -1 +1 @@
-1.0.0
+1.2.0

pipeline/action.yml

@@ -39,6 +39,11 @@ outputs:
 # workflow): DEALSAGE_FUNCTIONS_URL, SUPABASE_URL, SUPABASE_ANON_KEY, and either
 # CUSTOM_APPS_WORKER_URL (upload through the Worker) or R2_ACCOUNT_ID,
 # R2_BUCKET, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY (S3 upload).
+#
+# Gitea shows this whole action as one step of the job, so the scripts report
+# the steps below to DealSage themselves (lib.sh, "Pipeline steps") and the
+# build page in the app hub lists them. PIPELINE_STEPS in lib.sh names them in
+# this order; keep the two in step.
 runs:
   using: composite
   steps:
@@ -50,6 +55,12 @@ runs:
       shell: bash
       run: bash "${{ github.action_path }}/scripts/manifest.sh"
 
+    - name: Record the build
+      shell: bash
+      env:
+        PUBLISH_TOKEN: ${{ inputs.publish-token }}
+      run: bash "${{ github.action_path }}/scripts/start-build.sh"
+
     - name: Set up Node.js
       uses: actions/setup-node@v4
       with:
@@ -57,7 +68,10 @@ runs:
 
     - name: Install dependencies
       shell: bash
-      run: ${{ inputs.install-command }}
+      env:
+        PUBLISH_TOKEN: ${{ inputs.publish-token }}
+        INSTALL_COMMAND: ${{ inputs.install-command }}
+      run: bash "${{ github.action_path }}/scripts/install.sh"
 
     - name: Checks
       shell: bash
@@ -78,3 +92,19 @@ runs:
       env:
         PUBLISH_TOKEN: ${{ inputs.publish-token }}
       run: bash "${{ github.action_path }}/scripts/publish.sh"
+
+    - name: Close the build record (no publish)
+      if: inputs.skip-publish == 'true'
+      shell: bash
+      env:
+        PUBLISH_TOKEN: ${{ inputs.publish-token }}
+      run: bash "${{ github.action_path }}/scripts/finish-build.sh" checked
+
+    # Runs whatever happened above: settles the step list with how the run
+    # ended and closes a build record no script got to close.
+    - name: Report the steps
+      if: always()
+      shell: bash
+      env:
+        PUBLISH_TOKEN: ${{ inputs.publish-token }}
+      run: bash "${{ github.action_path }}/scripts/finish-steps.sh"

pipeline/scripts/build.sh

@@ -3,6 +3,7 @@
 # record the build as a check so DealSage shows it beside the declared ones.
 set -euo pipefail
 . "$(dirname "$0")/lib.sh"
+step_start build
 require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY SUPABASE_URL
 git_context
 
@@ -30,7 +31,9 @@ payload="$(jq -n \
     logTail:($log | split("\n") | .[-80:] | join("\n")), reportedBy:"ci",
     startedAt:$started, finishedAt:$finished}')"
 ds_post record-app-check "$payload" >/dev/null || warn "could not record the build check in DealSage"
-rm -f "$logf"
 summary "- build: **$status** (${dur}ms)"
-[ "$rc" = 0 ] || die "Build failed ($summary); nothing was published."
-log "Build ok: $(find dist -type f | wc -l) files"
+[ "$rc" = 0 ] || fail_build build_failed "Build failed ($summary); nothing was published." "$logf"
+rm -f "$logf"
+files="$(find dist -type f | wc -l | tr -d ' ')"
+log "Build ok: $files files"
+step_done build passed "$files $([ "$files" = 1 ] && echo file || echo files) in dist/"

pipeline/scripts/ensure-tools.sh

@@ -4,6 +4,9 @@
 # CUSTOM_APPS_WORKER_URL). Installs are apt-based, matching the runner image.
 set -euo pipefail
 . "$(dirname "$0")/lib.sh"
+# jq may not be here yet, so this step records itself at the end, back-dated.
+started="$(iso_now)"; t0="$(now_ms)"
+installed=()
 
 need_apt=()
 command -v curl >/dev/null || need_apt+=(curl)
@@ -11,6 +14,7 @@ command -v jq >/dev/null || need_apt+=(jq)
 if [ "${#need_apt[@]}" -gt 0 ]; then
   log "Installing ${need_apt[*]}"
   apt-get update -qq && apt-get install -y -qq --no-install-recommends "${need_apt[@]}" >/dev/null
+  installed+=("${need_apt[@]}")
 fi
 
 if [ -z "${CUSTOM_APPS_WORKER_URL:-}" ] && ! command -v aws >/dev/null; then
@@ -19,5 +23,13 @@ if [ -z "${CUSTOM_APPS_WORKER_URL:-}" ] && ! command -v aws >/dev/null; then
   curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
   unzip -q -o /tmp/awscliv2.zip -d /tmp
   /tmp/aws/install >/dev/null
+  installed+=("the AWS CLI")
 fi
 echo "tools ready: curl $(curl --version | head -n1 | cut -d' ' -f2), jq $(jq --version)$( command -v aws >/dev/null && echo ", aws $(aws --version 2>&1 | cut -d' ' -f1)")"
+
+step_start tooling "$started" "$t0"
+if [ "${#installed[@]}" -gt 0 ]; then
+  step_done tooling passed "installed $(printf '%s, ' "${installed[@]}" | sed 's/, $//')"
+else
+  step_done tooling passed
+fi

pipeline/scripts/finish-build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Close the build record when the run ends without publishing (skip-publish):
+# checks and build passed, nothing was allocated. Usage: finish-build.sh [outcome] [reason]
+set -euo pipefail
+. "$(dirname "$0")/lib.sh"
+require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY
+step_done publish skipped "Checks and build only (skip-publish); nothing was published"
+report_build_finish "${1:-checked}" "${2:-}"

pipeline/scripts/finish-steps.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# The pipeline's last step, run however the steps before it ended (if:
+# always()). Settles the step list with how the run ended and sends it one
+# last time:
+#   * a step still running never finished, so it failed;
+#   * a step that never started was skipped;
+#   * when nothing closed the build record (a step died without fail_build:
+#     Set up Node.js, a crash), close it as failed, naming the step, so the hub
+#     does not show the run as running until it goes stale.
+# Never fails: the run's result is already decided.
+set -uo pipefail
+. "$(dirname "$0")/lib.sh"
+
+settle() {
+  local f before stopped reason
+  f="$(steps_file)"
+  [ -s "$f" ] && command -v jq >/dev/null || return 0
+  before="$(cat "$f")"
+
+  # Why the run stopped, from the step that stopped it: one still running
+  # (never finished), else the first that failed, else the first that never
+  # started (it died before it could say so).
+  stopped="$(jq -r '
+    (map(select(.status == "running"))[0] | select(.) | .name + " did not finish.")
+    // (map(select(.status == "failed"))[0] | select(.) | .name + (if .summary then ": " + .summary else " failed." end))
+    // (map(select(.status == "pending"))[0] | select(.) | "The run stopped before " + .name + ".")
+    // empty' "$f")"
+
+  _steps_edit 'map(if .status == "running" then .status = "failed"
+                   elif .status == "pending" then .status = "skipped" else . end)' || return 0
+  # A step that was running when the run stopped gets its finish time too.
+  _steps_edit --arg at "$(iso_now)" --argjson ms "$(now_ms)" \
+    'map(if .status == "failed" and .startedMs and (.finishedAt | not)
+         then .finishedAt = $at | .durationMs = ($ms - .startedMs) else . end)' || return 0
+
+  if [ -n "${BUILD_ID:-}" ] && [ ! -e "$f.finished" ]; then
+    reason="${stopped:-The run stopped before it finished.}"
+    report_build_finish failed "$reason"
+  fi
+  [ "$(cat "$f")" = "$before" ] || steps_post
+}
+
+( settle ) || true
+exit 0

pipeline/scripts/install.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Install the app's dependencies with the action's install-command, run in the
+# repo root under bash -eo pipefail like any run step. The output is kept, so a
+# failure shows its tail on the build page, and a failure closes the build
+# record (build_failed) instead of leaving the run "running" until it goes
+# stale. Set up Node.js, the step before, ends here.
+set -euo pipefail
+. "$(dirname "$0")/lib.sh"
+step_next node install
+cmd="${INSTALL_COMMAND:-npm install --no-audit --no-fund}"
+
+logf="$(mktemp)"
+printf '$ %s\n' "$cmd"
+set +e
+bash -eo pipefail -c "$cmd" 2>&1 | tee "$logf"
+rc=${PIPESTATUS[0]}
+set -e
+[ "$rc" = 0 ] || fail_build build_failed "Installing dependencies failed (exit code $rc); nothing was published." "$logf"
+rm -f "$logf"
+step_done install passed

pipeline/scripts/lib.sh

@@ -11,21 +11,24 @@
 
 log() { printf '::group::%s\n' "$*" >/dev/null; printf '\n== %s\n' "$*"; }
 warn() { printf '::warning::%s\n' "$*"; }
-die() { printf '::error::%s\n' "$*"; exit 1; }
+# die MESSAGE — stop the job; the running step (see "Pipeline steps") fails with MESSAGE.
+die() { printf '::error::%s\n' "$*"; step_fail_running "$*"; exit 1; }
 
-# now_ms — wall clock in milliseconds (GNU date; falls back to seconds*1000).
-now_ms() { date +%s%3N 2>/dev/null || echo $(( $(date +%s) * 1000 )); }
+# now_ms — wall clock in milliseconds (GNU date; falls back to seconds*1000
+# where %N is not supported and comes back literally).
+now_ms() {
+  local ms; ms="$(date +%s%3N 2>/dev/null || true)"
+  case "$ms" in ''|*[!0-9]*) echo $(( $(date +%s) * 1000 )) ;; *) echo "$ms" ;; esac
+}
 
 # iso_now — UTC timestamp for API payloads.
 iso_now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
 
 # require_env NAME... — fail loudly when a required variable is missing.
 require_env() {
-  local missing=0 v
-  for v in "$@"; do
-    if [ -z "${!v:-}" ]; then printf '::error::%s is not set\n' "$v"; missing=1; fi
-  done
-  [ "$missing" = 0 ] || exit 1
+  local missing=() v
+  for v in "$@"; do [ -n "${!v:-}" ] || missing+=("$v"); done
+  [ "${#missing[@]}" = 0 ] || die "${missing[*]} $([ "${#missing[@]}" = 1 ] && echo is || echo are) not set"
 }
 
 # json_escape STRING — JSON string literal (with quotes) via jq.
@@ -36,11 +39,12 @@ tail_file() { tail -n "${2:-80}" "$1" 2>/dev/null || true; }
 
 # ds_post PATH JSON — POST JSON to a DealSage function with the publish token.
 # Prints the response body; returns non-zero on HTTP failure (body still printed
-# to stderr so the job log shows why).
+# to stderr so the job log shows why). DS_POST_MAX_TIME caps the request in
+# seconds (default 0: no cap).
 ds_post() {
   local path="$1" body="$2" out code
   out="$(mktemp)"
-  code="$(curl -sS -o "$out" -w '%{http_code}' -X POST \
+  code="$(curl -sS --max-time "${DS_POST_MAX_TIME:-0}" -o "$out" -w '%{http_code}' -X POST \
     "${DEALSAGE_FUNCTIONS_URL%/}/${path}" \
     -H "Authorization: Bearer ${PUBLISH_TOKEN}" \
     -H "apikey: ${SUPABASE_ANON_KEY}" \
@@ -86,3 +90,146 @@ persist_env() {
 
 # summary MARKDOWN — append to the run's step summary when the runner supports it.
 summary() { [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$*" >> "$GITHUB_STEP_SUMMARY" || true; }
+
+# ---------------------------------------------------------------------------
+# Build records (record-app-build): one row per pipeline run in DealSage, so a
+# run that never reaches a version (a failed required check, a broken build)
+# still shows up in the app hub. Best effort throughout: a DealSage outage must
+# not fail the build.
+# ---------------------------------------------------------------------------
+
+# ci_run_id — Gitea's run id plus attempt, so a retried "start" is idempotent.
+ci_run_id() {
+  if [ -n "${GITHUB_RUN_ID:-}" ]; then printf '%s-%s' "$GITHUB_RUN_ID" "${GITHUB_RUN_ATTEMPT:-1}"; fi
+}
+
+# report_build_start — POST start; persists BUILD_ID for the later steps.
+report_build_start() {
+  local body resp id
+  body="$(jq -n --arg appId "$APP_ID" --arg sha "$COMMIT_SHA" --arg branch "$BRANCH" \
+    --arg msg "$COMMIT_MESSAGE" --arg author "$COMMIT_AUTHOR" --arg run "$(ci_run_id)" --arg started "$(iso_now)" \
+    '{appId:$appId, action:"start", commitSha:$sha, branch:$branch, commitMessage:$msg, author:$author,
+      ciRunId:(if $run == "" then null else $run end), reportedBy:"ci", startedAt:$started}')"
+  if resp="$(ds_post record-app-build "$body" 2>/dev/null)"; then
+    id="$(jq -r '.buildId // empty' <<<"$resp")"
+    if [ -n "$id" ]; then persist_env BUILD_ID "$id"; log "Build recorded in DealSage ($id)"; return 0; fi
+  fi
+  warn "could not record the build start in DealSage; the run continues without a build record"
+}
+
+# report_build_finish OUTCOME [REASON] [VERSION] [PROMOTED] — close the record.
+# Leaves a marker beside the step list, so finish-steps.sh knows it was closed.
+report_build_finish() {
+  [ -n "${BUILD_ID:-}" ] || return 0
+  printf '%s\n' "$1" > "$(steps_file).finished" 2>/dev/null || true
+  local body
+  body="$(jq -n --arg appId "$APP_ID" --arg id "$BUILD_ID" --arg outcome "$1" --arg reason "${2:-}" \
+    --arg v "${3:-}" --arg promoted "${4:-false}" --arg finished "$(iso_now)" \
+    '{appId:$appId, action:"finish", buildId:$id, outcome:$outcome,
+      failureReason:(if $reason == "" then null else $reason end),
+      version:(if $v == "" then null else ($v|tonumber) end),
+      promoted:($promoted == "true"), finishedAt:$finished}')"
+  ds_post record-app-build "$body" >/dev/null 2>&1 || warn "could not record the build outcome in DealSage"
+}
+
+# fail_build OUTCOME MESSAGE [LOGFILE] — fail the running step (with the tail
+# of LOGFILE), record the outcome, then stop the job.
+fail_build() { step_fail_running "$2" "${3:-}"; report_build_finish "$1" "$2"; die "$2"; }
+
+# ---------------------------------------------------------------------------
+# Pipeline steps (record-app-build, action "steps"). Gitea sees this whole
+# action as one step of the job, so the pipeline reports its own steps and the
+# build page in the app hub lists them while they run: status, duration, a
+# short summary and, for a step that failed, the tail of its output.
+#
+# The list lives in a file the composite steps share, so whatever a dying step
+# wrote survives it, and finish-steps.sh (the always() step at the end)
+# settles it with how the run ended. Best effort, like the build record:
+# reporting never fails a step, and once DealSage refuses a post the run stops
+# trying. The list is only sent once the build is recorded (BUILD_ID); the
+# steps before that are sent with the first post.
+# ---------------------------------------------------------------------------
+
+# id|name of every step, in the order action.yml runs them (keep the two in step).
+PIPELINE_STEPS="tooling|Tooling
+manifest|Read the app manifest
+record|Record the build
+node|Set up Node.js
+install|Install dependencies
+checks|Checks
+build|Build
+publish|Publish"
+
+# steps_file — this run's step list, one per run attempt and job.
+steps_file() {
+  printf '%s/dealsage-steps-%s-%s-%s.json' "${RUNNER_TEMP:-/tmp}" \
+    "${GITHUB_RUN_ID:-local}" "${GITHUB_RUN_ATTEMPT:-1}" "${GITHUB_JOB:-job}"
+}
+
+# _steps_edit JQ_ARGS... — rewrite the list with a jq program; the first edit
+# of a run creates it with every step pending.
+_steps_edit() {
+  local f; f="$(steps_file)"
+  if [ ! -s "$f" ]; then
+    jq -cn --arg steps "$PIPELINE_STEPS" \
+      '$steps | split("\n") | map(select(length > 0) | split("|") | {id: .[0], name: .[1], status: "pending"})' \
+      > "$f" || return 1
+  fi
+  jq -c "$@" "$f" > "$f.tmp" && mv "$f.tmp" "$f"
+}
+
+# _step_begin ID [STARTED_ISO STARTED_MS] — ID is running (since the given time).
+_step_begin() {
+  _steps_edit --arg id "$1" --arg at "${2:-$(iso_now)}" --argjson ms "${3:-$(now_ms)}" \
+    'map(if .id == $id then {id, name, status: "running", startedAt: $at, startedMs: $ms} else . end)'
+}
+
+# _step_end ID STATUS [SUMMARY] [LOGFILE] — ID ended; a step that started gets
+# its finish time and duration.
+_step_end() {
+  local log=""
+  [ -z "${4:-}" ] || log="$(tail_file "$4")"
+  _steps_edit --arg id "$1" --arg status "$2" --arg summary "${3:-}" --arg log "$log" \
+    --arg at "$(iso_now)" --argjson ms "$(now_ms)" '
+    map(if .id != $id then . else
+      .status = $status
+      | (if .startedMs then .finishedAt = $at | .durationMs = ($ms - .startedMs) else . end)
+      | (if $summary != "" then .summary = $summary else . end)
+      | (if $log != "" then .logTail = $log else . end)
+    end)'
+}
+
+# steps_post — send the list to DealSage (a no-op until the build is recorded).
+steps_post() {
+  [ -n "${BUILD_ID:-}" ] || return 0
+  local f body; f="$(steps_file)"
+  [ -s "$f" ] && [ ! -e "$f.off" ] || return 0
+  body="$(jq -c --arg appId "${APP_ID:-}" --arg id "$BUILD_ID" \
+    '{appId: $appId, action: "steps", buildId: $id, steps: map(del(.startedMs))}' "$f")" || return 0
+  if ! DS_POST_MAX_TIME=20 ds_post record-app-build "$body" >/dev/null 2>&1; then
+    : > "$f.off"
+    warn "could not record the pipeline steps in DealSage; the build page will not list this run's steps"
+  fi
+}
+
+# step_start ID [STARTED_ISO STARTED_MS] — ID is running. The optional start
+# time is for the Tooling step, which can only record itself once jq is there.
+step_start() { ( set +e; _step_begin "$@" && steps_post ) || true; }
+
+# step_done ID STATUS [SUMMARY] [LOGFILE] — ID ended: passed, failed or
+# skipped, with a short summary and, when given, the tail of LOGFILE.
+step_done() { ( set +e; _step_end "$@" && steps_post ) || true; }
+
+# step_next DONE NEXT — DONE passed and NEXT is running, in one post. This is
+# how Set up Node.js (a stock action, nothing of ours runs inside it) is
+# tracked: the step before starts it, the step after ends it.
+step_next() { ( set +e; _step_end "$1" passed && _step_begin "$2" && steps_post ) || true; }
+
+# step_fail_running SUMMARY [LOGFILE] — the step that is running failed.
+step_fail_running() {
+  ( set +e
+    local id
+    id="$(jq -r 'map(select(.status == "running"))[0].id // empty' "$(steps_file)" 2>/dev/null)"
+    [ -z "$id" ] || { _step_end "$id" failed "$1" "${2:-}" && steps_post; }
+  ) || true
+}

pipeline/scripts/manifest.sh

@@ -7,6 +7,7 @@
 # fails fast, before install. The schema of record is ../manifest.schema.json.
 set -euo pipefail
 . "$(dirname "$0")/lib.sh"
+step_start manifest
 
 MANIFEST_PATH="${MANIFEST_PATH:-.dealsage/app.json}"
 [ -f "$MANIFEST_PATH" ] || die "Manifest not found at $MANIFEST_PATH"
@@ -45,5 +46,7 @@ manifest="$(jq -c . "$MANIFEST_PATH")"
 persist_env APP_ID "$app_id"
 persist_env MANIFEST_JSON "$manifest"
 persist_env CHECKS_JSON "$checks"
-log "Manifest: app $app_id, $(jq 'length' <<<"$checks") check(s)"
+count="$(jq 'length' <<<"$checks")"
+log "Manifest: app $app_id, $count check(s)"
 jq -r '.[] | "  - \(.id) (\(if .required then "required" else "advisory" end)): \(.run // ("platform:" + .uses))"' <<<"$checks"
+step_done manifest passed "$count $([ "$count" = 1 ] && echo check || echo checks) declared"

pipeline/scripts/publish.sh

@@ -11,6 +11,7 @@
 # The live pointer only ever moves after the bytes are in place.
 set -euo pipefail
 . "$(dirname "$0")/lib.sh"
+step_start publish
 require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY
 git_context
 [ -f dist/index.html ] || die "dist/index.html not found; run the build first"
@@ -21,14 +22,15 @@ checks_json="${CHECKS_RESULT_JSON:-[]}"
 alloc_body="$(jq -n \
   --arg appId "$APP_ID" --arg sha "$COMMIT_SHA" --arg branch "$BRANCH" \
   --arg msg "$COMMIT_MESSAGE" --arg author "$COMMIT_AUTHOR" \
-  --argjson manifest "$manifest_json" --argjson checks "$checks_json" \
+  --argjson manifest "$manifest_json" --argjson checks "$checks_json" --arg build "${BUILD_ID:-}" \
   '{appId:$appId, action:"allocate", builtBy:"ci", commitSha:$sha, branch:$branch,
-    commitMessage:$msg, author:$author, manifest:$manifest, checks:$checks}')"
-alloc="$(ds_post record-app-version "$alloc_body")" || die "allocate failed"
+    commitMessage:$msg, author:$author, manifest:$manifest, checks:$checks,
+    buildId:(if $build == "" then null else $build end)}')"
+alloc="$(ds_post record-app-version "$alloc_body")" || fail_build failed "allocate failed"
 VERSION="$(jq -r '.version // empty' <<<"$alloc")"
 PREFIX="$(jq -r '.r2Prefix // empty' <<<"$alloc")"
 UPLOAD_TOKEN="$(jq -r '.uploadToken // empty' <<<"$alloc")"
-[ -n "$VERSION" ] && [ -n "$PREFIX" ] || die "allocate returned no version/prefix: $alloc"
+[ -n "$VERSION" ] && [ -n "$PREFIX" ] || fail_build failed "allocate returned no version/prefix: $alloc"
 log "Allocated v$VERSION for $BRANCH@${COMMIT_SHA:0:7} -> $PREFIX"
 
 mark_failed() {
@@ -38,7 +40,7 @@ mark_failed() {
 
 # --- 2. upload --------------------------------------------------------------
 if [ -n "${CUSTOM_APPS_WORKER_URL:-}" ]; then
-  [ -n "$UPLOAD_TOKEN" ] || { mark_failed "no upload token"; die "allocate returned no uploadToken but CUSTOM_APPS_WORKER_URL is set"; }
+  [ -n "$UPLOAD_TOKEN" ] || { mark_failed "no upload token"; fail_build failed "allocate returned no uploadToken but CUSTOM_APPS_WORKER_URL is set"; }
   log "Uploading through the Worker"
   export WORKER="${CUSTOM_APPS_WORKER_URL%/}" UPLOAD_TOKEN APP_ID VERSION
   upload_one() { # relative path under dist/
@@ -50,7 +52,7 @@ if [ -n "${CUSTOM_APPS_WORKER_URL:-}" ]; then
   }
   export -f upload_one
   if ! (cd dist && find . -type f | sed 's|^\./||') | xargs -P 6 -n 1 bash -c 'upload_one "$0"'; then
-    mark_failed "upload failed"; die "Upload failed; v$VERSION marked failed and was not promoted."
+    mark_failed "upload failed"; fail_build upload_failed "Upload failed; v$VERSION marked failed and was not promoted."
   fi
 else
   require_env R2_ACCOUNT_ID R2_BUCKET R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY
@@ -60,23 +62,27 @@ else
      AWS_RESPONSE_CHECKSUM_VALIDATION=when_required \
      aws s3 sync dist "s3://$R2_BUCKET/${PREFIX%/}" \
        --endpoint-url "https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com" --delete; then
-    mark_failed "upload failed"; die "Upload failed; v$VERSION marked failed and was not promoted."
+    mark_failed "upload failed"; fail_build upload_failed "Upload failed; v$VERSION marked failed and was not promoted."
   fi
 fi
 log "Uploaded $(find dist -type f | wc -l) files"
 
 # --- 3. promote (DealSage applies the app's publish policy) ------------------
 promote="$(ds_post record-app-version "$(jq -n --arg appId "$APP_ID" --argjson v "$VERSION" \
-  '{appId:$appId, action:"promote", version:$v}')")" || die "promote request failed"
+  '{appId:$appId, action:"promote", version:$v}')")" || fail_build failed "promote request failed"
 promoted="$(jq -r '.promoted // false' <<<"$promote")"
 channel="$(jq -r '.channel // empty' <<<"$promote")"
 current="$(jq -r '.currentVersion // empty' <<<"$promote")"
 if [ "$promoted" = true ]; then
-  log "v$VERSION is live"
+  outcome_note="v$VERSION is live"
+  log "$outcome_note"
 else
   live_note="nothing is live yet"; [ -n "$current" ] && live_note="live stays at v$current"
+  outcome_note="v$VERSION kept on ${channel:-branch/$BRANCH}; $live_note"
   log "v$VERSION built and kept on channel '${channel:-branch/$BRANCH}' ($live_note); promote it from DealSage when ready"
 fi
+step_done publish passed "$outcome_note"
+report_build_finish built "" "$VERSION" "$promoted"
 {
   echo "version=$VERSION"; echo "promoted=$promoted"; echo "channel=${channel}"
 } >> "${GITHUB_OUTPUT:-/dev/null}"

pipeline/scripts/run-checks.sh

@@ -8,6 +8,7 @@
 # a failure: the build should not be blocked by the dashboard being down.
 set -euo pipefail
 . "$(dirname "$0")/lib.sh"
+step_start checks
 require_env APP_ID CHECKS_JSON DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY
 git_context
 CHECKS_DIR="$(cd "$(dirname "$0")/../checks" && pwd)"
@@ -28,10 +29,12 @@ post_check() { # id name status required duration_ms summary logfile started fin
 
 results='[]'
 required_failed=()
+advisory_failed=()
 count="$(jq 'length' <<<"$CHECKS_JSON")"
 if [ "$count" = 0 ]; then
   log "No checks declared in .dealsage/app.json"
   persist_env CHECKS_RESULT_JSON '[]'
+  step_done checks skipped "No checks declared in .dealsage/app.json"
   exit 0
 fi
 
@@ -59,7 +62,9 @@ while IFS= read -r check; do
   post_check "$id" "$name" "$status" "$required" "$dur" "$summary" "$logf" "$started" "$(iso_now)"
   results="$(jq -c --arg id "$id" --arg status "$status" --argjson d "$dur" --argjson req "$required" \
     '. + [{id:$id, status:$status, durationMs:$d, required:$req}]' <<<"$results")"
-  if [ "$status" != passed ] && [ "$required" = true ]; then required_failed+=("$name"); fi
+  if [ "$status" != passed ]; then
+    if [ "$required" = true ]; then required_failed+=("$name"); else advisory_failed+=("$name"); fi
+  fi
   rm -f "$logf"
 done < <(jq -c '.[]' <<<"$CHECKS_JSON")
 
@@ -68,6 +73,12 @@ summary "### Checks"
 summary "$(jq -r '.[] | "- \(.id): **\(.status)** (\(.durationMs)ms\(if .required then "" else ", advisory" end))"' <<<"$results")"
 
 if [ "${#required_failed[@]}" -gt 0 ]; then
-  die "Required check(s) failed: ${required_failed[*]}. The build was not published."
+  fail_build checks_failed "Required check(s) failed: ${required_failed[*]}. The build was not published."
 fi
 log "All required checks passed"
+passed="$(jq '[.[] | select(.status == "passed")] | length' <<<"$results")"
+note="$passed of $count passed"
+if [ "${#advisory_failed[@]}" -gt 0 ]; then
+  note="$note ($(printf '%s, ' "${advisory_failed[@]}" | sed 's/, $//') failed, not required)"
+fi
+step_done checks passed "$note"

pipeline/scripts/start-build.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Record the start of this pipeline run in DealSage (custom_app_builds), so the
+# app hub shows the run while it is in progress and keeps it if it fails before
+# a version exists. Later steps close it through report_build_finish. The first
+# post of the step list goes with it, and Set up Node.js (next) starts here.
+set -euo pipefail
+. "$(dirname "$0")/lib.sh"
+step_start record
+require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY
+git_context
+report_build_start
+step_next record node

test/run-local.sh

@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 # Exercise the pipeline scripts end to end against a stub DealSage, without
-# Gitea. Builds a throwaway app repo, then runs three scenarios:
+# Gitea. Builds a throwaway app repo, then runs these scenarios:
 #   1. auto policy: checks pass (one advisory fails), build, upload, promote → live
 #   2. a required check fails: nothing is allocated
 #   3. manual policy: built and kept on the branch channel, not promoted
+#   4. installing dependencies fails: the run closes as build_failed
+#   5. a step dies without closing the run (a missing variable)
+#   6. Set up Node.js (a stock action) fails
+# Every scenario also checks the step list the pipeline reports for the build page.
 # Usage: templates/ci-actions/test/run-local.sh
 set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 SCRIPTS="$HERE/../pipeline/scripts"
 PORT="${PORT:-8765}"
 WORK="$(mktemp -d)"
-trap 'kill ${STUB_PID:-} 2>/dev/null || true; rm -rf "$WORK"' EXIT
+FIXTURE_APP_ID="11111111-2222-4333-8444-555555555555"
+trap 'kill ${STUB_PID:-} 2>/dev/null || true; [ -n "${KEEP_WORK:-}" ] || rm -rf "$WORK"' EXIT
 pass=0; fail=0
 check() { if eval "$2"; then echo "  ok   $1"; pass=$((pass+1)); else echo "  FAIL $1"; fail=$((fail+1)); fi; }
 
@@ -23,7 +28,7 @@ make_app() { # dir  required_check_cmd
   "typecheck": "$reqcmd" } }
 JSON
   cat > "$d/.dealsage/app.json" <<JSON
-{ "appId": "11111111-2222-4333-8444-555555555555",
+{ "appId": "$FIXTURE_APP_ID",
   "checks": [
     { "id": "typecheck", "name": "TypeScript", "run": "npm run --silent typecheck" },
     { "id": "lint", "name": "Lint", "run": "echo 'lint problem' && exit 3", "required": false },
@@ -37,29 +42,46 @@ JSON
     && git -C "$d" -c user.name=t -c user.email=t@t commit -qm "Add fixture app"
 }
 
-run_pipeline() { # dir  -> runs manifest, checks, build, publish; echoes the failing stage or "ok"
-  local d="$1" ghenv ghout ghsum
-  ghenv="$(mktemp)"; ghout="$(mktemp)"; ghsum="$(mktemp)"
-  export GITHUB_ENV="$ghenv" GITHUB_OUTPUT="$ghout" GITHUB_STEP_SUMMARY="$ghsum"
-  export GITHUB_SHA="$(git -C "$d" rev-parse HEAD)" GITHUB_REF_NAME="feature/x" GITHUB_ACTOR="tester"
-  export DEALSAGE_FUNCTIONS_URL="http://127.0.0.1:$PORT" PUBLISH_TOKEN="test-publish-token"
-  export SUPABASE_ANON_KEY="anon" SUPABASE_URL="https://example.supabase.co" CUSTOM_APPS_WORKER_URL="http://127.0.0.1:$PORT"
-  export MANIFEST_PATH=".dealsage/app.json"
-  unset APP_ID MANIFEST_JSON CHECKS_JSON CHECKS_RESULT_JSON
-  local stage
-  for stage in manifest run-checks build publish; do
-    ( cd "$d" && bash "$SCRIPTS/$stage.sh" ) > "$WORK/$stage.log" 2>&1 || { echo "$stage"; cat "$ghenv" > "$WORK/last.env"; return 0; }
-    # composite steps share state through GITHUB_ENV (heredoc form); load it.
-    eval "$(python3 - "$ghenv" <<'PY'
+load_env() { # GITHUB_ENV file -> exports; composite steps share state through it (heredoc form)
+  eval "$(python3 - "$1" <<'PY'
 import re,sys,shlex
 txt=open(sys.argv[1]).read()
 for m in re.finditer(r'^(\w+)<<__DS_EOF__\n(.*?)\n__DS_EOF__$', txt, re.S|re.M):
     print(f"export {m.group(1)}={shlex.quote(m.group(2))}")
 PY
 )"
+}
+
+# run_pipeline DIR -> runs the action's steps in order, stopping at the first
+# that fails, then finish-steps.sh (if: always()); echoes the step that failed
+# or "ok". Knobs, as environment: INSTALL_COMMAND, UNSET_ENV (names to unset),
+# FAIL_NODE=1 (Set up Node.js fails; nothing of ours runs in that step).
+run_pipeline() {
+  local d="$1" ghenv ghout ghsum
+  ghenv="$(mktemp)"; ghout="$(mktemp)"; ghsum="$(mktemp)"
+  export GITHUB_ENV="$ghenv" GITHUB_OUTPUT="$ghout" GITHUB_STEP_SUMMARY="$ghsum"
+  export RUNNER_TEMP; RUNNER_TEMP="$(mktemp -d)"
+  export GITHUB_SHA="$(git -C "$d" rev-parse HEAD)" GITHUB_REF_NAME="feature/x" GITHUB_ACTOR="tester"
+  export DEALSAGE_FUNCTIONS_URL="http://127.0.0.1:$PORT" PUBLISH_TOKEN="test-publish-token"
+  export SUPABASE_ANON_KEY="anon" SUPABASE_URL="https://example.supabase.co" CUSTOM_APPS_WORKER_URL="http://127.0.0.1:$PORT"
+  export MANIFEST_PATH=".dealsage/app.json" INSTALL_COMMAND="${INSTALL_COMMAND:-echo 'up to date, audited 0 packages'}"
+  unset APP_ID MANIFEST_JSON CHECKS_JSON CHECKS_RESULT_JSON BUILD_ID
+  local v; for v in ${UNSET_ENV:-}; do unset "$v"; done
+  local stage failed=""
+  for stage in ensure-tools manifest start-build node install run-checks build publish; do
+    if [ "$stage" = node ]; then
+      if [ "${FAIL_NODE:-0}" = 1 ]; then failed=node; break; fi
+      continue
+    fi
+    ( cd "$d" && bash "$SCRIPTS/$stage.sh" ) > "$WORK/$stage.log" 2>&1 || failed="$stage"
+    load_env "$ghenv"
+    [ -z "$failed" ] || break
   done
+  if ! ( cd "$d" && bash "$SCRIPTS/finish-steps.sh" ) > "$WORK/finish-steps.log" 2>&1; then
+    failed="${failed:-finish-steps}"
+  fi
   cp "$ghout" "$WORK/last.out"; cp "$ghsum" "$WORK/last.summary"
-  echo ok
+  echo "${failed:-ok}"
 }
 
 start_stub() { # policy
@@ -70,6 +92,16 @@ start_stub() { # policy
 stop_stub() { kill "$STUB_PID" 2>/dev/null || true; wait "$STUB_PID" 2>/dev/null || true; }
 req() { jq -c "$1" "$WORK/requests.jsonl"; }
 
+# The step lists the pipeline posted (record-app-build, action "steps").
+first_steps_post() { jq -cs 'map(select(.body.action=="steps"))[0].body' "$WORK/requests.jsonl"; }
+last_steps() { jq -c 'select(.body.action=="steps") | .body.steps' "$WORK/requests.jsonl" | tail -n 1; }
+statuses_of() { jq -r 'map(.id + "=" + .status) | join(" ")'; }
+statuses() { last_steps | statuses_of; }
+step() { last_steps | jq -r --arg id "$1" "map(select(.id == \$id))[0] | $2"; }
+finishes() { req 'select(.body.action=="finish")' | wc -l | tr -d ' '; }
+finish_is() { req 'select(.body.action=="finish") | .body' | jq -e --arg o "$1" --arg r "$2" '.outcome == $o and .failureReason == $r' >/dev/null; }
+ALL_PASSED="tooling=passed manifest=passed record=passed node=passed install=passed checks=passed build=passed publish=passed"
+
 echo "Scenario 1: auto policy"
 make_app "$WORK/app1" "true"; start_stub auto
 r="$(run_pipeline "$WORK/app1")"
@@ -82,6 +114,21 @@ check "allocate sent branch, message, author, manifest and check results" "req '
 check "two files uploaded through the Worker with the upload token" "[ \"\$(req 'select(.method==\"PUT\" and .auth==\"Bearer upl-token\")' | wc -l)\" = 2 ]"
 check "promote requested after upload" "req 'select(.body.action==\"promote\") | .body.version' | grep -q 7"
 check "outputs: version=7 promoted=true" "grep -q '^version=7$' '$WORK/last.out' && grep -q '^promoted=true$' '$WORK/last.out'"
+check "build start recorded with commit, branch and author" "req 'select(.body.action==\"start\") | .body | (.commitSha|length)==40 and .branch==\"feature/x\" and .author==\"t\"' | grep -q true"
+check "allocate carries the build id" "req 'select(.body.action==\"allocate\") | .body.buildId' | grep -q '\"bld-1\"'"
+check "build closed as built, promoted, v7" "req 'select(.body.action==\"finish\") | .body | .buildId==\"bld-1\" and .outcome==\"built\" and .promoted==true and .version==7' | grep -q true"
+check "steps first sent with the build record, for this build and app" \
+  'first_steps_post | jq -e --arg app "$FIXTURE_APP_ID" ".buildId == \"bld-1\" and .appId == \$app" >/dev/null'
+check "the first post has the steps so far and Set up Node.js running" \
+  '[ "$(first_steps_post | jq -c .steps | statuses_of)" = "tooling=passed manifest=passed record=passed node=running install=pending checks=pending build=pending publish=pending" ]'
+check "every step passed, in action.yml order" '[ "$(statuses)" = "$ALL_PASSED" ]'
+check "each step has a start, a finish and a duration" \
+  'last_steps | jq -e "all(.[]; (.startedAt|type) == \"string\" and (.finishedAt|type) == \"string\" and (.durationMs|type) == \"number\" and .durationMs >= 0)" >/dev/null'
+check "internal fields stay out of the payload" '! req "select(.body.action==\"steps\") | .body.steps[] | select(has(\"startedMs\"))" | grep -q .'
+check "summaries: manifest, checks (the advisory failure named), build, publish" \
+  '[ "$(step manifest .summary)" = "3 checks declared" ] && [ "$(step checks .summary)" = "2 of 3 passed (Lint failed, not required)" ] && [ "$(step build .summary)" = "2 files in dist/" ] && [ "$(step publish .summary)" = "v7 is live" ]'
+check "no log tails on a green run" '[ "$(last_steps | jq "map(select(has(\"logTail\"))) | length")" = 0 ]'
+check "the finalizer did not close the run again" '[ "$(finishes)" = 1 ]'
 stop_stub
 
 echo "Scenario 2: required check fails"
@@ -91,6 +138,12 @@ check "stopped at run-checks" "[ '$r' = run-checks ]"
 check "failure still recorded" "req 'select(.body.checkId==\"typecheck\") | .body.status' | grep -q '\"failed\"'"
 check "nothing allocated" "! req 'select(.body.action==\"allocate\")' | grep -q ."
 check "advisory and platform checks still ran" "[ \"\$(req 'select(.path|endswith(\"/record-app-check\"))' | wc -l)\" = 3 ]"
+check "build closed as checks_failed with the reason" "req 'select(.body.action==\"finish\") | .body | .outcome==\"checks_failed\" and (.failureReason|test(\"TypeScript\"))' | grep -q true"
+check "steps: Checks failed, Build and Publish skipped" \
+  '[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=passed checks=failed build=skipped publish=skipped" ]'
+check "Checks names the required check that failed" 'step checks .summary | grep -q "Required check(s) failed: TypeScript"'
+check "skipped steps carry no timings" '[ "$(step build ".startedAt // \"none\"")" = none ]'
+check "the finalizer did not close the run again" '[ "$(finishes)" = 1 ]'
 stop_stub
 
 echo "Scenario 3: manual policy"
@@ -99,6 +152,44 @@ r="$(run_pipeline "$WORK/app3")"
 check "pipeline completed" "[ '$r' = ok ]"
 check "outputs: promoted=false, channel=branch/feature/x" "grep -q '^promoted=false$' '$WORK/last.out' && grep -q '^channel=branch/feature/x$' '$WORK/last.out'"
 check "summary mentions the channel" "grep -q 'channel' '$WORK/last.summary'"
+check "build closed as built but not promoted" "req 'select(.body.action==\"finish\") | .body | .outcome==\"built\" and .promoted==false and .version==7' | grep -q true"
+check "every step passed" '[ "$(statuses)" = "$ALL_PASSED" ]'
+check "Publish says where the build was kept" '[ "$(step publish .summary)" = "v7 kept on branch/feature/x; live stays at v6" ]'
+stop_stub
+
+echo "Scenario 4: installing dependencies fails"
+make_app "$WORK/app4" "true"; start_stub auto
+r="$(INSTALL_COMMAND="echo 'npm error 404 Not Found - GET https://registry.npmjs.org/left-pad-nope' >&2; exit 1" run_pipeline "$WORK/app4")"
+check "stopped at install" '[ "$r" = install ]'
+check "build closed as build_failed, saying why" \
+  'finish_is build_failed "Installing dependencies failed (exit code 1); nothing was published."'
+check "steps: Install dependencies failed, the rest skipped" \
+  '[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=failed checks=skipped build=skipped publish=skipped" ]'
+check "Install dependencies carries the tail of its output" 'step install .logTail | grep -q "npm error 404"'
+check "no checks ran and nothing was allocated" \
+  '! req "select((.path|endswith(\"/record-app-check\")) or .body.action == \"allocate\")" | grep -q .'
+check "the run was closed once" '[ "$(finishes)" = 1 ]'
+stop_stub
+
+echo "Scenario 5: a step dies without closing the run"
+make_app "$WORK/app5" "true"; start_stub auto
+r="$(UNSET_ENV="SUPABASE_URL" run_pipeline "$WORK/app5")"
+check "stopped at build" '[ "$r" = build ]'
+check "the finalizer closed the run as failed, naming the step and the cause" \
+  'finish_is failed "Build: SUPABASE_URL is not set"'
+check "steps: Build failed, Publish skipped" \
+  '[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=passed checks=passed build=failed publish=skipped" ]'
+check "the run was closed once" '[ "$(finishes)" = 1 ]'
+stop_stub
+
+echo "Scenario 6: Set up Node.js fails"
+make_app "$WORK/app6" "true"; start_stub auto
+r="$(FAIL_NODE=1 run_pipeline "$WORK/app6")"
+check "stopped at node" '[ "$r" = node ]'
+check "the finalizer failed the step that never finished and closed the run" \
+  'finish_is failed "Set up Node.js did not finish."'
+check "steps: Set up Node.js failed with a duration, the rest skipped" \
+  '[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=failed install=skipped checks=skipped build=skipped publish=skipped" ] && [ "$(step node ".durationMs | type")" = number ]'
 stop_stub
 
 echo; echo "$pass passed, $fail failed"

test/stub_server.py

@@ -7,6 +7,8 @@ like the real functions would:
                             promote  -> promoted per STUB_POLICY (auto|manual)
                             fail     -> ok
   POST /record-app-check    -> {"id": "..."}
+  POST /record-app-build    start -> {"buildId": "bld-1"}; finish -> echo;
+                            steps -> {"buildId": ..., "steps": <count>}
   PUT  /__upload/<app>/<v>/<path> -> 204
 """
 import json, os, sys
@@ -52,6 +54,17 @@ class H(BaseHTTPRequestHandler):
         if self.path.endswith("/record-app-check"):
             seq["n"] += 1
             return self._json(200, {"id": f"chk-{seq['n']}"})
+        if self.path.endswith("/record-app-build"):
+            if body.get("action") == "start":
+                return self._json(200, {"buildId": "bld-1"})
+            if body.get("action") == "finish":
+                return self._json(200, {"buildId": body.get("buildId"), "outcome": body.get("outcome")})
+            if body.get("action") == "steps":
+                steps = body.get("steps")
+                if not isinstance(steps, list) or not steps:
+                    return self._json(400, {"error": "steps must be a non-empty array"})
+                return self._json(200, {"buildId": body.get("buildId"), "steps": len(steps)})
+            return self._json(400, {"error": "bad action"})
         self._json(404, {"error": "no route"})
 
     def do_PUT(self):