From 36aeaf65c3cf13c3c2856f2cf1418ffdb4321daf Mon Sep 17 00:00:00 2001 From: DealSage template sync Date: Fri, 25 Sep 2026 17:56:38 +0000 Subject: [PATCH] Sync from deal-sage/deal-sage@13a420d134d4e76d4a6885c0e3cac2d2f1524543 --- README.md | 77 +++++++++++++++++++++++ VERSION | 1 + manifest.schema.json | 65 +++++++++++++++++++ pipeline/action.yml | 80 +++++++++++++++++++++++ pipeline/checks/conventions.sh | 54 ++++++++++++++++ pipeline/scripts/build.sh | 36 +++++++++++ pipeline/scripts/ensure-tools.sh | 23 +++++++ pipeline/scripts/lib.sh | 88 ++++++++++++++++++++++++++ pipeline/scripts/manifest.sh | 49 +++++++++++++++ pipeline/scripts/publish.sh | 84 +++++++++++++++++++++++++ pipeline/scripts/run-checks.sh | 73 +++++++++++++++++++++ test/run-local.sh | 105 +++++++++++++++++++++++++++++++ test/stub_server.py | 69 ++++++++++++++++++++ 13 files changed, 804 insertions(+) create mode 100644 README.md create mode 100644 VERSION create mode 100644 manifest.schema.json create mode 100644 pipeline/action.yml create mode 100755 pipeline/checks/conventions.sh create mode 100755 pipeline/scripts/build.sh create mode 100755 pipeline/scripts/ensure-tools.sh create mode 100755 pipeline/scripts/lib.sh create mode 100755 pipeline/scripts/manifest.sh create mode 100755 pipeline/scripts/publish.sh create mode 100755 pipeline/scripts/run-checks.sh create mode 100755 test/run-local.sh create mode 100755 test/stub_server.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..6aafc74 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# ci-actions — the platform-owned custom-app pipeline + +This directory is synced verbatim into the `dealsage-platform/ci-actions` repo on +each environment's Gitea by `infra/gitea/gitea-org.sh` (the same way +`templates/custom-app` is synced into the template repo), and tagged `v1` plus +`v` from the `VERSION` file. App repos call it from a one-step +workflow: + +```yaml +- uses: https://git.dealsage.io/dealsage-platform/ci-actions/pipeline@v1 + with: + publish-token: ${{ secrets.PUBLISH_TOKEN }} +``` + +Why it exists: an app repo receives its workflow once, when it is generated +from the template, and never again. Keeping the pipeline here means a fix ships +to every app on its next push, instead of being patched repo by repo. + +## What `pipeline` does + +| Step | Script | Notes | +|---|---|---| +| Tooling | `scripts/ensure-tools.sh` | curl + jq; AWS CLI only for the S3 upload path | +| Manifest | `scripts/manifest.sh` | validates `.dealsage/app.json`, exports `APP_ID`, `CHECKS_JSON` | +| Install | (action input) | `npm install --no-audit --no-fund` by default | +| Checks | `scripts/run-checks.sh` | runs each declared check, posts every result to `record-app-check`; a failed **required** check stops here | +| Build | `scripts/build.sh` | `npm run build` with `VITE_SUPABASE_URL/ANON_KEY` from org variables; recorded as the `build` check | +| Publish | `scripts/publish.sh` | allocate → upload → promote request; DealSage applies the app's publish policy | + +The action never decides whether a build goes live. `record-app-version`'s +`promote` action does, from the app's `publish_policy` and default branch, and +answers `promoted: true|false` plus the channel the version landed on +(`live` or `branch/`). Every build of every branch becomes a version, so +any of them can be opened by URL and promoted later. + +## The manifest (`.dealsage/app.json`) + +Owned by the app's team; schema in `manifest.schema.json`. + +```json +{ + "appId": "…", + "checks": [ + { "id": "typecheck", "name": "TypeScript", "run": "npm run typecheck" }, + { "id": "test", "name": "Unit tests", "run": "npm test -- --run", "required": true }, + { "id": "conventions", "uses": "conventions", "required": false } + ], + "dependencies": { "objectTypes": [ { "apiName": "contract_agreement", "schemaVersion": 2 } ] } +} +``` + +`required` defaults to true. Platform checks (`uses`) live in `pipeline/checks/` +and run inside the same job; `conventions` verifies the repo still has the +shape the platform relies on. + +## Environment the caller provides + +From org variables: `DEALSAGE_FUNCTIONS_URL`, `SUPABASE_URL`, +`SUPABASE_ANON_KEY`, and either `CUSTOM_APPS_WORKER_URL` (bundles are PUT +through the serving Worker with the short-lived upload token from `allocate`, +so CI holds no R2 key) or `R2_ACCOUNT_ID` + `R2_BUCKET` with the +`R2_ACCESS_KEY_ID` / `R2_SECRET_ACCESS_KEY` secrets (S3 sync, the path prod +uses today). `PUBLISH_TOKEN` is the app's own repo secret. + +## Testing locally + +`test/run-local.sh` builds a throwaway app repo and runs the scripts against a +stub DealSage (`test/stub_server.py`) for three scenarios: auto policy, a +failing required check, and manual policy. Run it after any change here; it +needs bash, jq, git, node and python3. + +## Releasing a change + +Bump `VERSION` for anything user-visible. Non-breaking changes keep the `v1` +tag moving; a breaking change (new required input, different env contract) +gets `v2` and a stub update in the template plus a platform-files refresh for +existing apps. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/manifest.schema.json b/manifest.schema.json new file mode 100644 index 0000000..31f62ad --- /dev/null +++ b/manifest.schema.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://dealsage.io/schemas/custom-app-manifest.json", + "title": "DealSage custom app manifest (.dealsage/app.json)", + "description": "The per-app contract the platform pipeline reads: which app this repo is, which checks to run before a build is published, and which platform objects the app depends on. The pipeline itself is platform-owned (dealsage-platform/ci-actions); this file is owned by the app's team.", + "type": "object", + "additionalProperties": false, + "required": ["appId"], + "properties": { + "appId": { + "type": "string", + "description": "The custom_apps.id this repo belongs to. Stamped by provisioning; never edit.", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "checks": { + "type": "array", + "description": "Checks the pipeline runs on every push, in order, before building. A failed required check stops the build; results are recorded in DealSage either way.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier, unique within the app (e.g. typecheck, test, lint).", + "pattern": "^[a-z][a-z0-9-]{0,39}$" + }, + "name": { "type": "string", "description": "Label shown in DealSage. Defaults to the id." }, + "run": { "type": "string", "description": "Shell command executed in the repo root. Exactly one of run or uses." }, + "uses": { + "type": "string", + "description": "A platform-provided check to run instead of a command. Known values: conventions (validates this manifest and the repo layout).", + "enum": ["conventions"] + }, + "required": { + "type": "boolean", + "default": true, + "description": "When true, a failure stops the build. When false, the result is recorded and the pipeline continues." + }, + "timeoutMinutes": { "type": "integer", "minimum": 1, "maximum": 60, "default": 15 } + }, + "oneOf": [{ "required": ["run"] }, { "required": ["uses"] }] + } + }, + "dependencies": { + "type": "object", + "description": "Platform objects the app is built against. Recorded with every version so DealSage can flag drift when a dependency changes.", + "additionalProperties": false, + "properties": { + "objectTypes": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["apiName"], + "properties": { + "apiName": { "type": "string" }, + "schemaVersion": { "type": "integer", "minimum": 1, "description": "The schema version the app was written against." } + } + } + } + } + } + } +} diff --git a/pipeline/action.yml b/pipeline/action.yml new file mode 100644 index 0000000..63b61c0 --- /dev/null +++ b/pipeline/action.yml @@ -0,0 +1,80 @@ +name: DealSage custom app pipeline +description: >- + Runs the checks declared in .dealsage/app.json, builds the app with the + DealSage connection baked in, uploads the bundle, and asks DealSage to + release it under the app's publish policy. Platform-owned: app repos call + this action from a one-step workflow, so pipeline changes ship here, not in + every repo. +author: DealSage + +inputs: + publish-token: + description: The app's PUBLISH_TOKEN repo secret (appId-scoped, minted by DealSage). + required: true + node-version: + description: Node.js version for install, checks and build. + required: false + default: "20" + install-command: + description: Dependency install command run in the repo root. + required: false + default: npm install --no-audit --no-fund + skip-publish: + description: Run checks and build only; do not allocate or upload a version. + required: false + default: "false" + +outputs: + version: + description: The version number allocated for this build (empty when skip-publish). + value: ${{ steps.publish.outputs.version }} + promoted: + description: "true when DealSage promoted this version to live under the app's policy." + value: ${{ steps.publish.outputs.promoted }} + channel: + description: The channel the version landed on (live, or branch/). + value: ${{ steps.publish.outputs.channel }} + +# Environment the caller provides (from org variables/secrets, see the template +# workflow): DEALSAGE_FUNCTIONS_URL, SUPABASE_URL, SUPABASE_ANON_KEY, and either +# CUSTOM_APPS_WORKER_URL (upload through the Worker) or R2_ACCOUNT_ID, +# R2_BUCKET, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY (S3 upload). +runs: + using: composite + steps: + - name: Tooling + shell: bash + run: bash "${{ github.action_path }}/scripts/ensure-tools.sh" + + - name: Read the app manifest + shell: bash + run: bash "${{ github.action_path }}/scripts/manifest.sh" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Install dependencies + shell: bash + run: ${{ inputs.install-command }} + + - name: Checks + shell: bash + env: + PUBLISH_TOKEN: ${{ inputs.publish-token }} + run: bash "${{ github.action_path }}/scripts/run-checks.sh" + + - name: Build + shell: bash + env: + PUBLISH_TOKEN: ${{ inputs.publish-token }} + run: bash "${{ github.action_path }}/scripts/build.sh" + + - name: Publish + id: publish + if: inputs.skip-publish != 'true' + shell: bash + env: + PUBLISH_TOKEN: ${{ inputs.publish-token }} + run: bash "${{ github.action_path }}/scripts/publish.sh" diff --git a/pipeline/checks/conventions.sh b/pipeline/checks/conventions.sh new file mode 100755 index 0000000..4bad358 --- /dev/null +++ b/pipeline/checks/conventions.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Platform check "conventions": the repo still has the shape the platform relies +# on. Hard failures exit 1 (the check fails); soft findings print a warning and +# pass. Runs from the repo root. Kept dependency-free (bash + jq). +set -uo pipefail +fail=0 +problem() { printf 'FAIL %s\n' "$*"; fail=1; } +note() { printf 'warn %s\n' "$*"; } +ok() { printf 'ok %s\n' "$*"; } + +# 1. Manifest present and shaped (manifest.sh already validated the checks list). +if [ -f .dealsage/app.json ] && jq -e '.appId | test("^[0-9a-f-]{36}$")' .dealsage/app.json >/dev/null 2>&1; then + ok ".dealsage/app.json has an appId" +else + problem ".dealsage/app.json is missing or has no appId" +fi +if jq -e 'keys - ["appId","checks","dependencies"] | length == 0' .dealsage/app.json >/dev/null 2>&1; then + ok "manifest has only known keys" +else + problem "manifest has unknown top-level keys: $(jq -c 'keys - ["appId","checks","dependencies"]' .dealsage/app.json 2>/dev/null)" +fi + +# 2. The workflow is the platform stub, so pipeline changes reach this repo. +wf=.gitea/workflows/publish.yml +if [ -f "$wf" ]; then + if grep -qE 'ci-actions/pipeline@' "$wf"; then + ok "workflow delegates to the platform pipeline" + else + note "workflow does not call ci-actions/pipeline; this repo will not receive pipeline updates" + fi +else + problem "$wf is missing" +fi + +# 3. Build contract. +if jq -e '.scripts.build' package.json >/dev/null 2>&1; then ok "package.json has a build script"; else problem "package.json has no build script"; fi + +# 4. Platform files the shell and the builder depend on. +for f in src/lib/dealsage.ts src/Root.tsx src/main.tsx; do + [ -f "$f" ] && ok "$f present" || problem "$f is missing (platform-owned file)" +done + +# 5. Guidance for agents. Soft: older apps predate it. +[ -f AGENTS.md ] && ok "AGENTS.md present" || note "AGENTS.md missing; coding agents get no project guidance" +if [ -f AGENTS.md ] && ! grep -q 'dealsage:generated:start' AGENTS.md; then + note "AGENTS.md has no generated block markers; the builder will not preserve hand-written sections" +fi + +# 6. Nothing secret committed. +for f in .env .env.local .env.production; do + [ -f "$f" ] && problem "$f is committed; the pipeline bakes the Supabase connection from org variables instead" +done + +exit $fail diff --git a/pipeline/scripts/build.sh b/pipeline/scripts/build.sh new file mode 100755 index 0000000..452c2ba --- /dev/null +++ b/pipeline/scripts/build.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Build the app with the DealSage connection baked in from org variables, and +# record the build as a check so DealSage shows it beside the declared ones. +set -euo pipefail +. "$(dirname "$0")/lib.sh" +require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY SUPABASE_URL +git_context + +logf="$(mktemp)"; started="$(iso_now)"; t0="$(now_ms)" +echo "::group::build" +set +e +VITE_SUPABASE_URL="$SUPABASE_URL" VITE_SUPABASE_ANON_KEY="$SUPABASE_ANON_KEY" \ + npm run build 2>&1 | tee "$logf" +rc=${PIPESTATUS[0]} +set -e +echo "::endgroup::" +dur=$(( $(now_ms) - t0 )) +if [ "$rc" = 0 ] && [ ! -f dist/index.html ]; then + echo "build succeeded but dist/index.html is missing" | tee -a "$logf"; rc=2 +fi +status=passed; summary="built in $(( dur / 1000 ))s" +[ "$rc" = 0 ] || { status=failed; summary="exit code $rc"; } + +payload="$(jq -n \ + --arg appId "$APP_ID" --arg sha "$COMMIT_SHA" --arg branch "$BRANCH" \ + --arg status "$status" --argjson duration "$dur" --arg summary "$summary" --rawfile log "$logf" \ + --arg started "$started" --arg finished "$(iso_now)" \ + '{appId:$appId, commitSha:$sha, branch:$branch, checkId:"build", name:"Build", status:$status, + required:true, durationMs:$duration, summary:$summary, + logTail:($log | split("\n") | .[-80:] | join("\n")), reportedBy:"ci", + startedAt:$started, finishedAt:$finished}')" +ds_post record-app-check "$payload" >/dev/null || warn "could not record the build check in DealSage" +rm -f "$logf" +summary "- build: **$status** (${dur}ms)" +[ "$rc" = 0 ] || die "Build failed ($summary); nothing was published." +log "Build ok: $(find dist -type f | wc -l) files" diff --git a/pipeline/scripts/ensure-tools.sh b/pipeline/scripts/ensure-tools.sh new file mode 100755 index 0000000..d704dd4 --- /dev/null +++ b/pipeline/scripts/ensure-tools.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Make sure the runner has what the pipeline shells out to: curl and jq always, +# the AWS CLI only when bundles go to R2 through the S3 API (no +# CUSTOM_APPS_WORKER_URL). Installs are apt-based, matching the runner image. +set -euo pipefail +. "$(dirname "$0")/lib.sh" + +need_apt=() +command -v curl >/dev/null || need_apt+=(curl) +command -v jq >/dev/null || need_apt+=(jq) +if [ "${#need_apt[@]}" -gt 0 ]; then + log "Installing ${need_apt[*]}" + apt-get update -qq && apt-get install -y -qq --no-install-recommends "${need_apt[@]}" >/dev/null +fi + +if [ -z "${CUSTOM_APPS_WORKER_URL:-}" ] && ! command -v aws >/dev/null; then + log "Installing the AWS CLI (bundle upload goes to R2 over S3)" + command -v unzip >/dev/null || { apt-get update -qq && apt-get install -y -qq --no-install-recommends unzip >/dev/null; } + curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip + unzip -q -o /tmp/awscliv2.zip -d /tmp + /tmp/aws/install >/dev/null +fi +echo "tools ready: curl $(curl --version | head -n1 | cut -d' ' -f2), jq $(jq --version)$( command -v aws >/dev/null && echo ", aws $(aws --version 2>&1 | cut -d' ' -f1)")" diff --git a/pipeline/scripts/lib.sh b/pipeline/scripts/lib.sh new file mode 100755 index 0000000..b08db77 --- /dev/null +++ b/pipeline/scripts/lib.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Shared helpers for the DealSage custom-app pipeline scripts. Sourced, not run. +# +# Conventions: +# * Every script is `set -euo pipefail` and sources this file first. +# * DealSage is reached only through DEALSAGE_FUNCTIONS_URL with the app's +# PUBLISH_TOKEN (appId-scoped, minted by DealSage). Nothing here holds a +# Supabase service key or an R2 account key. +# * Output meant for the job log goes through log(); machine-readable state +# is exchanged through $GITHUB_ENV / $GITHUB_OUTPUT like any other action. + +log() { printf '::group::%s\n' "$*" >/dev/null; printf '\n== %s\n' "$*"; } +warn() { printf '::warning::%s\n' "$*"; } +die() { printf '::error::%s\n' "$*"; exit 1; } + +# now_ms — wall clock in milliseconds (GNU date; falls back to seconds*1000). +now_ms() { date +%s%3N 2>/dev/null || echo $(( $(date +%s) * 1000 )); } + +# iso_now — UTC timestamp for API payloads. +iso_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# require_env NAME... — fail loudly when a required variable is missing. +require_env() { + local missing=0 v + for v in "$@"; do + if [ -z "${!v:-}" ]; then printf '::error::%s is not set\n' "$v"; missing=1; fi + done + [ "$missing" = 0 ] || exit 1 +} + +# json_escape STRING — JSON string literal (with quotes) via jq. +json_escape() { jq -Rn --arg s "$1" '$s'; } + +# tail_file FILE [LINES] — last LINES lines of FILE (default 80), for log tails. +tail_file() { tail -n "${2:-80}" "$1" 2>/dev/null || true; } + +# ds_post PATH JSON — POST JSON to a DealSage function with the publish token. +# Prints the response body; returns non-zero on HTTP failure (body still printed +# to stderr so the job log shows why). +ds_post() { + local path="$1" body="$2" out code + out="$(mktemp)" + code="$(curl -sS -o "$out" -w '%{http_code}' -X POST \ + "${DEALSAGE_FUNCTIONS_URL%/}/${path}" \ + -H "Authorization: Bearer ${PUBLISH_TOKEN}" \ + -H "apikey: ${SUPABASE_ANON_KEY}" \ + -H "content-type: application/json" \ + --data-binary "$body")" || { cat "$out" >&2; rm -f "$out"; return 1; } + if [ "${code:0:1}" != "2" ]; then + printf 'DealSage %s returned HTTP %s: ' "$path" "$code" >&2; cat "$out" >&2; echo >&2 + rm -f "$out"; return 1 + fi + cat "$out"; rm -f "$out" +} + +# git_context — export the commit/branch/author the pipeline is publishing. +# Gitea Actions provides the GITHUB_* variables; the event payload carries the +# head commit. Falls back to git itself (the checkout has the commit). +git_context() { + local ev="${GITHUB_EVENT_PATH:-}" + COMMIT_SHA="${GITHUB_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" + BRANCH="${GITHUB_REF_NAME:-}" + if [ -z "$BRANCH" ]; then + BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" + fi + COMMIT_MESSAGE=""; COMMIT_AUTHOR="" + if [ -n "$ev" ] && [ -f "$ev" ]; then + COMMIT_MESSAGE="$(jq -r '.head_commit.message // .commits[-1].message // empty' "$ev" 2>/dev/null || true)" + COMMIT_AUTHOR="$(jq -r '.head_commit.author.name // .pusher.login // .pusher.username // empty' "$ev" 2>/dev/null || true)" + fi + [ -n "$COMMIT_MESSAGE" ] || COMMIT_MESSAGE="$(git log -1 --pretty=%B 2>/dev/null | head -c 2000 || true)" + [ -n "$COMMIT_AUTHOR" ] || COMMIT_AUTHOR="$(git log -1 --pretty=%an 2>/dev/null || echo "${GITHUB_ACTOR:-}")" + # First line only for the summary field; the full message travels separately. + COMMIT_SUBJECT="$(printf '%s' "$COMMIT_MESSAGE" | head -n 1 | head -c 200)" + export COMMIT_SHA BRANCH COMMIT_MESSAGE COMMIT_AUTHOR COMMIT_SUBJECT +} + +# persist_env NAME VALUE — make a value visible to later composite steps. +persist_env() { + if [ -n "${GITHUB_ENV:-}" ]; then + # Heredoc form handles multi-line values safely. + { printf '%s<<__DS_EOF__\n' "$1"; printf '%s\n' "$2"; printf '__DS_EOF__\n'; } >> "$GITHUB_ENV" + fi + export "$1=$2" +} + +# summary MARKDOWN — append to the run's step summary when the runner supports it. +summary() { [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$*" >> "$GITHUB_STEP_SUMMARY" || true; } diff --git a/pipeline/scripts/manifest.sh b/pipeline/scripts/manifest.sh new file mode 100755 index 0000000..a512606 --- /dev/null +++ b/pipeline/scripts/manifest.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Read and validate .dealsage/app.json, then export what later steps need: +# APP_ID the custom app this repo publishes +# MANIFEST_JSON the whole manifest, compacted (recorded with the version) +# CHECKS_JSON the checks array with defaults applied +# Validation is deliberately jq-only (no npm dependency) so a broken manifest +# fails fast, before install. The schema of record is ../manifest.schema.json. +set -euo pipefail +. "$(dirname "$0")/lib.sh" + +MANIFEST_PATH="${MANIFEST_PATH:-.dealsage/app.json}" +[ -f "$MANIFEST_PATH" ] || die "Manifest not found at $MANIFEST_PATH" +jq -e . "$MANIFEST_PATH" >/dev/null 2>&1 || die "$MANIFEST_PATH is not valid JSON" + +app_id="$(jq -r '.appId // empty' "$MANIFEST_PATH")" +[[ "$app_id" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] \ + || die "$MANIFEST_PATH: appId is missing or not a UUID (got '${app_id:-}'). Provisioning stamps it; do not edit by hand." + +# Normalise checks: fill defaults, reject malformed entries with a clear message. +checks="$(jq -c ' + def bad(msg): error("checks: " + msg); + (.checks // []) + | if type != "array" then bad("must be an array") else . end + | map( + if type != "object" then bad("each check must be an object") else . end + | if (.id|type) != "string" or ((.id|test("^[a-z][a-z0-9-]{0,39}$"))|not) + then bad("id must match ^[a-z][a-z0-9-]{0,39}$ (got " + (.id|tostring) + ")") else . end + | if ((.run|type) == "string") == ((.uses|type) == "string") + then bad("check " + .id + " needs exactly one of run or uses") else . end + | if (.uses? // "") != "" and ((.uses|IN("conventions"))|not) + then bad("check " + .id + " uses an unknown platform check " + .uses) else . end + | { + id: .id, + name: (.name // .id), + run: (.run // null), + uses: (.uses // null), + required: (if .required == null then true else .required end), + timeoutMinutes: (.timeoutMinutes // 15) + } + ) + | if (map(.id) | unique | length) != length then bad("ids must be unique") else . end +' "$MANIFEST_PATH")" || exit 1 + +manifest="$(jq -c . "$MANIFEST_PATH")" +persist_env APP_ID "$app_id" +persist_env MANIFEST_JSON "$manifest" +persist_env CHECKS_JSON "$checks" +log "Manifest: app $app_id, $(jq 'length' <<<"$checks") check(s)" +jq -r '.[] | " - \(.id) (\(if .required then "required" else "advisory" end)): \(.run // ("platform:" + .uses))"' <<<"$checks" diff --git a/pipeline/scripts/publish.sh b/pipeline/scripts/publish.sh new file mode 100755 index 0000000..404eb85 --- /dev/null +++ b/pipeline/scripts/publish.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Publish dist/ as a new version of the app: +# 1. allocate — DealSage assigns version N for this commit/branch and returns +# where the bytes go plus a short-lived upload token. +# 2. upload — through the serving Worker when CUSTOM_APPS_WORKER_URL is set +# (no R2 key on CI), else `aws s3 sync` with the org's R2 keys. +# 3. promote — ask DealSage to release it. DealSage decides: a default-branch +# build under an "auto" policy goes live; anything else is kept +# as a built version on its branch channel for preview and a +# later, deliberate promotion. CI never knows the policy. +# The live pointer only ever moves after the bytes are in place. +set -euo pipefail +. "$(dirname "$0")/lib.sh" +require_env APP_ID DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY +git_context +[ -f dist/index.html ] || die "dist/index.html not found; run the build first" + +# --- 1. allocate ------------------------------------------------------------ +manifest_json="${MANIFEST_JSON:-{\}}" +checks_json="${CHECKS_RESULT_JSON:-[]}" +alloc_body="$(jq -n \ + --arg appId "$APP_ID" --arg sha "$COMMIT_SHA" --arg branch "$BRANCH" \ + --arg msg "$COMMIT_MESSAGE" --arg author "$COMMIT_AUTHOR" \ + --argjson manifest "$manifest_json" --argjson checks "$checks_json" \ + '{appId:$appId, action:"allocate", builtBy:"ci", commitSha:$sha, branch:$branch, + commitMessage:$msg, author:$author, manifest:$manifest, checks:$checks}')" +alloc="$(ds_post record-app-version "$alloc_body")" || die "allocate failed" +VERSION="$(jq -r '.version // empty' <<<"$alloc")" +PREFIX="$(jq -r '.r2Prefix // empty' <<<"$alloc")" +UPLOAD_TOKEN="$(jq -r '.uploadToken // empty' <<<"$alloc")" +[ -n "$VERSION" ] && [ -n "$PREFIX" ] || die "allocate returned no version/prefix: $alloc" +log "Allocated v$VERSION for $BRANCH@${COMMIT_SHA:0:7} -> $PREFIX" + +mark_failed() { + ds_post record-app-version "$(jq -n --arg appId "$APP_ID" --argjson v "$VERSION" --arg reason "$1" \ + '{appId:$appId, action:"fail", version:$v, reason:$reason}')" >/dev/null 2>&1 || true +} + +# --- 2. upload -------------------------------------------------------------- +if [ -n "${CUSTOM_APPS_WORKER_URL:-}" ]; then + [ -n "$UPLOAD_TOKEN" ] || { mark_failed "no upload token"; die "allocate returned no uploadToken but CUSTOM_APPS_WORKER_URL is set"; } + log "Uploading through the Worker" + export WORKER="${CUSTOM_APPS_WORKER_URL%/}" UPLOAD_TOKEN APP_ID VERSION + upload_one() { # relative path under dist/ + local rel="$1" code + code="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + "$WORKER/__upload/$APP_ID/$VERSION/$rel" \ + -H "Authorization: Bearer $UPLOAD_TOKEN" --data-binary "@dist/$rel")" || code=000 + case "$code" in 2*) ;; *) echo "upload failed ($code): $rel" >&2; return 1 ;; esac + } + export -f upload_one + if ! (cd dist && find . -type f | sed 's|^\./||') | xargs -P 6 -n 1 bash -c 'upload_one "$0"'; then + mark_failed "upload failed"; die "Upload failed; v$VERSION marked failed and was not promoted." + fi +else + require_env R2_ACCOUNT_ID R2_BUCKET R2_ACCESS_KEY_ID R2_SECRET_ACCESS_KEY + log "Uploading with the AWS CLI to s3://$R2_BUCKET/${PREFIX%/}" + if ! AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY" \ + AWS_DEFAULT_REGION=auto AWS_REQUEST_CHECKSUM_CALCULATION=when_required \ + AWS_RESPONSE_CHECKSUM_VALIDATION=when_required \ + aws s3 sync dist "s3://$R2_BUCKET/${PREFIX%/}" \ + --endpoint-url "https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com" --delete; then + mark_failed "upload failed"; die "Upload failed; v$VERSION marked failed and was not promoted." + fi +fi +log "Uploaded $(find dist -type f | wc -l) files" + +# --- 3. promote (DealSage applies the app's publish policy) ------------------ +promote="$(ds_post record-app-version "$(jq -n --arg appId "$APP_ID" --argjson v "$VERSION" \ + '{appId:$appId, action:"promote", version:$v}')")" || die "promote request failed" +promoted="$(jq -r '.promoted // false' <<<"$promote")" +channel="$(jq -r '.channel // empty' <<<"$promote")" +current="$(jq -r '.currentVersion // empty' <<<"$promote")" +if [ "$promoted" = true ]; then + log "v$VERSION is live" +else + live_note="nothing is live yet"; [ -n "$current" ] && live_note="live stays at v$current" + log "v$VERSION built and kept on channel '${channel:-branch/$BRANCH}' ($live_note); promote it from DealSage when ready" +fi +{ + echo "version=$VERSION"; echo "promoted=$promoted"; echo "channel=${channel}" +} >> "${GITHUB_OUTPUT:-/dev/null}" +summary "### Published" +summary "- **v$VERSION** from \`$BRANCH\` @ \`${COMMIT_SHA:0:7}\` — $( [ "$promoted" = true ] && echo "live" || echo "built, channel \`${channel:-branch/$BRANCH}\`" )" diff --git a/pipeline/scripts/run-checks.sh b/pipeline/scripts/run-checks.sh new file mode 100755 index 0000000..805858b --- /dev/null +++ b/pipeline/scripts/run-checks.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run the checks declared in the manifest, record each result in DealSage, and +# stop the pipeline if a required check failed. Advisory checks never block. +# +# Every result is posted to record-app-check with the commit and branch, so +# DealSage can show checks per commit before a version exists and attach them +# to the version once it is allocated. A DealSage outage is a warning here, not +# a failure: the build should not be blocked by the dashboard being down. +set -euo pipefail +. "$(dirname "$0")/lib.sh" +require_env APP_ID CHECKS_JSON DEALSAGE_FUNCTIONS_URL PUBLISH_TOKEN SUPABASE_ANON_KEY +git_context +CHECKS_DIR="$(cd "$(dirname "$0")/../checks" && pwd)" + +post_check() { # id name status required duration_ms summary logfile started finished + local payload + payload="$(jq -n \ + --arg appId "$APP_ID" --arg sha "$COMMIT_SHA" --arg branch "$BRANCH" \ + --arg id "$1" --arg name "$2" --arg status "$3" --argjson required "$4" \ + --argjson duration "$5" --arg summary "$6" --rawfile log "$7" \ + --arg started "$8" --arg finished "$9" \ + '{appId:$appId, commitSha:$sha, branch:$branch, checkId:$id, name:$name, status:$status, + required:$required, durationMs:$duration, summary:$summary, + logTail:($log | split("\n") | .[-80:] | join("\n")), reportedBy:"ci", + startedAt:$started, finishedAt:$finished}')" + ds_post record-app-check "$payload" >/dev/null || warn "could not record check $1 in DealSage" +} + +results='[]' +required_failed=() +count="$(jq 'length' <<<"$CHECKS_JSON")" +if [ "$count" = 0 ]; then + log "No checks declared in .dealsage/app.json" + persist_env CHECKS_RESULT_JSON '[]' + exit 0 +fi + +while IFS= read -r check; do + id="$(jq -r .id <<<"$check")"; name="$(jq -r .name <<<"$check")" + run="$(jq -r '.run // empty' <<<"$check")"; uses="$(jq -r '.uses // empty' <<<"$check")" + required="$(jq -r .required <<<"$check")"; timeout_min="$(jq -r .timeoutMinutes <<<"$check")" + logf="$(mktemp)"; started="$(iso_now)"; t0="$(now_ms)" + echo "::group::check: $name ($id)" + if [ -n "$uses" ]; then + printf '$ platform check %s\n' "$uses" | tee "$logf" + set +e; timeout "${timeout_min}m" bash "$CHECKS_DIR/$uses.sh" 2>&1 | tee -a "$logf"; rc=${PIPESTATUS[0]}; set -e + else + printf '$ %s\n' "$run" | tee "$logf" + set +e; timeout "${timeout_min}m" bash -o pipefail -c "$run" 2>&1 | tee -a "$logf"; rc=${PIPESTATUS[0]}; set -e + fi + echo "::endgroup::" + dur=$(( $(now_ms) - t0 )) + case "$rc" in + 0) status=passed; summary="passed in $(( dur / 1000 ))s" ;; + 124) status=error; summary="timed out after ${timeout_min} minutes" ;; + *) status=failed; summary="exit code $rc" ;; + esac + printf '%s: %s (%s)\n' "$name" "$status" "$summary" + post_check "$id" "$name" "$status" "$required" "$dur" "$summary" "$logf" "$started" "$(iso_now)" + results="$(jq -c --arg id "$id" --arg status "$status" --argjson d "$dur" --argjson req "$required" \ + '. + [{id:$id, status:$status, durationMs:$d, required:$req}]' <<<"$results")" + if [ "$status" != passed ] && [ "$required" = true ]; then required_failed+=("$name"); fi + rm -f "$logf" +done < <(jq -c '.[]' <<<"$CHECKS_JSON") + +persist_env CHECKS_RESULT_JSON "$results" +summary "### Checks" +summary "$(jq -r '.[] | "- \(.id): **\(.status)** (\(.durationMs)ms\(if .required then "" else ", advisory" end))"' <<<"$results")" + +if [ "${#required_failed[@]}" -gt 0 ]; then + die "Required check(s) failed: ${required_failed[*]}. The build was not published." +fi +log "All required checks passed" diff --git a/test/run-local.sh b/test/run-local.sh new file mode 100755 index 0000000..46b1cae --- /dev/null +++ b/test/run-local.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Exercise the pipeline scripts end to end against a stub DealSage, without +# Gitea. Builds a throwaway app repo, then runs three scenarios: +# 1. auto policy: checks pass (one advisory fails), build, upload, promote → live +# 2. a required check fails: nothing is allocated +# 3. manual policy: built and kept on the branch channel, not promoted +# Usage: templates/ci-actions/test/run-local.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +SCRIPTS="$HERE/../pipeline/scripts" +PORT="${PORT:-8765}" +WORK="$(mktemp -d)" +trap 'kill ${STUB_PID:-} 2>/dev/null || true; rm -rf "$WORK"' EXIT +pass=0; fail=0 +check() { if eval "$2"; then echo " ok $1"; pass=$((pass+1)); else echo " FAIL $1"; fail=$((fail+1)); fi; } + +make_app() { # dir required_check_cmd + local d="$1" reqcmd="$2" + mkdir -p "$d/.dealsage" "$d/.gitea/workflows" "$d/src/lib" + cat > "$d/package.json" <ok' > dist/index.html && echo 'console.log(1)' > dist/assets/app.js", + "typecheck": "$reqcmd" } } +JSON + cat > "$d/.dealsage/app.json" < "$d/.gitea/workflows/publish.yml" + : > "$d/src/lib/dealsage.ts"; : > "$d/src/Root.tsx"; : > "$d/src/main.tsx" + printf '# Agent guide\n\n\n' > "$d/AGENTS.md" + git -C "$d" init -q -b feature/x && git -C "$d" -c user.name=t -c user.email=t@t add -A \ + && git -C "$d" -c user.name=t -c user.email=t@t commit -qm "Add fixture app" +} + +run_pipeline() { # dir -> runs manifest, checks, build, publish; echoes the failing stage or "ok" + local d="$1" ghenv ghout ghsum + ghenv="$(mktemp)"; ghout="$(mktemp)"; ghsum="$(mktemp)" + export GITHUB_ENV="$ghenv" GITHUB_OUTPUT="$ghout" GITHUB_STEP_SUMMARY="$ghsum" + export GITHUB_SHA="$(git -C "$d" rev-parse HEAD)" GITHUB_REF_NAME="feature/x" GITHUB_ACTOR="tester" + export DEALSAGE_FUNCTIONS_URL="http://127.0.0.1:$PORT" PUBLISH_TOKEN="test-publish-token" + export SUPABASE_ANON_KEY="anon" SUPABASE_URL="https://example.supabase.co" CUSTOM_APPS_WORKER_URL="http://127.0.0.1:$PORT" + export MANIFEST_PATH=".dealsage/app.json" + unset APP_ID MANIFEST_JSON CHECKS_JSON CHECKS_RESULT_JSON + local stage + for stage in manifest run-checks build publish; do + ( cd "$d" && bash "$SCRIPTS/$stage.sh" ) > "$WORK/$stage.log" 2>&1 || { echo "$stage"; cat "$ghenv" > "$WORK/last.env"; return 0; } + # composite steps share state through GITHUB_ENV (heredoc form); load it. + eval "$(python3 - "$ghenv" <<'PY' +import re,sys,shlex +txt=open(sys.argv[1]).read() +for m in re.finditer(r'^(\w+)<<__DS_EOF__\n(.*?)\n__DS_EOF__$', txt, re.S|re.M): + print(f"export {m.group(1)}={shlex.quote(m.group(2))}") +PY +)" + done + cp "$ghout" "$WORK/last.out"; cp "$ghsum" "$WORK/last.summary" + echo ok +} + +start_stub() { # policy + : > "$WORK/requests.jsonl" + STUB_LOG="$WORK/requests.jsonl" STUB_POLICY="$1" python3 "$HERE/stub_server.py" "$PORT" & STUB_PID=$! + for _ in $(seq 1 30); do curl -s -o /dev/null "http://127.0.0.1:$PORT/" && break; sleep 0.1; done +} +stop_stub() { kill "$STUB_PID" 2>/dev/null || true; wait "$STUB_PID" 2>/dev/null || true; } +req() { jq -c "$1" "$WORK/requests.jsonl"; } + +echo "Scenario 1: auto policy" +make_app "$WORK/app1" "true"; start_stub auto +r="$(run_pipeline "$WORK/app1")" +check "pipeline completed" "[ '$r' = ok ]" +check "four checks recorded (typecheck, lint, conventions, build)" "[ \"\$(req 'select(.path|endswith(\"/record-app-check\"))' | wc -l)\" = 4 ]" +check "advisory lint failure recorded as failed" "req 'select(.body.checkId==\"lint\") | .body.status' | grep -q '\"failed\"'" +check "conventions passed" "req 'select(.body.checkId==\"conventions\") | .body.status' | grep -q '\"passed\"'" +check "check payload carries commit, branch and log tail" "req 'select(.body.checkId==\"typecheck\") | .body | (.commitSha|length)==40 and .branch==\"feature/x\" and (.logTail|length)>0' | grep -q true" +check "allocate sent branch, message, author, manifest and check results" "req 'select(.body.action==\"allocate\") | .body | .branch==\"feature/x\" and .commitMessage==\"Add fixture app\" and .author==\"t\" and (.manifest.dependencies.objectTypes|length)==1 and (.checks|length)==3' | grep -q true" +check "two files uploaded through the Worker with the upload token" "[ \"\$(req 'select(.method==\"PUT\" and .auth==\"Bearer upl-token\")' | wc -l)\" = 2 ]" +check "promote requested after upload" "req 'select(.body.action==\"promote\") | .body.version' | grep -q 7" +check "outputs: version=7 promoted=true" "grep -q '^version=7$' '$WORK/last.out' && grep -q '^promoted=true$' '$WORK/last.out'" +stop_stub + +echo "Scenario 2: required check fails" +make_app "$WORK/app2" "echo 'type error' && exit 1"; start_stub auto +r="$(run_pipeline "$WORK/app2")" +check "stopped at run-checks" "[ '$r' = run-checks ]" +check "failure still recorded" "req 'select(.body.checkId==\"typecheck\") | .body.status' | grep -q '\"failed\"'" +check "nothing allocated" "! req 'select(.body.action==\"allocate\")' | grep -q ." +check "advisory and platform checks still ran" "[ \"\$(req 'select(.path|endswith(\"/record-app-check\"))' | wc -l)\" = 3 ]" +stop_stub + +echo "Scenario 3: manual policy" +make_app "$WORK/app3" "true"; start_stub manual +r="$(run_pipeline "$WORK/app3")" +check "pipeline completed" "[ '$r' = ok ]" +check "outputs: promoted=false, channel=branch/feature/x" "grep -q '^promoted=false$' '$WORK/last.out' && grep -q '^channel=branch/feature/x$' '$WORK/last.out'" +check "summary mentions the channel" "grep -q 'channel' '$WORK/last.summary'" +stop_stub + +echo; echo "$pass passed, $fail failed" +[ "$fail" = 0 ] diff --git a/test/stub_server.py b/test/stub_server.py new file mode 100755 index 0000000..c0faf44 --- /dev/null +++ b/test/stub_server.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Stub of the DealSage endpoints the pipeline talks to, for local tests. + +Records every request as JSON lines in the file named by STUB_LOG, and answers +like the real functions would: + POST /record-app-version allocate -> version 7 + upload token + promote -> promoted per STUB_POLICY (auto|manual) + fail -> ok + POST /record-app-check -> {"id": "..."} + PUT /__upload/// -> 204 +""" +import json, os, sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +LOG = os.environ.get("STUB_LOG", "/tmp/stub-requests.jsonl") +POLICY = os.environ.get("STUB_POLICY", "auto") +seq = {"n": 0} + +class H(BaseHTTPRequestHandler): + def log_message(self, *a): # quiet + pass + + def _record(self, body): + with open(LOG, "a") as f: + f.write(json.dumps({"method": self.command, "path": self.path, + "auth": self.headers.get("Authorization"), "body": body}) + "\n") + + def _json(self, code, obj): + data = json.dumps(obj).encode() + self.send_response(code); self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))); self.end_headers(); self.wfile.write(data) + + def do_POST(self): + n = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(n).decode() if n else "" + try: body = json.loads(raw) if raw else {} + except json.JSONDecodeError: body = {"_raw": raw} + self._record(body) + if self.headers.get("Authorization") != "Bearer test-publish-token": + return self._json(401, {"error": "bad token"}) + if self.path.endswith("/record-app-version"): + a = body.get("action") + if a == "allocate": + return self._json(200, {"version": 7, "r2Prefix": f"apps/{body.get('appId')}/7/", "uploadToken": "upl-token"}) + if a == "promote": + if POLICY == "auto": + return self._json(200, {"version": 7, "currentVersion": 7, "promoted": True, "channel": "live"}) + return self._json(200, {"version": 7, "currentVersion": 6, "promoted": False, "channel": "branch/feature/x"}) + if a == "fail": + return self._json(200, {"ok": True}) + return self._json(400, {"error": "bad action"}) + if self.path.endswith("/record-app-check"): + seq["n"] += 1 + return self._json(200, {"id": f"chk-{seq['n']}"}) + self._json(404, {"error": "no route"}) + + def do_PUT(self): + n = int(self.headers.get("content-length") or 0) + self.rfile.read(n) + self._record({"bytes": n}) + if not self.path.startswith("/__upload/"): + return self._json(404, {"error": "no route"}) + if self.headers.get("Authorization") != "Bearer upl-token": + return self._json(401, {"error": "bad upload token"}) + self.send_response(204); self.end_headers() + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8765 + HTTPServer(("127.0.0.1", port), H).serve_forever()