DealSage template sync
197 lines
13 KiB
Bash
Executable File
197 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Exercise the pipeline scripts end to end against a stub DealSage, without
|
|
# Gitea. Builds a throwaway app repo, then runs these scenarios:
|
|
# 1. auto policy: checks pass (one advisory fails), build, upload, promote → live
|
|
# 2. a required check fails: nothing is allocated
|
|
# 3. manual policy: built and kept on the branch channel, not promoted
|
|
# 4. installing dependencies fails: the run closes as build_failed
|
|
# 5. a step dies without closing the run (a missing variable)
|
|
# 6. Set up Node.js (a stock action) fails
|
|
# Every scenario also checks the step list the pipeline reports for the build page.
|
|
# Usage: templates/ci-actions/test/run-local.sh
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
SCRIPTS="$HERE/../pipeline/scripts"
|
|
PORT="${PORT:-8765}"
|
|
WORK="$(mktemp -d)"
|
|
FIXTURE_APP_ID="11111111-2222-4333-8444-555555555555"
|
|
trap 'kill ${STUB_PID:-} 2>/dev/null || true; [ -n "${KEEP_WORK:-}" ] || rm -rf "$WORK"' EXIT
|
|
pass=0; fail=0
|
|
check() { if eval "$2"; then echo " ok $1"; pass=$((pass+1)); else echo " FAIL $1"; fail=$((fail+1)); fi; }
|
|
|
|
make_app() { # dir required_check_cmd
|
|
local d="$1" reqcmd="$2"
|
|
mkdir -p "$d/.dealsage" "$d/.gitea/workflows" "$d/src/lib"
|
|
cat > "$d/package.json" <<JSON
|
|
{ "name": "fixture-app", "private": true, "scripts": {
|
|
"build": "mkdir -p dist/assets && echo '<html>ok</html>' > dist/index.html && echo 'console.log(1)' > dist/assets/app.js",
|
|
"typecheck": "$reqcmd" } }
|
|
JSON
|
|
cat > "$d/.dealsage/app.json" <<JSON
|
|
{ "appId": "$FIXTURE_APP_ID",
|
|
"checks": [
|
|
{ "id": "typecheck", "name": "TypeScript", "run": "npm run --silent typecheck" },
|
|
{ "id": "lint", "name": "Lint", "run": "echo 'lint problem' && exit 3", "required": false },
|
|
{ "id": "conventions", "uses": "conventions" } ],
|
|
"dependencies": { "objectTypes": [ { "apiName": "contract_agreement", "schemaVersion": 2 } ] } }
|
|
JSON
|
|
printf 'name: dealsage\non: push\njobs:\n p:\n steps:\n - uses: https://git.example/dealsage-platform/ci-actions/pipeline@v1\n' > "$d/.gitea/workflows/publish.yml"
|
|
: > "$d/src/lib/dealsage.ts"; : > "$d/src/Root.tsx"; : > "$d/src/main.tsx"
|
|
printf '# Agent guide\n<!-- dealsage:generated:start -->\n<!-- dealsage:generated:end -->\n' > "$d/AGENTS.md"
|
|
git -C "$d" init -q -b feature/x && git -C "$d" -c user.name=t -c user.email=t@t add -A \
|
|
&& git -C "$d" -c user.name=t -c user.email=t@t commit -qm "Add fixture app"
|
|
}
|
|
|
|
load_env() { # GITHUB_ENV file -> exports; composite steps share state through it (heredoc form)
|
|
eval "$(python3 - "$1" <<'PY'
|
|
import re,sys,shlex
|
|
txt=open(sys.argv[1]).read()
|
|
for m in re.finditer(r'^(\w+)<<__DS_EOF__\n(.*?)\n__DS_EOF__$', txt, re.S|re.M):
|
|
print(f"export {m.group(1)}={shlex.quote(m.group(2))}")
|
|
PY
|
|
)"
|
|
}
|
|
|
|
# run_pipeline DIR -> runs the action's steps in order, stopping at the first
|
|
# that fails, then finish-steps.sh (if: always()); echoes the step that failed
|
|
# or "ok". Knobs, as environment: INSTALL_COMMAND, UNSET_ENV (names to unset),
|
|
# FAIL_NODE=1 (Set up Node.js fails; nothing of ours runs in that step).
|
|
run_pipeline() {
|
|
local d="$1" ghenv ghout ghsum
|
|
ghenv="$(mktemp)"; ghout="$(mktemp)"; ghsum="$(mktemp)"
|
|
export GITHUB_ENV="$ghenv" GITHUB_OUTPUT="$ghout" GITHUB_STEP_SUMMARY="$ghsum"
|
|
export RUNNER_TEMP; RUNNER_TEMP="$(mktemp -d)"
|
|
export GITHUB_SHA="$(git -C "$d" rev-parse HEAD)" GITHUB_REF_NAME="feature/x" GITHUB_ACTOR="tester"
|
|
export DEALSAGE_FUNCTIONS_URL="http://127.0.0.1:$PORT" PUBLISH_TOKEN="test-publish-token"
|
|
export SUPABASE_ANON_KEY="anon" SUPABASE_URL="https://example.supabase.co" CUSTOM_APPS_WORKER_URL="http://127.0.0.1:$PORT"
|
|
export MANIFEST_PATH=".dealsage/app.json" INSTALL_COMMAND="${INSTALL_COMMAND:-echo 'up to date, audited 0 packages'}"
|
|
unset APP_ID MANIFEST_JSON CHECKS_JSON CHECKS_RESULT_JSON BUILD_ID
|
|
local v; for v in ${UNSET_ENV:-}; do unset "$v"; done
|
|
local stage failed=""
|
|
for stage in ensure-tools manifest start-build node install run-checks build publish; do
|
|
if [ "$stage" = node ]; then
|
|
if [ "${FAIL_NODE:-0}" = 1 ]; then failed=node; break; fi
|
|
continue
|
|
fi
|
|
( cd "$d" && bash "$SCRIPTS/$stage.sh" ) > "$WORK/$stage.log" 2>&1 || failed="$stage"
|
|
load_env "$ghenv"
|
|
[ -z "$failed" ] || break
|
|
done
|
|
if ! ( cd "$d" && bash "$SCRIPTS/finish-steps.sh" ) > "$WORK/finish-steps.log" 2>&1; then
|
|
failed="${failed:-finish-steps}"
|
|
fi
|
|
cp "$ghout" "$WORK/last.out"; cp "$ghsum" "$WORK/last.summary"
|
|
echo "${failed:-ok}"
|
|
}
|
|
|
|
start_stub() { # policy
|
|
: > "$WORK/requests.jsonl"
|
|
STUB_LOG="$WORK/requests.jsonl" STUB_POLICY="$1" python3 "$HERE/stub_server.py" "$PORT" & STUB_PID=$!
|
|
for _ in $(seq 1 30); do curl -s -o /dev/null "http://127.0.0.1:$PORT/" && break; sleep 0.1; done
|
|
}
|
|
stop_stub() { kill "$STUB_PID" 2>/dev/null || true; wait "$STUB_PID" 2>/dev/null || true; }
|
|
req() { jq -c "$1" "$WORK/requests.jsonl"; }
|
|
|
|
# The step lists the pipeline posted (record-app-build, action "steps").
|
|
first_steps_post() { jq -cs 'map(select(.body.action=="steps"))[0].body' "$WORK/requests.jsonl"; }
|
|
last_steps() { jq -c 'select(.body.action=="steps") | .body.steps' "$WORK/requests.jsonl" | tail -n 1; }
|
|
statuses_of() { jq -r 'map(.id + "=" + .status) | join(" ")'; }
|
|
statuses() { last_steps | statuses_of; }
|
|
step() { last_steps | jq -r --arg id "$1" "map(select(.id == \$id))[0] | $2"; }
|
|
finishes() { req 'select(.body.action=="finish")' | wc -l | tr -d ' '; }
|
|
finish_is() { req 'select(.body.action=="finish") | .body' | jq -e --arg o "$1" --arg r "$2" '.outcome == $o and .failureReason == $r' >/dev/null; }
|
|
ALL_PASSED="tooling=passed manifest=passed record=passed node=passed install=passed checks=passed build=passed publish=passed"
|
|
|
|
echo "Scenario 1: auto policy"
|
|
make_app "$WORK/app1" "true"; start_stub auto
|
|
r="$(run_pipeline "$WORK/app1")"
|
|
check "pipeline completed" "[ '$r' = ok ]"
|
|
check "four checks recorded (typecheck, lint, conventions, build)" "[ \"\$(req 'select(.path|endswith(\"/record-app-check\"))' | wc -l)\" = 4 ]"
|
|
check "advisory lint failure recorded as failed" "req 'select(.body.checkId==\"lint\") | .body.status' | grep -q '\"failed\"'"
|
|
check "conventions passed" "req 'select(.body.checkId==\"conventions\") | .body.status' | grep -q '\"passed\"'"
|
|
check "check payload carries commit, branch and log tail" "req 'select(.body.checkId==\"typecheck\") | .body | (.commitSha|length)==40 and .branch==\"feature/x\" and (.logTail|length)>0' | grep -q true"
|
|
check "allocate sent branch, message, author, manifest and check results" "req 'select(.body.action==\"allocate\") | .body | .branch==\"feature/x\" and .commitMessage==\"Add fixture app\" and .author==\"t\" and (.manifest.dependencies.objectTypes|length)==1 and (.checks|length)==3' | grep -q true"
|
|
check "two files uploaded through the Worker with the upload token" "[ \"\$(req 'select(.method==\"PUT\" and .auth==\"Bearer upl-token\")' | wc -l)\" = 2 ]"
|
|
check "promote requested after upload" "req 'select(.body.action==\"promote\") | .body.version' | grep -q 7"
|
|
check "outputs: version=7 promoted=true" "grep -q '^version=7$' '$WORK/last.out' && grep -q '^promoted=true$' '$WORK/last.out'"
|
|
check "build start recorded with commit, branch and author" "req 'select(.body.action==\"start\") | .body | (.commitSha|length)==40 and .branch==\"feature/x\" and .author==\"t\"' | grep -q true"
|
|
check "allocate carries the build id" "req 'select(.body.action==\"allocate\") | .body.buildId' | grep -q '\"bld-1\"'"
|
|
check "build closed as built, promoted, v7" "req 'select(.body.action==\"finish\") | .body | .buildId==\"bld-1\" and .outcome==\"built\" and .promoted==true and .version==7' | grep -q true"
|
|
check "steps first sent with the build record, for this build and app" \
|
|
'first_steps_post | jq -e --arg app "$FIXTURE_APP_ID" ".buildId == \"bld-1\" and .appId == \$app" >/dev/null'
|
|
check "the first post has the steps so far and Set up Node.js running" \
|
|
'[ "$(first_steps_post | jq -c .steps | statuses_of)" = "tooling=passed manifest=passed record=passed node=running install=pending checks=pending build=pending publish=pending" ]'
|
|
check "every step passed, in action.yml order" '[ "$(statuses)" = "$ALL_PASSED" ]'
|
|
check "each step has a start, a finish and a duration" \
|
|
'last_steps | jq -e "all(.[]; (.startedAt|type) == \"string\" and (.finishedAt|type) == \"string\" and (.durationMs|type) == \"number\" and .durationMs >= 0)" >/dev/null'
|
|
check "internal fields stay out of the payload" '! req "select(.body.action==\"steps\") | .body.steps[] | select(has(\"startedMs\"))" | grep -q .'
|
|
check "summaries: manifest, checks (the advisory failure named), build, publish" \
|
|
'[ "$(step manifest .summary)" = "3 checks declared" ] && [ "$(step checks .summary)" = "2 of 3 passed (Lint failed, not required)" ] && [ "$(step build .summary)" = "2 files in dist/" ] && [ "$(step publish .summary)" = "v7 is live" ]'
|
|
check "no log tails on a green run" '[ "$(last_steps | jq "map(select(has(\"logTail\"))) | length")" = 0 ]'
|
|
check "the finalizer did not close the run again" '[ "$(finishes)" = 1 ]'
|
|
stop_stub
|
|
|
|
echo "Scenario 2: required check fails"
|
|
make_app "$WORK/app2" "echo 'type error' && exit 1"; start_stub auto
|
|
r="$(run_pipeline "$WORK/app2")"
|
|
check "stopped at run-checks" "[ '$r' = run-checks ]"
|
|
check "failure still recorded" "req 'select(.body.checkId==\"typecheck\") | .body.status' | grep -q '\"failed\"'"
|
|
check "nothing allocated" "! req 'select(.body.action==\"allocate\")' | grep -q ."
|
|
check "advisory and platform checks still ran" "[ \"\$(req 'select(.path|endswith(\"/record-app-check\"))' | wc -l)\" = 3 ]"
|
|
check "build closed as checks_failed with the reason" "req 'select(.body.action==\"finish\") | .body | .outcome==\"checks_failed\" and (.failureReason|test(\"TypeScript\"))' | grep -q true"
|
|
check "steps: Checks failed, Build and Publish skipped" \
|
|
'[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=passed checks=failed build=skipped publish=skipped" ]'
|
|
check "Checks names the required check that failed" 'step checks .summary | grep -q "Required check(s) failed: TypeScript"'
|
|
check "skipped steps carry no timings" '[ "$(step build ".startedAt // \"none\"")" = none ]'
|
|
check "the finalizer did not close the run again" '[ "$(finishes)" = 1 ]'
|
|
stop_stub
|
|
|
|
echo "Scenario 3: manual policy"
|
|
make_app "$WORK/app3" "true"; start_stub manual
|
|
r="$(run_pipeline "$WORK/app3")"
|
|
check "pipeline completed" "[ '$r' = ok ]"
|
|
check "outputs: promoted=false, channel=branch/feature/x" "grep -q '^promoted=false$' '$WORK/last.out' && grep -q '^channel=branch/feature/x$' '$WORK/last.out'"
|
|
check "summary mentions the channel" "grep -q 'channel' '$WORK/last.summary'"
|
|
check "build closed as built but not promoted" "req 'select(.body.action==\"finish\") | .body | .outcome==\"built\" and .promoted==false and .version==7' | grep -q true"
|
|
check "every step passed" '[ "$(statuses)" = "$ALL_PASSED" ]'
|
|
check "Publish says where the build was kept" '[ "$(step publish .summary)" = "v7 kept on branch/feature/x; live stays at v6" ]'
|
|
stop_stub
|
|
|
|
echo "Scenario 4: installing dependencies fails"
|
|
make_app "$WORK/app4" "true"; start_stub auto
|
|
r="$(INSTALL_COMMAND="echo 'npm error 404 Not Found - GET https://registry.npmjs.org/left-pad-nope' >&2; exit 1" run_pipeline "$WORK/app4")"
|
|
check "stopped at install" '[ "$r" = install ]'
|
|
check "build closed as build_failed, saying why" \
|
|
'finish_is build_failed "Installing dependencies failed (exit code 1); nothing was published."'
|
|
check "steps: Install dependencies failed, the rest skipped" \
|
|
'[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=failed checks=skipped build=skipped publish=skipped" ]'
|
|
check "Install dependencies carries the tail of its output" 'step install .logTail | grep -q "npm error 404"'
|
|
check "no checks ran and nothing was allocated" \
|
|
'! req "select((.path|endswith(\"/record-app-check\")) or .body.action == \"allocate\")" | grep -q .'
|
|
check "the run was closed once" '[ "$(finishes)" = 1 ]'
|
|
stop_stub
|
|
|
|
echo "Scenario 5: a step dies without closing the run"
|
|
make_app "$WORK/app5" "true"; start_stub auto
|
|
r="$(UNSET_ENV="SUPABASE_URL" run_pipeline "$WORK/app5")"
|
|
check "stopped at build" '[ "$r" = build ]'
|
|
check "the finalizer closed the run as failed, naming the step and the cause" \
|
|
'finish_is failed "Build: SUPABASE_URL is not set"'
|
|
check "steps: Build failed, Publish skipped" \
|
|
'[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=passed install=passed checks=passed build=failed publish=skipped" ]'
|
|
check "the run was closed once" '[ "$(finishes)" = 1 ]'
|
|
stop_stub
|
|
|
|
echo "Scenario 6: Set up Node.js fails"
|
|
make_app "$WORK/app6" "true"; start_stub auto
|
|
r="$(FAIL_NODE=1 run_pipeline "$WORK/app6")"
|
|
check "stopped at node" '[ "$r" = node ]'
|
|
check "the finalizer failed the step that never finished and closed the run" \
|
|
'finish_is failed "Set up Node.js did not finish."'
|
|
check "steps: Set up Node.js failed with a duration, the rest skipped" \
|
|
'[ "$(statuses)" = "tooling=passed manifest=passed record=passed node=failed install=skipped checks=skipped build=skipped publish=skipped" ] && [ "$(step node ".durationMs | type")" = number ]'
|
|
stop_stub
|
|
|
|
echo; echo "$pass passed, $fail failed"
|
|
[ "$fail" = 0 ]
|