Regression testing in CI/CD

{
  "image": {"port": 8008, "path": "/chat", "health_path": "/health"},
  "channel_config": {
    "init": null,
    "channel": {
      "type": "http",
      "url": null,
      "method": "POST",
      "request": {"message_field": "message", "session_field": "session_id"},
      "response": {"type": "json", "message_field": "response", "session_field": "session_id"}
    }
  }
}

#!/usr/bin/env bash
# Add this build's image to a long-lived trial as a new candidate, wait for its
# results, and fail when even the top of its confidence interval is under MIN_PASS_RATE.
set -euo pipefail
 
for v in BENCH_API BENCH_ID VERIS_API_KEY TRIAL_NAME IMAGE_REF CANDIDATE_JSON WAIT_MINUTES MIN_PASS_RATE; do
  [ -n "${!v:-}" ] || { echo "Set $v." >&2; exit 1; }
done
[[ $MIN_PASS_RATE =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "MIN_PASS_RATE must be a number from 0 to 100, not \"$MIN_PASS_RATE\"." >&2; exit 1; }
[[ $WAIT_MINUTES =~ ^[0-9]+$ ]] || { echo "WAIT_MINUTES must be a whole number, not \"$WAIT_MINUTES\"." >&2; exit 1; }
 
# One bench API call: prints the response, or the API's error on stderr and fails.
call() {
  local body
  body=$(curl -sS --max-time 60 --fail-with-body \
    -H "Authorization: Bearer $VERIS_API_KEY" -H "Content-Type: application/json" "$@") \
    || { echo "Bench API error: $body" >&2; return 1; }
  printf '%s\n' "$body"
}
get() { call --retry 3 "$@"; }  # reads retry brief outages; writes never retry, so nothing is created twice
send() { call "$@"; }
 
# 1. Find the trial by name, and refuse one whose tasks changed since it started.
trials='[]'; offset=0
while :; do
  page=$(get "$BENCH_API/benches/$BENCH_ID/trials?limit=200&offset=$offset")
  trials=$(jq -c --argjson found "$trials" --arg n "$TRIAL_NAME" '$found + [.items[] | select(.name == $n)]' <<<"$page")
  offset=$((offset + 200))
  [ "$offset" -lt "$(jq .total <<<"$page")" ] || break
done
if [ "$(jq length <<<"$trials")" -gt 1 ]; then
  echo "More than one trial is named \"$TRIAL_NAME\". Use a unique name." >&2; exit 1
fi
trial_id=$(jq -r '.[0].id // empty' <<<"$trials")
if [ -n "$trial_id" ] && jq -e '.[0] | .outdated or (.edited_task_ids + .stale_persona_ids | length > 0)' <<<"$trials" >/dev/null; then
  echo "Trial $trial_id changed under its tasks: $(jq -c '.[0] | {outdated_reasons, edited_task_ids, stale_persona_ids}' <<<"$trials")" >&2
  echo "Start a new series with a new TRIAL_NAME." >&2; exit 1
fi
 
# 2. Check that the bench can pull the image, then register it as a new candidate.
check=$(jq -n --arg r "$IMAGE_REF" '{ref: $r}' | send -X POST "$BENCH_API/registries/check-image" -d @-)
case $(jq -r .status <<<"$check") in
  ok | public) ;;
  *) echo "The bench can't pull $IMAGE_REF: $(jq -r '"\(.status): \(.detail)"' <<<"$check")" >&2; exit 1 ;;
esac
sha=${GITHUB_SHA:-local}
candidate_id=$(jq --arg n "$TRIAL_NAME $(date -u +%F) ${sha:0:7}" --arg r "$IMAGE_REF" '. + {name: $n} | .image.ref = $r' "$CANDIDATE_JSON" \
  | send -X POST "$BENCH_API/benches/$BENCH_ID/candidates" -d @- | jq -r .id)
echo "Candidate $candidate_id ($IMAGE_REF)"
 
# 3. Add it to the trial, or start the trial on the first run.
if [ -n "$trial_id" ]; then
  jq -n --arg c "$candidate_id" '{candidate_ids: [$c]}' \
    | send -X POST "$BENCH_API/trials/$trial_id/candidates" -d @- >/dev/null
else
  trial_id=$(jq -n --arg n "$TRIAL_NAME" --arg c "$candidate_id" '{name: $n, kind: "full", candidate_ids: [$c]}' \
    | send -X POST "$BENCH_API/benches/$BENCH_ID/trials" -d @- | jq -r .id)
fi
echo "Trial $trial_id: https://benchmark.veris.ai/benchmarks/$BENCH_ID?trial=$trial_id"
 
# 4. Wait until none of the candidate's attempts is running or waiting for its grade.
#    Each read also moves the trial forward.
deadline=$(( $(date +%s) + WAIT_MINUTES * 60 ))
while :; do
  row=$(get "$BENCH_API/trials/$trial_id/results" \
    | jq -c --arg c "$candidate_id" '.rollups[] | select(.candidate_id == $c)')
  [ -n "$row" ] || { echo "Trial $trial_id has no results row for candidate $candidate_id." >&2; exit 1; }
  [ "$(jq .pending <<<"$row")" -eq 0 ] && break
  if [ "$(date +%s)" -ge "$deadline" ]; then
    echo "Trial $trial_id still running after $WAIT_MINUTES minutes." >&2; exit 1
  fi
  sleep 60
done
 
# 5. Gate on the candidate's row.
jq '{pass_rate, ci_low, ci_high, n, passes}' <<<"$row"
if jq -e --argjson min "$MIN_PASS_RATE" '.ci_high < $min' <<<"$row" >/dev/null; then
  echo "Pass rate $(jq .pass_rate <<<"$row")% is below $MIN_PASS_RATE% even at the top of its interval." >&2; exit 1
fi

name: Bench regression
 
on:
  workflow_dispatch:
  schedule:
    - cron: "0 6 * * *" # every day at 06:00 UTC
 
permissions:
  contents: read
  packages: write # push the image to GHCR
 
concurrency:
  group: bench-regression
  cancel-in-progress: false
 
jobs:
  bench:
    runs-on: ubuntu-latest
    timeout-minutes: 360
    steps:
      - uses: actions/checkout@v4
 
      - name: Image name # GHCR needs it in lowercase
        run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
 
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
 
      - name: Build and push
        id: build
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          tags: ${{ env.IMAGE }}:${{ github.sha }}
 
      - name: Run on the bench
        env:
          BENCH_API: ${{ vars.BENCH_API }}
          BENCH_ID: ${{ vars.BENCH_ID }}
          MIN_PASS_RATE: ${{ vars.MIN_PASS_RATE }}
          VERIS_API_KEY: ${{ secrets.BENCH_API_KEY }}
          TRIAL_NAME: regression
          IMAGE_REF: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }}
          CANDIDATE_JSON: bench/candidate.json
          WAIT_MINUTES: "300"
        run: bash .github/bench/regression.sh