CtrlK
BlogDocsLog inGet started
Tessl Logo

jbaruch/coding-policy

General-purpose coding policy for Baruch's AI agents

Quality

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide

SecuritybySnyk

Low

Low-risk findings worth noting

Overview
Quality
Evals
Security
Files

test_verify_moderation_cleared.shskills/release/tests/

#!/usr/bin/env bash
# Outcome-based tests for verify-moderation-cleared.sh.
#
# Covers the behaviors the script promises:
#   1. Cleared immediately — first `tessl api` returns moderationStatus
#      "pass"; script emits {"ok":true,...}, exits 0, never sleeps.
#   2. Cleared via the boolean — moderationPassed true (status absent/other)
#      also counts as cleared.
#   3. Deferred clear — first calls return "pending", a later call returns
#      "pass"; script polls (sleeps), then exits 0.
#   4. Blocked by status — a terminal block state (e.g. "flagged") exits 1
#      immediately (fail loud), without waiting out the budget.
#   5. Blocked by moderationError — a non-null error exits 1 immediately.
#   6. Budget exhausted — every call "pending"; script exits 1 with a
#      budget diagnostic and bounded total sleep.
#   7. Arg-count + empty-arg validation — exit 2 with usage/diagnostic.
#   8. Env-var validation — non-positive / inverted bounds exit 2.
#   9. tessl failure — non-zero `tessl api` exits 2 with a diagnostic.
#  10. Unparseable body — no moderation fields exits 2.
#
# Approach mirrors test_resolve_publish_run.sh: source the script (the
# main() guard prevents auto-run when sourced) and override `tessl` and
# `sleep` as shell functions. `main` runs the tessl call inside a command
# substitution (`body=$(...)`), so per-call state lives in tempfiles, not
# shell vars — a counter file the mock increments and a queue directory it
# indexes into (file `N` = the Nth call's response; missing N falls back to
# the highest-numbered file so "always pending" needs a single fixture).
#
# Run: bash skills/release/tests/test_verify_moderation_cleared.sh
# Exit 0 on all-pass; non-zero with a per-test diagnostic on failure.

set -uo pipefail

SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/verify-moderation-cleared.sh"
[[ -x "$SCRIPT" ]] || { echo "fatal: verify-moderation-cleared.sh not executable at $SCRIPT" >&2; exit 2; }

# Shrink the backoff so the budget-exhausted test stays fast. Exit codes
# and call counts are what's asserted, not wall-clock timing.
export VERIFY_MODERATION_BASE_DELAY_SEC=1
export VERIFY_MODERATION_MAX_DELAY_SEC=2
export VERIFY_MODERATION_BUDGET_SEC=4

# shellcheck disable=SC1090
source "$SCRIPT"
set +e

FAIL_COUNT=0
PASS_COUNT=0

TMPDIR_TEST=$(mktemp -d -t verify-mod-test.XXXXXX)
# Named handler ending `return 0`, not a bare `trap 'rm -rf ...'`: the
# EXIT trap's final command status becomes the process's exit status, so
# a failed cleanup would turn an all-green run non-zero and flake CI
# (rules/error-handling.md Shell Error Handling).
cleanup_tmp() {
  if [[ -n "${TMPDIR_TEST:-}" ]]; then
    if ! rm -rf "$TMPDIR_TEST"; then
      echo "warning: could not remove temp dir ${TMPDIR_TEST} — remove it by hand" >&2
    fi
  fi
  return 0
}
trap cleanup_tmp EXIT
export MOCK_COUNT_FILE="$TMPDIR_TEST/count"
export MOCK_CALLS_FILE="$TMPDIR_TEST/calls"
export MOCK_SLEEP_FILE="$TMPDIR_TEST/sleeps"
export MOCK_QUEUE_DIR="$TMPDIR_TEST/queue"

# Mocks. A response file beginning with __FAIL__ makes `tessl` echo the
# rest to stderr and return non-zero (simulates a registry/tool failure).
tessl() {
  local n resp
  # Absent count file is the expected first-call state (=> 0); a present but
  # unreadable file is a harness fault that must not read as 0
  # (rules/error-handling.md — expected non-result vs tool failure).
  if [[ -f "$MOCK_COUNT_FILE" ]]; then
    n=$(cat "$MOCK_COUNT_FILE") || { echo "test harness: cannot read $MOCK_COUNT_FILE" >&2; exit 2; }
  else
    n=0
  fi
  n=$(( n + 1 ))
  echo "$n" > "$MOCK_COUNT_FILE"
  echo "tessl $*" >> "$MOCK_CALLS_FILE"
  if [[ -f "$MOCK_QUEUE_DIR/$n" ]]; then
    resp="$MOCK_QUEUE_DIR/$n"
  else
    # Sticky last: highest-numbered fixture file. queue() names files by
    # integer call number, so a numeric max over the basenames (glob, no
    # `ls` parsing) picks the latest queued response.
    local highest=0 f
    resp=""
    for f in "$MOCK_QUEUE_DIR"/*; do
      [[ -f "$f" ]] || continue
      if (( ${f##*/} > highest )); then highest=${f##*/}; resp="$f"; fi
    done
  fi
  [[ -z "$resp" || ! -f "$resp" ]] && { echo "mock: no queued response for call $n" >&2; return 1; }
  if [[ "$(head -1 "$resp")" == "__FAIL__" ]]; then
    tail -n +2 "$resp" >&2
    return 1
  fi
  cat "$resp"
}
sleep() { echo "$*" >> "$MOCK_SLEEP_FILE"; }
# The suite sources the script rather than spawning it, so the export is
# belt-and-braces. Warn rather than suppress: silently losing it would
# surface as a mock that never fires, which reads as a script bug
# (rules/error-handling.md Shell Error Handling).
if ! export -f tessl sleep 2>/dev/null; then
  echo "warning: 'export -f' unavailable — mocks apply to this shell only" >&2
fi

reset_mocks() {
  rm -rf "$MOCK_QUEUE_DIR"; mkdir -p "$MOCK_QUEUE_DIR"
  : > "$MOCK_CALLS_FILE"; : > "$MOCK_SLEEP_FILE"; echo 0 > "$MOCK_COUNT_FILE"
}

# Build a moderation JSON body. $1=status (empty to omit), $2=passed
# (empty to omit), $3=error (empty => null).
body_json() {
  local status="$1" passed="$2" err="$3"
  local attrs=""
  [[ -n "$status" ]] && attrs="\"moderationStatus\":\"$status\""
  [[ -n "$passed" ]] && attrs="${attrs:+$attrs,}\"moderationPassed\":$passed"
  if [[ -n "$err" ]]; then attrs="${attrs:+$attrs,}\"moderationError\":\"$err\""; else attrs="${attrs:+$attrs,}\"moderationError\":null"; fi
  printf '{"data":{"attributes":{%s}}}' "$attrs"
}

queue() { printf '%s' "$2" > "$MOCK_QUEUE_DIR/$1"; }

count_calls() { wc -l < "$MOCK_CALLS_FILE" | tr -d ' '; }
count_sleeps() { [[ -s "$MOCK_SLEEP_FILE" ]] || { echo 0; return; }; wc -l < "$MOCK_SLEEP_FILE" | tr -d ' '; }

pass() { PASS_COUNT=$(( PASS_COUNT + 1 )); echo "  PASS: $1"; }
fail() { FAIL_COUNT=$(( FAIL_COUNT + 1 )); echo "  FAIL: $1" >&2; }

# --- 1. Cleared immediately ---
reset_mocks
queue 1 "$(body_json pass true '')"
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 0 ]] && echo "$out" | jq -e '.ok == true' >/dev/null 2>&1 && [[ "$(count_sleeps)" == "0" ]]; then
  pass "cleared immediately: exit 0, ok=true, no sleep"
else
  fail "cleared immediately (rc=$rc out=$out sleeps=$(count_sleeps))"
fi

# --- 2. Cleared via boolean only ---
reset_mocks
queue 1 "$(body_json '' true '')"
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 0 ]] && echo "$out" | jq -e '.ok == true' >/dev/null 2>&1; then
  pass "cleared via moderationPassed=true with no status"
else
  fail "cleared via boolean (rc=$rc out=$out)"
fi

# --- 3. Deferred clear ---
reset_mocks
queue 1 "$(body_json pending '' '')"
queue 2 "$(body_json pending '' '')"
queue 3 "$(body_json pass true '')"
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 0 ]] && echo "$out" | jq -e '.ok == true and .attempts == 3' >/dev/null 2>&1 && [[ "$(count_sleeps)" -ge 2 ]]; then
  pass "deferred clear: polled to pass on 3rd check, exit 0"
else
  fail "deferred clear (rc=$rc out=$out calls=$(count_calls) sleeps=$(count_sleeps))"
fi

# --- 4. Blocked by status (fail loud, immediate) ---
reset_mocks
queue 1 "$(body_json flagged false '')"
queue 2 "$(body_json pass true '')"   # would clear if it kept polling; it must NOT
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 1 ]] && echo "$out" | jq -e '.ok == false' >/dev/null 2>&1 && [[ "$(count_calls)" == "1" ]] && [[ "$(count_sleeps)" == "0" ]]; then
  pass "blocked status: exit 1 immediately, no further polling"
else
  fail "blocked status (rc=$rc out=$out calls=$(count_calls))"
fi

# --- 5. Blocked by moderationError ---
reset_mocks
queue 1 "$(body_json pending '' 'policy violation: embedded credentials')"
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 1 ]] && echo "$out" | jq -e '.ok == false' >/dev/null 2>&1 && [[ "$(count_calls)" == "1" ]]; then
  pass "blocked by non-null moderationError: exit 1 immediately"
else
  fail "blocked by error (rc=$rc out=$out calls=$(count_calls))"
fi

# --- 5b. Blocked by moderationError ALONE (no status, no passed) ---
# Regression guard: the parse guard must treat moderationError as a
# parsed field, so an error-only body is the rc 1 blocked finding, not
# an rc 2 "could not parse" tool error.
reset_mocks
queue 1 '{"data":{"attributes":{"moderationError":"policy violation"}}}'
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 1 ]] && echo "$out" | jq -e '.ok == false' >/dev/null 2>&1; then
  pass "blocked by moderationError alone: exit 1 (not exit 2 parse error)"
else
  fail "error-only body (rc=$rc out=$out)"
fi

# --- 6. Budget exhausted ---
reset_mocks
queue 1 "$(body_json pending '' '')"   # sticky-last => every call is pending
out=$(main acme widget 1.2.3); rc=$?
if [[ $rc -eq 1 ]] && echo "$out" | jq -e '.ok == false' >/dev/null 2>&1 && echo "$out" | grep -q "budget"; then
  pass "budget exhausted: exit 1 with budget diagnostic"
else
  fail "budget exhausted (rc=$rc out=$out)"
fi

# --- 7. Arg validation ---
# Wrap in subshells: main's `exit` would otherwise terminate this harness
# when not run inside a command substitution.
reset_mocks
( main acme widget ) >/dev/null 2>&1; rc=$?
if [[ $rc -eq 2 ]]; then pass "missing arg: exit 2"; else fail "missing arg (rc=$rc)"; fi
( main acme widget "" ) >/dev/null 2>&1; rc=$?
if [[ $rc -eq 2 ]]; then pass "empty version arg: exit 2"; else fail "empty version (rc=$rc)"; fi

# --- 8. Env-var validation ---
# main() reads the resolved globals (BASE_DELAY_SEC, MAX_DELAY_SEC,
# BUDGET_SEC) set once at source time — it does NOT re-read the
# VERIFY_MODERATION_* env vars per call. So the override must set those
# globals, command-scoped to the main() call (a bash env-prefix on a
# function makes the var visible inside it and restores it after). The
# subshell only isolates main's `exit`.
reset_mocks
( BASE_DELAY_SEC=0 main acme widget 1.2.3 ) >/dev/null 2>&1; rc=$?
if [[ $rc -eq 2 ]]; then pass "base delay 0: exit 2"; else fail "base delay 0 (rc=$rc)"; fi
( BASE_DELAY_SEC=10 MAX_DELAY_SEC=5 main acme widget 1.2.3 ) >/dev/null 2>&1; rc=$?
if [[ $rc -eq 2 ]]; then pass "base > max: exit 2"; else fail "base > max (rc=$rc)"; fi
( BASE_DELAY_SEC=100 MAX_DELAY_SEC=100 BUDGET_SEC=10 main acme widget 1.2.3 ) >/dev/null 2>&1; rc=$?
if [[ $rc -eq 2 ]]; then pass "base > budget: exit 2"; else fail "base > budget (rc=$rc)"; fi

# --- 9. tessl failure ---
reset_mocks
queue 1 "$(printf '__FAIL__\n500 Internal Server Error')"
out=$(main acme widget 1.2.3 2>/dev/null); rc=$?
if [[ $rc -eq 2 ]]; then pass "tessl api failure: exit 2"; else fail "tessl failure (rc=$rc out=$out)"; fi

# --- 10. Unparseable body ---
reset_mocks
queue 1 '{"data":{"attributes":{}}}'
out=$(main acme widget 1.2.3 2>/dev/null); rc=$?
if [[ $rc -eq 2 ]]; then pass "no moderation fields: exit 2"; else fail "unparseable (rc=$rc out=$out)"; fi

# --- 11. Body that is not JSON at all ---
# The field extractions used to carry `2>/dev/null || true`, which read a
# non-JSON body (a proxy error page, an HTML 502) as "no moderation fields
# present". The script still exited 2 — this was never a vacuous pass — but
# it reported the wrong cause: "the registry response shape may have
# changed; update verify-moderation-cleared.sh's jq paths", sending the
# operator to edit a parse that is working fine against a registry that is
# returning 502s. Per rules/error-handling.md Actionable Messages, the
# message must name what to do; naming the wrong thing is worse than
# terse, because it is confidently wrong.
reset_mocks
queue 1 '<html><head><title>502 Bad Gateway</title></head></html>'
err=$( { main acme widget 1.2.3 >/dev/null; } 2>&1 ); rc=$?
if [[ $rc -eq 2 ]] && [[ "$err" == *"not valid JSON"* ]]; then
  pass "non-JSON body: exit 2 naming the real fault"
else
  fail "non-JSON body (rc=$rc err=$err)"
fi

# --- 12. Truncated JSON body ---
# jq fails outright here rather than returning empty; the old suppression
# turned that failure into the same misdirected "update the jq paths"
# message as case 11.
reset_mocks
queue 1 '{"data":{"attributes":{"moderationStatus":'
err=$( { main acme widget 1.2.3 >/dev/null; } 2>&1 ); rc=$?
if [[ $rc -eq 2 ]] && [[ "$err" == *"not valid JSON"* ]]; then
  pass "truncated JSON body: exit 2 naming the real fault"
else
  fail "truncated JSON (rc=$rc err=$err)"
fi

# --- 13/14. Valid JSON, WRONG SHAPE — must reach the shape-changed
# diagnostic, not crash on jq's raw index error. `.data` a string, and
# `.data.attributes` a string (nested), are the two levels a one-step
# `.attributes?` guard misses; only `(.data.attributes.FIELD)?` catches both.
reset_mocks
queue 1 '{"data":"a-string-not-an-object"}'
err=$( { main acme widget 1.2.3 >/dev/null; } 2>&1 ); rc=$?
if [[ $rc -eq 2 ]] && [[ "$err" == *"response shape may have changed"* ]]; then
  pass "wrong shape (.data string): shape-changed diagnostic, not a jq crash"
else
  fail "wrong shape .data string (rc=$rc err=$err)"
fi

reset_mocks
queue 1 '{"data":{"attributes":"not-an-object"}}'
err=$( { main acme widget 1.2.3 >/dev/null; } 2>&1 ); rc=$?
if [[ $rc -eq 2 ]] && [[ "$err" == *"response shape may have changed"* ]]; then
  pass "wrong shape (.data.attributes string): shape-changed diagnostic"
else
  fail "wrong shape .attributes string (rc=$rc err=$err)"
fi

echo
echo "verify-moderation-cleared: ${PASS_COUNT} passed, ${FAIL_COUNT} failed"
[[ $FAIL_COUNT -eq 0 ]]

README.md

tile.json