General-purpose coding policy for Baruch's AI agents
73
91%
Does it follow best practices?
Run evals on this skill
Adds up to 20 points to the overall score
View guide
Low
Low-risk findings worth noting
#!/usr/bin/env bash
# Outcome-based tests for skills/herdr-foreman/resolve-gates.sh.
#
# Real directory trees per case, so the fixtures share no state and run in any
# order (rules/testing-standards.md Independence).
#
# The harness drops `set -e` to aggregate results; every fixture command is
# checked explicitly (rules/error-handling.md aggregate-reporting carve-out).
#
# Covers:
# 1. No declaration -> declared:false, workflows still reported, exit 0.
# 2. A declaration -> its instructions, runners and notes come back.
# 3. A rotted declaration -> the absent paths land in `missing`, not silence.
# 4. Malformed JSON -> exit 2, a repair, never an empty map.
# 5. Wrong schema / keys -> exit 2, named.
# 6. Bad value types -> exit 2, named.
# 7. Workflows -> .yml and .yaml, sorted, relative; absent dir is [].
# 8. Usage / bad checkout -> exit 2, no JSON.
# 9. Unrenderable text -> a control, format, separator, surrogate,
# private-use or unassigned character in a path,
# a note or a workflow filename, or a backtick
# in a path or workflow filename: exit 2. A
# backtick in a note stays accepted.
# 10. Broken install -> foreman/renderable.py missing, lacking
# `offenders`, syntactically invalid or cut off
# mid-identifier: exit 2 naming the reinstall,
# never a traceback.
#
# No case asserts a filename this script recognises, because it recognises none.
# An earlier draft matched a hardcoded list of names, which is the enumerated
# failure #480 retired: the set of filenames meaning "runner" is not enumerable.
set -uo pipefail
PASS=0
FAIL=0
pass() { PASS=$((PASS + 1)); }
fail() { FAIL=$((FAIL + 1)); echo " ✗ FAIL: $*" >&2; }
die() { echo "fatal: $*" >&2; exit 2; }
SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/resolve-gates.sh" || die "cannot resolve the script"
run() { # <checkout>
OUT="$(bash "$SCRIPT" "$1" 2>"$ERRFILE")"
RC=$?
ERRTEXT="$(cat "$ERRFILE")"
}
list() { printf '%s' "$1" | python3 -c 'import json,sys; print(",".join(json.load(sys.stdin)[sys.argv[1]]))' "$2"; }
field() { printf '%s' "$1" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps(eval(sys.argv[1])))' "$2"; }
# An EXIT trap's final status becomes the script's, so cleanup ends on zero.
cleanup() { if [ -n "${TMP:-}" ]; then rm -rf "$TMP"; fi; return 0; }
main() {
TMP="$(mktemp -d "${TMPDIR:-/tmp}/resolve-gates-tests.XXXXXX")" || die "mktemp"
trap cleanup EXIT
ERRFILE="$TMP/err"
echo "▶ an undeclared repo" >&2
mkdir -p "$TMP/bare/.github/workflows" || die "mkdir bare"
printf 'x\n' > "$TMP/bare/.github/workflows/tests.yml" || die "write workflow"
printf 'x\n' > "$TMP/bare/Makefile" || die "write Makefile"
run "$TMP/bare"
# A Makefile sits there and is deliberately NOT reported: nothing guesses.
if [[ $RC -eq 0 ]] && [[ "$(field "$OUT" 'd["declared"]')" == "false" ]] \
&& [[ "$(list "$OUT" runners)" == "" ]] \
&& [[ "$(list "$OUT" workflows)" == ".github/workflows/tests.yml" ]] \
&& [[ "$(field "$OUT" 'd["brief"]')" == '"undeclared"' ]]; then
pass; else fail "undeclared reports workflows and guesses nothing, got RC=$RC OUT=$OUT"; fi
echo "▶ a declared repo" >&2
mkdir -p "$TMP/decl/.herdr" "$TMP/decl/scripts" || die "mkdir decl"
printf 'x\n' > "$TMP/decl/AGENTS.md" || die "write AGENTS.md"
printf 'x\n' > "$TMP/decl/scripts/verify.sh" || die "write verify.sh"
cat > "$TMP/decl/.herdr/gates.json" <<'JSON' || die "write declaration"
{"schema_version": 1, "instructions": ["AGENTS.md"],
"runners": ["scripts/verify.sh"], "notes": "run verify before pushing"}
JSON
run "$TMP/decl"
# `scripts/verify.sh` is a name no hardcoded list would hold. A declaration
# needs none.
if [[ $RC -eq 0 ]] && [[ "$(field "$OUT" 'd["declared"]')" == "true" ]] \
&& [[ "$(list "$OUT" instructions)" == "AGENTS.md" ]] \
&& [[ "$(list "$OUT" runners)" == "scripts/verify.sh" ]] \
&& [[ "$(field "$OUT" 'd["notes"]')" == '"run verify before pushing"' ]]; then
pass; else fail "a declaration is reported as written, got RC=$RC OUT=$OUT"; fi
# The GATES value arrives rendered: the foreman copies it, never builds it.
# The backticks are literal Markdown the brief carries, not a command substitution.
# shellcheck disable=SC2016
expected='"- Instructions: `AGENTS.md`\n- Runners: `scripts/verify.sh`\n- Notes: run verify before pushing"'
if [[ "$(field "$OUT" 'd["brief"]')" == "$expected" ]]; then
pass; else fail "a declared repo's brief is the ready GATES list, got OUT=$OUT"; fi
echo "▶ a declaration that rotted" >&2
mkdir -p "$TMP/rot/.herdr" || die "mkdir rot"
cat > "$TMP/rot/.herdr/gates.json" <<'JSON' || die "write rotted declaration"
{"schema_version": 1, "instructions": ["GONE.md"], "runners": ["scripts/gone.sh"]}
JSON
run "$TMP/rot"
# A rotted path is named in `missing` and never handed to a worker.
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" missing)" == "GONE.md,scripts/gone.sh" ]] \
&& [[ "$(field "$OUT" 'd["brief"]')" == '"undeclared"' ]]; then
pass; else fail "a declared path that is gone lands in missing, got RC=$RC OUT=$OUT"; fi
echo "▶ a declaration that cannot be trusted" >&2
mkdir -p "$TMP/bad/.herdr" || die "mkdir bad"
for broken in '{broken' '{"schema_version": 2}' '{"schema_version": 1, "extra": 1}' \
'{"schema_version": 1, "instructions": "AGENTS.md"}' \
'{"schema_version": 1, "runners": [""]}' \
'{"schema_version": 1, "notes": " "}' \
'{"schema_version": 1, "runners": ["../outside.sh"]}' \
'{"schema_version": 1, "instructions": ["/etc/hosts"]}'; do
printf '%s\n' "$broken" > "$TMP/bad/.herdr/gates.json" || die "write broken declaration"
run "$TMP/bad"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'resolve-gates:'; then
pass; else fail "'$broken' must be a repair, never an empty map, got RC=$RC OUT=$OUT"; fi
done
echo "▶ text the GATES block cannot render" >&2
# Each value lands in every worker's Markdown GATES block: a backtick closes
# the path's code span, a newline injects a line, and a NUL used to crash
# realpath outside the exit-2 contract. JSON escapes carry the characters.
mkdir -p "$TMP/render/.herdr" || die "mkdir render"
# The backticks are literal fixture content, not command substitution.
# shellcheck disable=SC2016
for unsafe in '{"schema_version": 1, "runners": ["scripts/a`b.sh"]}' \
'{"schema_version": 1, "instructions": ["AGENTS.md\n- Runners: `evil.sh`"]}' \
'{"schema_version": 1, "runners": ["scripts/nul\u0000.sh"]}' \
'{"schema_version": 1, "runners": ["scripts/esc\u001b.sh"]}' \
'{"schema_version": 1, "runners": ["scripts/c1\u0085.sh"]}' \
'{"schema_version": 1, "runners": ["scripts/sur\ud800.sh"]}' \
'{"schema_version": 1, "runners": ["scripts/ls\u2028.sh"]}' \
'{"schema_version": 1, "instructions": ["ps\u2029.md"]}' \
'{"schema_version": 1, "runners": ["scripts/bidi\u202e.sh"]}' \
'{"schema_version": 1, "notes": "one\u2028- injected"}' \
'{"schema_version": 1, "notes": "one\u2029two"}' \
'{"schema_version": 1, "notes": "line one\n- injected"}'; do
printf '%s\n' "$unsafe" > "$TMP/render/.herdr/gates.json" || die "write unsafe declaration"
run "$TMP/render"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'cannot render intact' \
&& ! printf '%s' "$ERRTEXT" | grep -q 'Traceback'; then
pass; else fail "'$unsafe' must be refused before rendering, got RC=$RC OUT=$OUT ERR=$ERRTEXT"; fi
done
# A backtick is harmless in notes, which render as plain text, not a code span.
# shellcheck disable=SC2016
printf '%s\n' '{"schema_version": 1, "notes": "run `make check` first"}' \
> "$TMP/render/.herdr/gates.json" || die "write notes declaration"
run "$TMP/render"
if [[ $RC -eq 0 ]]; then
pass; else fail "a backtick in notes stays accepted, got RC=$RC ERR=$ERRTEXT"; fi
# A workflow filename is rendered the same way, so the same refusal holds.
mkdir -p "$TMP/wfbad/.github/workflows" || die "mkdir wfbad"
# shellcheck disable=SC2016
printf 'x\n' > "$TMP/wfbad/.github/workflows/a\`b.yml" || die "write backtick workflow"
run "$TMP/wfbad"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'cannot render intact'; then
pass; else fail "a backtick workflow filename must be refused, got RC=$RC OUT=$OUT"; fi
mkdir -p "$TMP/wfls/.github/workflows" || die "mkdir wfls"
printf 'x\n' > "$TMP/wfls/.github/workflows/$(printf 'a\342\200\250b.yml')" || die "write separator workflow"
run "$TMP/wfls"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'cannot render intact'; then
pass; else fail "a line-separator workflow filename must be refused, got RC=$RC OUT=$OUT"; fi
mkdir -p "$TMP/wfnl/.github/workflows" || die "mkdir wfnl"
printf 'x\n' > "$TMP/wfnl/.github/workflows/$(printf 'a\nb.yml')" || die "write newline workflow"
run "$TMP/wfnl"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'cannot render intact'; then
pass; else fail "a newline workflow filename must be refused, got RC=$RC OUT=$OUT"; fi
# A symlinked workflow is not a workflow file, as it was under find -type f.
mkdir -p "$TMP/wflink/.github/workflows" || die "mkdir wflink"
printf 'x\n' > "$TMP/wflink/.github/workflows/real.yml" || die "write real workflow"
ln -s real.yml "$TMP/wflink/.github/workflows/link.yml" || die "link workflow"
run "$TMP/wflink"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == ".github/workflows/real.yml" ]]; then
pass; else fail "a symlinked workflow is not listed, got RC=$RC OUT=$OUT"; fi
# Nor is a symlinked workflows directory, whose files live elsewhere.
mkdir -p "$TMP/wfdirlink/.github" "$TMP/outside-workflows" || die "mkdir wfdirlink"
printf 'x\n' > "$TMP/outside-workflows/foreign.yml" || die "write foreign workflow"
ln -s "$TMP/outside-workflows" "$TMP/wfdirlink/.github/workflows" || die "link workflows dir"
run "$TMP/wfdirlink"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == "" ]]; then
pass; else fail "a symlinked workflows directory lists nothing, got RC=$RC OUT=$OUT"; fi
# Nor is a workflows directory reached through a symlinked `.github`.
mkdir -p "$TMP/ghlink" "$TMP/outside-github/workflows" || die "mkdir ghlink"
printf 'x\n' > "$TMP/outside-github/workflows/foreign.yml" || die "write foreign workflow"
ln -s "$TMP/outside-github" "$TMP/ghlink/.github" || die "link .github"
run "$TMP/ghlink"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == "" ]]; then
pass; else fail "a symlinked .github lists nothing, got RC=$RC OUT=$OUT"; fi
# A workflows directory that exists but cannot be probed is a tool error,
# never an empty list. Root can probe anything, so the case needs a user.
if [[ "$(id -u)" -ne 0 ]]; then
mkdir -p "$TMP/wfperm/.github/workflows" || die "mkdir wfperm"
chmod 000 "$TMP/wfperm/.github" || die "chmod wfperm"
run "$TMP/wfperm"
chmod 755 "$TMP/wfperm/.github" || die "restore wfperm"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'cannot inspect'; then
pass; else fail "an unprobeable workflows directory is a tool error, got RC=$RC OUT=$OUT ERR=$ERRTEXT"; fi
fi
# A `.github` that is a file holds no workflows directory: absent, not an error.
mkdir -p "$TMP/ghfile" || die "mkdir ghfile"
printf 'x\n' > "$TMP/ghfile/.github" || die "write .github file"
run "$TMP/ghfile"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == "" ]]; then
pass; else fail "a .github file means no workflows, got RC=$RC OUT=$OUT ERR=$ERRTEXT"; fi
# Each refusal names the repair for the field it refused.
printf '%s\n' '{"schema_version": 1, "notes": "one\ntwo"}' > "$TMP/render/.herdr/gates.json" \
|| die "write notes declaration"
run "$TMP/render"
if [[ $RC -eq 2 ]] && printf '%s' "$ERRTEXT" | grep -q 'Rewrite notes as one line of printable text'; then
pass; else fail "a notes refusal names the notes repair, got ERR=$ERRTEXT"; fi
printf '%s\n' '{"schema_version": 1, "runners": ["a\tb.sh"]}' > "$TMP/render/.herdr/gates.json" \
|| die "write runner declaration"
run "$TMP/render"
if [[ $RC -eq 2 ]] && printf '%s' "$ERRTEXT" | grep -q 'Remove those characters from the declared path'; then
pass; else fail "a path refusal names the path repair, got ERR=$ERRTEXT"; fi
echo "▶ workflows and usage" >&2
mkdir -p "$TMP/wf/.github/workflows" || die "mkdir wf"
printf 'x\n' > "$TMP/wf/.github/workflows/b.yaml" || die "write b"
printf 'x\n' > "$TMP/wf/.github/workflows/a.yml" || die "write a"
mkdir -p "$TMP/wf/.github/workflows/nested.yml" || die "mkdir nested"
run "$TMP/wf"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == ".github/workflows/a.yml,.github/workflows/b.yaml" ]]; then
pass; else fail "workflows are files, sorted and relative, got $(list "$OUT" workflows)"; fi
mkdir -p "$TMP/noghub" || die "mkdir noghub"
run "$TMP/noghub"
if [[ $RC -eq 0 ]] && [[ "$(list "$OUT" workflows)" == "" ]] \
&& ! printf '%s' "$OUT" | grep -q '\*'; then
pass; else fail "an absent .github is an empty list, never a literal glob, got RC=$RC OUT=$OUT"; fi
OUT="$(bash "$SCRIPT" 2>"$ERRFILE")"
RC=$?
ERRTEXT="$(cat "$ERRFILE")"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'usage'; then
pass; else fail "no argument is a usage error, got RC=$RC OUT=$OUT"; fi
run "$TMP/does-not-exist"
if [[ $RC -eq 2 && -z "$OUT" ]] && printf '%s' "$ERRTEXT" | grep -q 'not a directory'; then
pass; else fail "an absent checkout is a tool error, got RC=$RC OUT=$OUT"; fi
echo "▶ a broken install" >&2
# A copy of the script in a skill directory whose shared check is missing,
# then one whose module lacks the function: exit 2 naming the reinstall.
# A truncated module (syntax error) and one cut off mid-identifier (name
# error) are the same damaged install.
mkdir -p "$TMP/nomod" "$TMP/plain" || die "mkdir broken install"
cp "$SCRIPT" "$TMP/nomod/resolve-gates.sh" || die "copy script (nomod)"
local broken
for broken in badmod syntaxmod namemod; do
mkdir -p "$TMP/$broken/foreman" || die "mkdir $broken"
cp "$SCRIPT" "$TMP/$broken/resolve-gates.sh" || die "copy script ($broken)"
: > "$TMP/$broken/foreman/__init__.py" || die "write $broken package"
done
printf 'UNRENDERABLE_CATEGORIES = frozenset()\n' > "$TMP/badmod/foreman/renderable.py" || die "write badmod module"
printf 'UNRENDERABLE_CATEGORIES = frozenset({"Cc",\n' > "$TMP/syntaxmod/foreman/renderable.py" || die "write syntaxmod module"
printf 'import unicodedata\nUNRENDERABLE_CATEGORIES = frozen\n' > "$TMP/namemod/foreman/renderable.py" || die "write namemod module"
for broken in nomod badmod syntaxmod namemod; do
OUT="$(bash "$TMP/$broken/resolve-gates.sh" "$TMP/plain" 2>"$ERRFILE")"
RC=$?
ERRTEXT="$(cat "$ERRFILE")"
if [[ $RC -eq 2 && -z "$OUT" ]] \
&& printf '%s' "$ERRTEXT" | grep -q '^resolve-gates: cannot load the shared renderable-text check' \
&& printf '%s' "$ERRTEXT" | grep -q 'tessl install jbaruch/coding-policy' \
&& ! printf '%s' "$ERRTEXT" | grep -q 'Traceback'; then
pass; else fail "an unimportable shared check ($broken) is exit 2 naming the reinstall, got RC=$RC OUT=$OUT ERR=$ERRTEXT"; fi
done
echo "─────────────────────────────────────────────" >&2
if [[ $FAIL -gt 0 ]]; then echo "FAILED: ${FAIL} failed, ${PASS} passed" >&2; exit 1; fi
echo "PASSED: all ${PASS} checks" >&2
}
[[ "${BASH_SOURCE[0]}" == "${0}" ]] && main "$@".tessl-plugin
hooks
rules
skills
adopt-fork-pr
herdr-foreman
classify
foreman
references
templates
tests
herdr-standup
migrate-to-plugin
onboard-repo
release
references
tests