CtrlK
BlogDocsLog inGet started
Tessl Logo

jbaruch/coding-policy

General-purpose coding policy for Baruch's AI agents

73

Quality

91%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide

SecuritybySnyk

Low

Low-risk findings worth noting

Overview
Quality
Evals
Security
Files

test_runnable.pyskills/herdr-foreman/tests/

"""Every hint that tells the reader to run a subcommand names the runnable launcher (#532)."""

import argparse
import ast
import os
import re
import shlex
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from foreman import runnable
from foreman.cli import build_parser

PACKAGE = Path(__file__).resolve().parents[1] / "foreman"
#: Verbs that put a one-word subcommand in a command position.
IMPERATIVE_VERBS = "|".join(("[Rr]un", "[Rr]e-?run", "[Rr]etry", "[Rr]eload", "[Uu]se", "with", "[Ii]nspect",
                             "save", "[Vv]alidate", "[Ff]inish", "through"))


def subcommands():
    parser = build_parser()
    action = next(item for item in parser._actions if isinstance(item, argparse._SubParsersAction))
    return sorted(action.choices, key=len, reverse=True)


def _docstrings(tree):
    """The string nodes that are docstrings, which describe rather than direct."""
    found = set()
    for node in ast.walk(tree):
        if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) and node.body:
            first = node.body[0]
            if isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str):
                found.add(id(first.value))
    return found


def _help_strings(tree):
    """The string nodes inside argparse `help=` values, read from inside the CLI they document."""
    found = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.keyword) and node.arg == "help":
            found.update(id(inner) for inner in ast.walk(node.value))
    return found


#: Calls whose string arguments name a subcommand as data, never as a hint to the reader.
NAMING_CALLS = frozenset({"runnable.command", "command", "sub.add_parser"})


def _named_as_data(tree, names):
    """String nodes that are subcommand names as data: a rendering argument, or a bare name (a dispatch key)."""
    found = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Call) and ast.unparse(node.func) in NAMING_CALLS:
            found.update(id(inner) for inner in ast.walk(node))
        elif isinstance(node, ast.Constant) and node.value in names:
            found.add(id(node))
    return found


def string_literals(source, names=()):
    """Every directive string literal in one module: (line, text).

    Docstrings, argparse help text and subcommand names used as data are excluded.
    """
    tree = ast.parse(source)
    skipped = _docstrings(tree) | _help_strings(tree) | _named_as_data(tree, set(names))
    return [(node.lineno, node.value) for node in ast.walk(tree)
            if isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in skipped]


def bare_hints(text, names):
    """The subcommand references in `text` that do not run as written.

    A hyphenated name is never English, so any bare occurrence is a reference;
    only a message opening with its own command's name (`start-judge requires
    ...`) and the resume template's `{tl}` launcher placeholder are rendered
    forms. A one-word name (`plan`, `state`) counts only in a command position:
    backticked, after `foreman `, or after a verb that tells the reader to run it.
    A bare `foreman --<flag>` (#590) never runs as written either.
    """
    alternation = "|".join(re.escape(name) for name in names)
    hyphenated = "|".join(re.escape(name) for name in names if "-" in name)
    end = r"(?![\w-])"
    patterns = [
        r"`(?:foreman )?(?:{}){}".format(alternation, end),
        r"\bforeman (?:{}){}".format(alternation, end),
        r"(?<![\w-])foreman --?[a-z][\w-]*",
        r"\b(?:{}) (?:{}){}".format(IMPERATIVE_VERBS, alternation, end),
    ]
    if hyphenated:
        patterns.append(r"(?<!^)(?<![\w./`-])(?<!\{{tl\}} )(?<!foreman )(?:{})(?![\w/-]|\.\w)".format(hyphenated))
    found = []
    for pattern in patterns:
        found.extend(match.group(0) for match in re.finditer(pattern, text) if match.group(0) not in found)
    return found


class BareHintTest(unittest.TestCase):
    def test_the_detector_flags_each_bare_shape_and_passes_the_rendered_one(self):
        names = ["measure", "supervision-bind", "state"]
        self.assertEqual(bare_hints("run `foreman measure` first", names), ["`foreman measure", "foreman measure"])
        self.assertEqual(bare_hints("run supervision-bind from the pane", names),
                         ["run supervision-bind", "supervision-bind"])
        self.assertEqual(bare_hints("inspect foreman state", names), ["foreman state"])
        self.assertEqual(bare_hints("re-run `measure`", names), ["`measure"])
        self.assertEqual(bare_hints("run `{}` first", names), [])
        self.assertEqual(bare_hints("the foreman's state file", names), [])
        self.assertEqual(bare_hints("Retry state for the owner", names), ["Retry state"])
        self.assertEqual(bare_hints("the handoff needs supervision-bind first", names),
                         ["supervision-bind"])
        self.assertEqual(bare_hints("supervision-bind requires a pane", names), [])
        self.assertEqual(bare_hints("run `{tl} supervision-bind {flags}`", names), [])
        self.assertEqual(bare_hints("the .supervision-bind.json lock", names), [])
        self.assertEqual(bare_hints("the owner needs supervision-bind.", names), ["supervision-bind"])
        self.assertEqual(bare_hints("Unknown supervision command; run foreman --help.", names), ["foreman --help"])
        self.assertEqual(bare_hints("run `foreman -h` first", names), ["foreman -h"])
        self.assertEqual(bare_hints("run `bash /x/foreman.sh --help`", names), [])

    def test_no_package_string_names_a_subcommand_that_does_not_run_as_written(self):
        names = subcommands()
        self.assertIn("measure", names)
        offenders = []
        for path in sorted(PACKAGE.glob("*.py")):
            for line, text in string_literals(path.read_text(encoding="utf-8"), names):
                for hint in bare_hints(text, names):
                    offenders.append("{}:{}: {}".format(path.name, line, hint))
        self.assertEqual(offenders, [], "render these through foreman.runnable.command()")


class CommandTest(unittest.TestCase):
    def test_a_task_identity_in_a_hint_is_one_shell_word(self):
        from foreman.cli import _dispatched_seat_briefs
        from foreman.errors import UsageError
        with self.assertRaises(UsageError) as caught:
            _dispatched_seat_briefs({"task_context": {"task": "other"}}, {}, [], "repo task 322; rm -rf x")
        found = re.search(r"`(bash [^`]+)`", caught.exception.message)
        self.assertIsNotNone(found, caught.exception.message)
        assert found is not None  # narrows the Optional for pyright; the assertion above reports the message
        self.assertEqual(shlex.split(found.group(1))[-2:], ["--task", "repo task 322; rm -rf x"])

    def test_command_names_the_quoted_launcher_beside_the_package(self):
        with patch.object(runnable, "launcher", return_value="/opt/my plugins/foreman.sh"):
            self.assertEqual(runnable.command("measure --state F"),
                             "bash '/opt/my plugins/foreman.sh' measure --state F")

    def test_the_launcher_is_the_skill_script(self):
        self.assertEqual(Path(runnable.launcher()), PACKAGE.parent / "foreman.sh")
        self.assertTrue(Path(runnable.launcher()).is_file())

    def test_a_rendered_command_runs_as_written(self):
        with tempfile.TemporaryDirectory(prefix="foreman-runnable-") as root:
            env = {key: value for key, value in os.environ.items() if not key.startswith(("HERDR", "FOREMAN"))}
            env.update(HOME=root, XDG_STATE_HOME=root, XDG_CONFIG_HOME=root)
            for tail in ("measure --help", "supervision-bind --help", "close-task --help"):
                with self.subTest(tail=tail):
                    result = subprocess.run(shlex.split(runnable.command(tail)), capture_output=True, text=True,
                                            env=env, cwd=root, check=False)
                    self.assertEqual(result.returncode, 0, result.stderr)
                    self.assertIn("usage: foreman " + tail.split()[0], result.stdout)

    def test_the_unknown_supervision_fallback_names_a_runnable_help(self):
        from foreman import supervision_runtime
        from foreman.errors import UsageError
        args = argparse.Namespace(command="supervision-bogus", now=None, record=None)
        with self.assertRaises(UsageError) as caught:
            supervision_runtime.run_command(args, "unused-state.json", "2026-01-01T00:00:00Z")
        found = re.search(r"`(bash [^`]+)`", caught.exception.message)
        self.assertIsNotNone(found, caught.exception.message)
        assert found is not None  # narrows the Optional for pyright; the assertion above reports the message
        with tempfile.TemporaryDirectory(prefix="foreman-runnable-") as root:
            env = {key: value for key, value in os.environ.items() if not key.startswith(("HERDR", "FOREMAN"))}
            env.update(HOME=root, XDG_STATE_HOME=root, XDG_CONFIG_HOME=root)
            result = subprocess.run(shlex.split(found.group(1)), capture_output=True, text=True,
                                    env=env, cwd=root, check=False)
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertIn("usage: foreman", result.stdout)


if __name__ == "__main__":
    unittest.main()

skills

herdr-foreman

tests

__init__.py

fakes.py

test_assign.py

test_attention.py

test_billing.py

test_bounded_run.sh

test_capabilities.py

test_capability_routing.py

test_chronology.py

test_churn.py

test_classify.sh

test_claude_native.py

test_cli.py

test_compose_briefs.sh

test_composer.py

test_composition.py

test_config.py

test_continuity_cli.py

test_cost_report.py

test_diagnostics.py

test_engagement.py

test_entrypoints.py

test_foreman_launcher.sh

test_foreman_queue.py

test_foreman_reset.py

test_foreman_seat.py

test_foreman_tier_check.py

test_freeze.py

test_herdr.py

test_historical.py

test_home.py

test_label_workspaces.sh

test_launch.py

test_legacy_recovery.py

test_load_set.py

test_measure.py

test_members.py

test_memory.py

test_oracle.py

test_parsers.py

test_partition.py

test_planner.py

test_probe.py

test_provision_worktree.sh

test_prune_remote_branches.sh

test_prune_report_caches.py

test_prune_result.py

test_prune_worktrees.sh

test_recovery_cli.py

test_recovery.py

test_renderable.py

test_report_contract.py

test_report_delivery.py

test_report_gates.py

test_report_verdict.py

test_resolve_gates.sh

test_resolve_policy_paths.py

test_restoration.py

test_retrospective_runtime.py

test_retrospective.py

test_review_package.py

test_role_clear.py

test_roster.sh

test_round_preflight.sh

test_runnable.py

test_scoring.py

test_script_dir_newline.sh

test_seat_holds.py

test_selection.py

test_skill_invocations.sh

test_slice_scope_parity.py

test_specialist_cli.py

test_specialist_delivery.py

test_specialist_recovery.py

test_specialist_retention.py

test_stale_grok_delivery.py

test_start_judge_worker.py

test_state.py

test_supervision_cli.py

test_supervision_diagnostics.py

test_supervision_gate.py

test_supervision_replay.py

test_supervision.py

test_sweep_worktrees.sh

test_tier_integration.py

test_tiers.py

test_triggers.py

test_typesafe_client.py

test_verdict_gates.py

test_verify_authority.sh

test_wait_report.sh

tier_fixture.py

bounded-run.sh

compose-briefs.sh

config.example.json

foreman-tier-check.py

foreman.sh

label-workspaces.sh

provision-worktree.sh

prune-remote-branches.sh

prune-report-caches.py

prune-worktrees.sh

resolve-gates.sh

resolve-policy-paths.sh

review-package.sh

roster.sh

round-preflight.sh

SKILL.md

start-judge-worker.sh

state-schema.md

sweep-worktrees.sh

verify-authority.sh

wait-report.sh

README.md

tile.json