CtrlK
BlogDocsLog inGet started
Tessl Logo

jbaruch/coding-policy

General-purpose coding policy for Baruch's AI agents

74

Quality

93%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Medium

Suggest reviewing before use

Overview
Quality
Evals
Security
Files

test_tiers.pyskills/herdr-foreman/tests/

"""Policy boundaries for tier choice and launch proof."""

import json
import sys
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from foreman.assign import retained_tier
from foreman.errors import ConfigError, HerdrError, UsageError
from foreman.tiers import TOP_MODELS, MissingTierError, SEATABLE_ROLES, launch_flags, mechanical_allowed, parse_tiers, require_seatable, select_tier as _select_tier, verify_argv, verify_worker_permissions, worker_launch_args


def select_tier(*args, **kwargs):
    result = _select_tier(*args, **kwargs)
    assert result is not None, "the test configured a tier table"
    return result


def agent(kind="claude"):
    model = {"claude": "opus-5", "codex": "gpt-5.6-sol", "grok": "grok-4.6"}[kind]
    return SimpleNamespace(name=kind, kind=kind, tiers=parse_tiers({
        "review": {"model": model, "effort": "high"},
        "hostile_verify": {"model": model, "effort": "high"},
        "build": {"model": "sonnet-5" if kind == "claude" else model, "effort": "high"},
        "fix": {"model": "sonnet-5" if kind == "claude" else model, "effort": "high"},
        "mechanical": ({"model": "claude-haiku-4-5"} if kind == "claude" else {"model": model, "effort": "low"}),
    }, kind))


def mechanical_context():
    """A round licensed cheap by a whole-result oracle, not by assertions."""
    return {"oracle": {"kind": "digest", "value": "a" * 64}}


#: Synthetic session identifiers for resumed-process proof (#382); no real session.
SESSION = "00000000-0000-0000-0000-000000000000"
OTHER_SESSION = "11111111-1111-4111-8111-111111111111"
YOLO = {"claude": "--dangerously-skip-permissions", "codex": "--dangerously-bypass-approvals-and-sandbox",
        "grok": "--always-approve"}


class TierConfigTest(unittest.TestCase):
    def test_claude_all_five_efforts_and_haiku_omission(self):
        for effort in ("low", "medium", "high", "xhigh", "max"):
            self.assertEqual(parse_tiers({"build": {"model": "opus-5", "effort": effort}}, "claude")["build"]["effort"], effort)
        tier = parse_tiers({"mechanical": {"model": "claude-haiku-4-5"}}, "claude")["mechanical"]
        self.assertIsNone(tier["effort"])

    def test_effort_missing_invalid_or_impossible_is_refused(self):
        for entry in ({"model": "opus-5"}, {"model": "opus-5", "effort": "ultra"},
                      {"model": "claude-haiku-4-5", "effort": "low"}):
            with self.subTest(entry=entry), self.assertRaises(ConfigError):
                parse_tiers({"build": entry}, "claude")

    def test_judgment_cannot_be_lowered_by_config(self):
        for round_type in ("review", "critic", "recheck", "hostile_verify", "architect", "reconciliation", "release_adjudication"):
            for entry in ({"model": "sonnet-5", "effort": "high"}, {"model": "opus-5", "effort": "medium"}):
                with self.subTest(round_type=round_type, entry=entry), self.assertRaises(ConfigError):
                    parse_tiers({round_type: entry}, "claude")

    def test_dead_rounds_kinds_and_bad_costs_are_refused(self):
        with self.assertRaises(ConfigError):
            parse_tiers({"review": {"model": "gemini-3.1-pro", "effort": "high"}}, "agy")
        with self.assertRaises(ConfigError):
            parse_tiers({"typo": {"model": "opus-5", "effort": "high"}}, "claude")
        for multiplier in (0, -1, True, "0.5", float("nan"), float("inf")):
            with self.subTest(multiplier=multiplier), self.assertRaises(ConfigError):
                parse_tiers({"build": {"model": "opus-5", "effort": "high", "multiplier": multiplier}}, "claude")


class SelectionTest(unittest.TestCase):
    def test_only_missing_candidate_tiers_have_a_skippable_error(self):
        worker = agent()
        # A table without a `consultation` row (config schema 3) keeps the
        # advisor's old judgment default.
        with self.assertRaisesRegex(MissingTierError, "'architect'"):
            select_tier(worker, "advisor")
        worker.tiers.pop("review")
        with self.assertRaisesRegex(MissingTierError, "review tier"):
            select_tier(worker, "developer", context={"prior_high_miss": True})
        for role, requested, context in (("advisor", "build", {}), ("developer", "build", {"unknown": True})):
            with self.subTest(role=role), self.assertRaises(UsageError) as caught:
                select_tier(worker, role, requested, context)
            self.assertNotIsInstance(caught.exception, MissingTierError)

    def test_consultations_default_below_the_judgment_floor(self):
        # coding-policy#518: an investigator gathers evidence and decides
        # nothing, and an advisor answers a bounded question; neither is a gate.
        worker = agent()
        worker.tiers.update(parse_tiers({
            "consultation": {"model": "sonnet-5", "effort": "high"},
            "architect": {"model": "opus-5", "effort": "high"},
            "reconciliation": {"model": "opus-5", "effort": "high"},
        }, worker.kind))
        for role, escalated in (("advisor", "architect"), ("investigator", "reconciliation")):
            with self.subTest(role=role):
                tier = select_tier(worker, role)
                self.assertEqual((tier["round"], tier["model"], tier["effort"]), ("consultation", "sonnet-5", "high"))
                explicit = select_tier(worker, role, escalated)
                self.assertEqual((explicit["round"], explicit["model"]), (escalated, "opus-5"))
                for forbidden in ("build", "fix", "mechanical", "release_mechanics", "review", "hostile_verify", "recheck"):
                    with self.assertRaisesRegex(UsageError, "cannot perform role"):
                        select_tier(worker, role, forbidden, mechanical_context())

    def test_consultation_and_test_plan_accept_a_non_top_row_and_gates_do_not(self):
        for round_type in ("consultation", "test_plan"):
            with self.subTest(round_type=round_type):
                self.assertEqual(parse_tiers({round_type: {"model": "sonnet-5", "effort": "high"}}, "claude")[round_type]["model"],
                                 "sonnet-5")
        for round_type in ("reconciliation", "architect", "hostile_verify", "recheck"):
            with self.subTest(round_type=round_type), self.assertRaises(ConfigError):
                parse_tiers({round_type: {"model": "sonnet-5", "effort": "high"}}, "claude")

    def test_scarcity_declines_a_consultation_bump_but_never_an_escalated_round(self):
        worker = agent("codex")
        worker.tiers.update(parse_tiers({
            "consultation": {"model": "gpt-5.6-sol", "effort": "medium"},
            "reconciliation": {"model": "gpt-5.6-sol", "effort": "high"},
        }, worker.kind))
        risk = {"risk_flags": ["network", "persistence"]}
        scarce = select_tier(worker, "investigator", context=risk, headroom=1.0)
        self.assertEqual((scarce["effort"], scarce["de_escalated"]), ("medium", True))
        self.assertEqual(select_tier(worker, "investigator", context=risk, headroom=80.0)["effort"], "xhigh")
        escalated = select_tier(worker, "investigator", "reconciliation", context=risk, headroom=1.0)
        self.assertEqual((escalated["effort"], escalated["de_escalated"]), ("xhigh", False))

    def test_recorded_evidence_selects_the_judgment_round_without_an_override(self):
        worker = agent()
        worker.tiers.update(parse_tiers({
            "consultation": {"model": "sonnet-5", "effort": "high"},
            "architect": {"model": "opus-5", "effort": "high"},
            "reconciliation": {"model": "opus-5", "effort": "high"},
        }, worker.kind))
        for role, context, expected in (("investigator", {"diagnosis_input": True}, "reconciliation"),
                                        ("investigator", {"prior_high_miss": True}, "reconciliation"),
                                        ("advisor", {"security_trigger": True}, "architect"),
                                        ("advisor", {"diagnosis_input": True}, "consultation"),
                                        ("investigator", {"security_trigger": True}, "consultation")):
            with self.subTest(role=role, context=context):
                tier = select_tier(worker, role, context=context, headroom=1.0)
                self.assertEqual((tier["round"], tier["de_escalated"]), (expected, False))

    def test_an_explicit_consultation_cannot_override_escalation_evidence(self):
        worker = agent()
        worker.tiers.update(parse_tiers({"consultation": {"model": "sonnet-5", "effort": "high"}}, worker.kind))
        with self.assertRaisesRegex(UsageError, "diagnosis_input requires investigator to settle this"):
            select_tier(worker, "investigator", "consultation", {"diagnosis_input": True})
        for flag in ("diagnosis_input", "security_trigger"):
            with self.subTest(flag=flag), self.assertRaisesRegex(UsageError, "JSON boolean"):
                select_tier(worker, "advisor", context={flag: "yes"})

    def test_a_table_written_before_schema_4_keeps_the_judgment_default(self):
        worker = agent()
        worker.tiers.update(parse_tiers({
            "architect": {"model": "opus-5", "effort": "high"},
            "reconciliation": {"model": "opus-5", "effort": "high"},
        }, worker.kind))
        self.assertEqual(select_tier(worker, "advisor")["round"], "architect")
        self.assertEqual(select_tier(worker, "investigator")["round"], "reconciliation")

    def test_architect_role_default_is_unchanged(self):
        worker = agent()
        worker.tiers.update(parse_tiers({"architect": {"model": "opus-5", "effort": "high"}}, worker.kind))
        self.assertEqual(select_tier(worker, "architect")["round"], "architect")

    def test_release_defaults_to_mechanics_without_an_oracle(self):
        # coding-policy#521: a release worker edits no source; its skill's own
        # gates are the loud failure, and it has no pre-written whole result.
        worker = agent()
        worker.tiers.update(parse_tiers({
            "release_mechanics": {"model": "sonnet-5", "effort": "medium"},
            "release_adjudication": {"model": "opus-5", "effort": "high"},
        }, worker.kind))
        tier = select_tier(worker, "release")
        self.assertEqual((tier["round"], tier["model"]), ("release_mechanics", "sonnet-5"))
        adjudication = select_tier(worker, "release", "release_adjudication")
        self.assertEqual((adjudication["model"], adjudication["effort"]), ("opus-5", "high"))

    def test_a_developer_mechanical_round_still_needs_an_oracle(self):
        with self.assertRaisesRegex(UsageError, "Mechanical eligibility"):
            select_tier(agent(), "developer", "mechanical", {})
        self.assertEqual(select_tier(agent(), "developer", "mechanical", mechanical_context())["round"], "mechanical")

    def test_the_shipped_example_is_the_documented_template(self):
        # coding-policy#518/#519: the example is what operators copy.
        example = json.loads((Path(__file__).resolve().parents[1] / "config.example.json").read_text())
        workers = example.get("worker_kinds", example.get("agents"))
        agents = workers if isinstance(workers, list) else list(workers.values())
        tiered = [worker for worker in agents if worker.get("tiers")]
        self.assertEqual(sorted(worker["kind"] for worker in tiered), ["claude", "codex", "grok"])
        for worker in tiered:
            tiers = parse_tiers(worker["tiers"], worker["kind"])
            with self.subTest(kind=worker["kind"]):
                self.assertEqual(tiers["hostile_verify"]["effort"], "high")
                self.assertIn(tiers["hostile_verify"]["model"], TOP_MODELS[worker["kind"]])
                if worker["kind"] == "claude":
                    self.assertNotIn(tiers["consultation"]["model"], TOP_MODELS["claude"])
                self.assertEqual(tiers["consultation"], {**tiers["test_plan"]})
                probe = SimpleNamespace(name=worker["kind"], kind=worker["kind"], tiers=tiers)
                empty = select_tier(probe, "tester")
                self.assertEqual(empty["effort"], "high")
                risky = select_tier(probe, "tester", context={"risk_flags": ["network", "persistence"]})
                self.assertEqual(risky["effort"], "high" if worker["kind"] == "grok" else "xhigh")

    def test_initial_build_and_late_fix_have_different_models(self):
        worker = agent()
        self.assertEqual(select_tier(worker, "developer")["model"], "sonnet-5")
        self.assertEqual(select_tier(worker, "developer", fix_round=4)["model"], "opus-5")
        for number in (6, 7):
            self.assertEqual(select_tier(worker, "developer", fix_round=number)["model"], "opus-5")
        self.assertEqual(select_tier(worker, "developer", context={"failed_gates": 2})["model"], "opus-5")
        for context, fix_round, requested in (({}, None, "build"), ({"failed_gates": 2}, None, "build"),
                                             ({}, 4, "fix"), ({"prior_high_miss": True}, None, "build")):
            tier = select_tier(worker, "developer", context=context, fix_round=fix_round)
            self.assertEqual(tier["round"], requested)
            self.assertEqual(tier["tier_row"], "review" if context or fix_round else "build")

    def test_recorded_risk_evidence_selects_top_xhigh(self):
        for context in ({"risk_flags": ["network", "persistence"]}, {"input_bytes": 250001}, {"prior_high_miss": True}):
            with self.subTest(context=context):
                tier = select_tier(agent(), "developer", context=context)
                self.assertEqual((tier["model"], tier["effort"]), ("opus-5", "xhigh"))

    def test_a_tester_round_escalates_on_evidence_not_on_its_name(self):
        # coding-policy#477: `hostile_verify` is the tester's DEFAULT round, so
        # escalating on the round name pinned every tester round in the fleet to
        # the top model at xhigh whatever the surface. The round's name is not
        # evidence; its floor is the operator's configured row.
        for kind in ("claude", "codex", "grok"):
            with self.subTest(kind=kind):
                tier = select_tier(agent(kind), "tester")
                self.assertEqual(tier["round"], "hostile_verify")
                self.assertEqual(tier["effort"], "high")

    def test_a_tester_round_still_escalates_when_the_evidence_says_so(self):
        for context in ({"risk_flags": ["network", "persistence"]}, {"input_bytes": 250001},
                        {"prior_high_miss": True}):
            with self.subTest(context=context):
                self.assertEqual(select_tier(agent("codex"), "tester", context=context)["effort"], "xhigh")

    def test_measured_scarcity_declines_a_discretionary_escalation(self):
        # coding-policy#477: headroom could buy a cheaper PAIR in the planner
        # and never a cheaper ROUND, and every branch in this module moved up.
        risk = {"risk_flags": ["network", "persistence"]}
        abundant = select_tier(agent(), "developer", context=risk, headroom=80.0)
        self.assertEqual((abundant["model"], abundant["effort"], abundant["tier_row"]),
                         ("opus-5", "xhigh", "review"))
        self.assertFalse(abundant["de_escalated"])
        scarce = select_tier(agent(), "developer", context=risk, headroom=13.0)
        self.assertEqual((scarce["model"], scarce["effort"], scarce["tier_row"]),
                         ("sonnet-5", "high", "build"))
        self.assertTrue(scarce["de_escalated"])
        self.assertEqual(scarce["pressure_headroom"], 13.0)

    def test_a_round_promoted_to_a_judgment_row_never_de_escalates(self):
        # A build that failed its gates twice, or a fourth fix, runs on the
        # review row; scarcity must not strip that row's escalation.
        risk = {"risk_flags": ["network", "persistence"]}
        for tier in (select_tier(agent(), "developer", "build", {**risk, "failed_gates": 2}, headroom=1.0),
                     select_tier(agent(), "developer", context=risk, fix_round=4, headroom=1.0)):
            with self.subTest(tier_row=tier["tier_row"]):
                self.assertEqual(tier["tier_row"], "review")
                self.assertFalse(tier["de_escalated"])
                self.assertEqual(tier["effort"], "xhigh")

    def test_de_escalated_is_recorded_only_when_the_escalation_would_have_changed_the_tier(self):
        # coding-policy#490: a row already at its top model and effort, or a
        # kind with no effort above its configured one, has nothing to decline.
        risk = {"risk_flags": ["network", "persistence"]}
        top = (("claude", "opus-5", "xhigh"), ("claude", "opus-5", "max"),
               ("codex", "gpt-5.6-sol", "xhigh"), ("grok", "grok-4.6", "high"))
        for kind, model, effort in top:
            worker = SimpleNamespace(name=kind, kind=kind, tiers=parse_tiers(
                {"build": {"model": model, "effort": effort}}, kind))
            with self.subTest(kind=kind, effort=effort):
                tier = select_tier(worker, "developer", context=risk, headroom=1.0)
                self.assertEqual((tier["model"], tier["effort"], tier["tier_row"]), (model, effort, "build"))
                self.assertFalse(tier["de_escalated"])
                self.assertEqual(tier["pressure_headroom"], 1.0)

    def test_a_declined_row_switch_is_recorded_even_where_effort_cannot_rise(self):
        # Grok has no effort above `high`, but a lower build model would still
        # have moved to the review row: that step is real and was declined.
        tier = select_tier(agent("grok"), "developer", "build",
                           {"risk_flags": ["network", "persistence"]}, headroom=1.0)
        self.assertEqual(tier["tier_row"], "build")
        self.assertFalse(tier["de_escalated"])
        worker = SimpleNamespace(name="grok", kind="grok", tiers=parse_tiers({
            "review": {"model": "grok-4.6", "effort": "high"},
            "build": {"model": "grok-code-fast", "effort": "high"}}, "grok"))
        scarce = select_tier(worker, "developer", context={"risk_flags": ["network", "persistence"]}, headroom=1.0)
        self.assertEqual((scarce["model"], scarce["tier_row"], scarce["de_escalated"]), ("grok-code-fast", "build", True))

    def test_scarcity_on_a_row_with_no_review_fallback_declines_rather_than_refuses(self):
        # The step it would have taken needs a review row; under scarcity that
        # step is declined, so the configured row runs without one.
        worker = SimpleNamespace(name="claude", kind="claude", tiers=parse_tiers(
            {"build": {"model": "sonnet-5", "effort": "high"}}, "claude"))
        risk = {"risk_flags": ["network", "persistence"]}
        scarce = select_tier(worker, "developer", context=risk, headroom=1.0)
        self.assertEqual((scarce["model"], scarce["de_escalated"]), ("sonnet-5", True))
        with self.assertRaises(MissingTierError):
            select_tier(worker, "developer", context=risk, headroom=80.0)

    def test_an_overflowing_headroom_reads_as_unmeasured(self):
        tier = select_tier(agent(), "developer", context={"risk_flags": ["network", "persistence"]}, headroom=10 ** 1000)
        self.assertIsNone(tier["pressure_headroom"])
        self.assertFalse(tier["de_escalated"])

    def test_de_escalation_never_goes_below_the_configured_row(self):
        # The operator's table is the floor. What scarcity declines is the step
        # ABOVE it, which is the only part of the selection nobody wrote down.
        scarce = select_tier(agent(), "developer", context={"risk_flags": ["network", "persistence"]},
                             headroom=0.0)
        configured = agent().tiers["build"]
        self.assertEqual((scarce["model"], scarce["effort"]),
                         (configured["model"], configured["effort"]))

    def test_a_judgment_round_never_de_escalates(self):
        risk = {"risk_flags": ["network", "persistence"]}
        for role, row in (("reviewer", "review"), ("tester", "hostile_verify")):
            with self.subTest(role=role):
                tier = select_tier(agent(), role, context=risk, headroom=0.0)
                self.assertEqual(tier["effort"], "xhigh")
                self.assertFalse(tier["de_escalated"])
                self.assertIn(tier["tier_row"], {row, "review"})

    def test_unmeasured_headroom_resolves_exactly_as_an_unmeasured_fleet_did(self):
        # Unknown pressure must not read as scarcity, and must not read as
        # abundance either: it resolves the tier the way it always has.
        risk = {"risk_flags": ["network", "persistence"]}
        baseline = select_tier(agent(), "developer", context=risk)
        for headroom in (None, "13", True, float("nan"), float("inf")):
            with self.subTest(headroom=headroom):
                tier = select_tier(agent(), "developer", context=risk, headroom=headroom)
                self.assertEqual((tier["model"], tier["effort"], tier["tier_row"]),
                                 (baseline["model"], baseline["effort"], baseline["tier_row"]))
                self.assertFalse(tier["de_escalated"])
                self.assertIsNone(tier["pressure_headroom"])

    def test_scarcity_alone_escalates_nothing_and_lowers_nothing(self):
        # Pressure is not itself evidence. With no risk recorded there is no
        # discretionary step to decline, so the configured row runs unchanged.
        quiet = select_tier(agent(), "developer", headroom=1.0)
        self.assertEqual((quiet["model"], quiet["effort"], quiet["tier_row"]),
                         ("sonnet-5", "high", "build"))
        self.assertFalse(quiet["de_escalated"])

    def test_a_recorded_whole_result_oracle_licenses_the_cheap_round(self):
        # coding-policy#480: the retired predicate wanted a task named in a
        # closed list of eight, ten hand-typed booleans and two size caps. It
        # fired zero times in 702 assignments and could not fire.
        context = mechanical_context()
        self.assertTrue(mechanical_allowed(context))
        self.assertEqual(select_tier(agent(), "developer", "mechanical", context)["model"], "claude-haiku-4-5")

    def test_a_declared_oracle_is_checked_rather_than_taken(self):
        with tempfile.TemporaryDirectory() as root:
            written = Path(root) / "exact.patch"
            written.write_text("--- a\n+++ b\n", encoding="utf-8")
            for label, oracle in (
                ("a written patch", {"kind": "patch", "path": str(written)}),
                ("a written fixture", {"kind": "fixture", "path": str(written)}),
            ):
                with self.subTest(label=label):
                    self.assertTrue(mechanical_allowed({"oracle": oracle}))
            for label, oracle in (
                ("a patch nobody wrote", {"kind": "patch", "path": str(Path(root) / "absent.patch")}),
                ("a directory", {"kind": "fixture", "path": root}),
                # A plan replays at apply, possibly from another directory.
                ("a relative path", {"kind": "patch", "path": "exact.patch"}),
            ):
                with self.subTest(label=label):
                    self.assertFalse(mechanical_allowed({"oracle": oracle}))

    def test_a_malformed_oracle_licenses_nothing(self):
        for label, context in (
            ("no oracle", {}),
            ("not an object", {"oracle": True}),
            ("unknown kind", {"oracle": {"kind": "vibes", "value": "a" * 64}}),
            ("unhashable kind", {"oracle": {"kind": ["digest"], "value": "a" * 64}}),
            ("short digest", {"oracle": {"kind": "digest", "value": "a" * 63}}),
            ("uppercase digest", {"oracle": {"kind": "digest", "value": "A" * 64}}),
            ("digest carrying a path", {"oracle": {"kind": "digest", "value": "a" * 64, "path": "/x"}}),
            ("patch carrying a digest", {"oracle": {"kind": "patch", "path": "/x", "value": "a" * 64}}),
            ("stray key", {"oracle": {"kind": "digest", "value": "a" * 64, "extra": 1}}),
        ):
            with self.subTest(label=label):
                self.assertFalse(mechanical_allowed(context))
                with self.assertRaises(UsageError):
                    select_tier(agent(), "developer", "mechanical", context)

    def test_a_context_written_for_the_retired_predicate_names_its_replacement(self):
        for field, value in (("task_kind", "rebase"), ("spec_complete", True), ("files", 2),
                             ("whole_result_oracle", True), ("tool_retries", 0),
                             ("gate_red_after_repair", False)):
            with self.subTest(field=field):
                with self.assertRaisesRegex(UsageError, "Declare an `oracle` instead"):
                    select_tier(agent(), "developer", "mechanical", {field: value})

    def test_reviewer_cannot_claim_mechanical_role(self):
        with self.assertRaises(UsageError):
            select_tier(agent(), "reviewer", "mechanical", mechanical_context())

    def test_missing_tier_never_falls_back_to_untiered_dispatch(self):
        worker = agent()
        del worker.tiers["review"]
        with self.assertRaises(UsageError):
            select_tier(worker, "reviewer")


class ArgvTest(unittest.TestCase):
    def test_invalid_live_options_are_distinct_from_restrictive_modes(self):
        for argv in (["grok", "--always-approve", "--always-approve"],
                     ["grok", "--permission-mode", "unknown"]):
            with self.subTest(argv=argv), self.assertRaisesRegex(HerdrError, "invalid permission/UI") as caught:
                verify_worker_permissions("grok", argv)
            self.assertNotIn("restrictive permission", str(caught.exception))
        with self.assertRaisesRegex(HerdrError, "restrictive permission"):
            verify_worker_permissions("grok", ["grok", "--permission-mode", "plan"])

    def test_legacy_live_permission_proof_accepts_canonical_and_equivalent_modes(self):
        for kind, argv in (
            ("claude", ["claude", "--dangerously-skip-permissions", "--model", "opus-5"]),
            ("claude", ["claude", "--permission-mode", "bypassPermissions"]),
            ("codex", ["codex", "--dangerously-bypass-approvals-and-sandbox", "-c", "model_reasoning_effort=high"]),
            ("codex", ["codex", "-a", "never", "-s", "danger-full-access", "--no-alt-screen"]),
            ("grok", ["grok", "--always-approve", "--no-subagents", "--reasoning-effort", "high"]),
        ):
            with self.subTest(kind=kind, argv=argv):
                self.assertIsNone(verify_worker_permissions(kind, argv))

    def test_legacy_live_permission_proof_rejects_missing_overrides_and_lookalikes(self):
        for kind, argv in (
            ("claude", ["claude", "--model", "opus-5"]),
            ("claude", ["claude", "--model", "--dangerously-skip-permissions"]),
            ("claude", ["claude", "--dangerously-skip-permissions", "--permission-mode", "plan"]),
            ("claude", ["claude", "--permission-mode"]),
            ("codex", ["codex", "-a", "never"]),
            ("codex", ["codex", "-s", "danger-full-access"]),
            ("codex", ["codex", "--dangerously-bypass-approvals-and-sandbox", "-c", 'approval_policy="on-request"']),
            ("grok", ["grok", "--always-approve", "--resume", "session"]),
            ("grok", ["grok", "--always-approve", "--permission-mode", "default"]),
            ("grok", ["wrapper", "grok", "--always-approve"]),
            ("grok", "grok --always-approve"), ("grok", ["grok", 1]),
            ("unknown", []),
        ):
            with self.subTest(kind=kind, argv=argv), self.assertRaisesRegex(HerdrError, "before dispatch"):
                verify_worker_permissions(kind, argv)

    def test_resumed_live_permission_proof_accepts_one_explicit_session_with_yolo(self):
        for kind, argv in (
            # The issue's own reproduction: a resumed Codex worker with the explicit YOLO flag.
            ("codex", ["codex", "resume", SESSION, "--dangerously-bypass-approvals-and-sandbox"]),
            ("codex", ["/opt/homebrew/bin/codex", "resume", "--dangerously-bypass-approvals-and-sandbox", SESSION, "--no-alt-screen"]),
            ("codex", ["codex", "resume", SESSION, "-a", "never", "-s", "danger-full-access", "-c", "model_reasoning_effort=high"]),
            ("claude", ["claude", "--resume", SESSION, "--dangerously-skip-permissions"]),
            ("claude", ["claude", "--dangerously-skip-permissions", "-r", SESSION.upper(), "--model", "opus-5", "--effort", "high"]),
            ("claude", ["claude", "--permission-mode", "bypassPermissions", "--resume", SESSION]),
            ("grok", ["grok", "-r", SESSION, "--always-approve", "--no-subagents"]),
            ("grok", ["grok", "--resume", SESSION, "--permission-mode", "bypassPermissions", "--reasoning-effort", "high"]),
        ):
            with self.subTest(kind=kind, argv=argv):
                self.assertIsNone(verify_worker_permissions(kind, argv))

    def test_resumed_proof_still_requires_yolo_and_refuses_restrictive_modes(self):
        for kind, argv, pattern in (
            ("codex", ["codex", "resume", SESSION], "do not prove YOLO"),
            ("codex", ["codex", "resume", SESSION, "-a", "never"], "do not prove YOLO"),
            ("codex", ["codex", "resume", SESSION, "--full-auto"], "restrictive permission"),
            ("codex", ["codex", "resume", SESSION, "--dangerously-bypass-approvals-and-sandbox", "-s", "read-only"], "restrictive permission"),
            ("codex", ["codex", "resume", SESSION, "--dangerously-bypass-approvals-and-sandbox", "-c", 'approval_policy="on-request"'], "config override"),
            ("claude", ["claude", "--resume", SESSION, "--permission-mode", "plan"], "restrictive permission"),
            ("claude", ["claude", "--resume", SESSION, "--dangerously-skip-permissions", "--permission-mode", "acceptEdits"], "restrictive permission"),
            ("claude", ["claude", "--resume", SESSION, "--model", "opus-5"], "do not prove YOLO"),
            ("grok", ["grok", "--resume", SESSION], "do not prove YOLO"),
            ("grok", ["grok", "--resume", SESSION, "--always-approve", "--always-approve"], "invalid permission/UI"),
        ):
            with self.subTest(kind=kind, argv=argv), self.assertRaisesRegex(HerdrError, pattern) as caught:
                verify_worker_permissions(kind, argv)
            # The diagnostic names both recoveries: a fresh worker, or the same-session restoration.
            self.assertIn("dispatch-recovery.md", str(caught.exception))
            self.assertIn("before dispatch", str(caught.exception))

    def test_ambiguous_or_identity_changing_resume_forms_are_refused(self):
        for kind, extra in (
            ("claude", ["--resume"]), ("claude", ["--resume", "--model", "opus-5"]),
            ("claude", ["--continue"]), ("claude", ["-c"]),
            ("claude", ["--resume", SESSION, "--fork-session"]), ("claude", ["--session-id", SESSION]),
            ("claude", ["--resume", "search term"]), ("claude", ["--resume", SESSION.replace("-", "")]),
            ("claude", ["--resume", SESSION, "--resume", SESSION]), ("claude", ["-r", SESSION, "--resume", OTHER_SESSION]),
            ("claude", ["--resume=" + SESSION]), ("claude", ["--from-pr", "12"]), ("claude", ["--teleport", SESSION]),
            ("codex", ["resume"]), ("codex", ["resume", "--last"]), ("codex", ["resume", "--all", SESSION]),
            ("codex", ["resume", "--include-non-interactive", SESSION]), ("codex", ["resume", "my-session-name"]),
            ("codex", ["resume", SESSION, "fix the bug"]), ("codex", ["resume", SESSION, OTHER_SESSION]),
            ("codex", [SESSION]),
            ("grok", ["--resume"]), ("grok", ["--continue"]), ("grok", ["-c"]),
            ("grok", ["--resume", "Session Title"]), ("grok", ["--resume", SESSION, "--fork-session"]),
            ("grok", ["-s", SESSION]), ("grok", ["--session-id", SESSION]),
            ("grok", ["--resume", SESSION, "--restore-code"]), ("grok", ["--resume", SESSION, "--worktree", "feat"]),
        ):
            argv = [kind] + extra + [YOLO[kind]]
            with self.subTest(kind=kind, argv=argv), self.assertRaisesRegex(HerdrError, "before dispatch"):
                verify_worker_permissions(kind, argv)
        for kind, argv in (
            ("codex", ["codex", "--dangerously-bypass-approvals-and-sandbox", "resume", SESSION]),
            ("codex", ["sh", "-c", "codex resume " + SESSION + " --dangerously-bypass-approvals-and-sandbox"]),
            ("codex", ["wrapper", "codex", "resume", SESSION, "--dangerously-bypass-approvals-and-sandbox"]),
        ):
            with self.subTest(kind=kind, argv=argv), self.assertRaisesRegex(HerdrError, "before dispatch"):
                verify_worker_permissions(kind, argv)

    def test_ambiguous_resume_diagnostics_name_the_explicit_session_requirement(self):
        for kind, argv in (
            ("codex", ["codex", "resume", "--last", "--dangerously-bypass-approvals-and-sandbox"]),
            ("codex", ["codex", "resume", "--dangerously-bypass-approvals-and-sandbox"]),
            ("claude", ["claude", "--continue", "--dangerously-skip-permissions"]),
            ("claude", ["claude", "--resume", "--dangerously-skip-permissions"]),
            ("grok", ["grok", "--resume", "Session Title", "--always-approve"]),
        ):
            with self.subTest(kind=kind, argv=argv), self.assertRaisesRegex(HerdrError, "explicit session UUID"):
                verify_worker_permissions(kind, argv)

    def test_launch_args_and_tier_proof_keep_refusing_resume_forms(self):
        for kind, options in (("claude", ["--resume", SESSION]), ("codex", ["resume", SESSION]), ("grok", ["-r", SESSION])):
            with self.subTest(kind=kind), self.assertRaisesRegex(ConfigError, "resume"):
                worker_launch_args(kind, options)
        tier = select_tier(agent(), "reviewer")
        with self.assertRaises(HerdrError):
            verify_argv("claude", tier, ["claude", "--dangerously-skip-permissions", "--resume", SESSION, "--model", "opus-5", "--effort", "high"],
                        ["--dangerously-skip-permissions"])

    def test_worker_defaults_and_ui_options_use_yolo_for_each_cli(self):
        for kind, options, expected in (
            ("claude", [], ["--dangerously-skip-permissions"]),
            ("codex", ["--no-alt-screen"], ["--dangerously-bypass-approvals-and-sandbox", "--no-alt-screen"]),
            ("grok", ["--no-subagents", "--no-alt-screen"], ["--always-approve", "--no-subagents", "--no-alt-screen"]),
        ):
            with self.subTest(kind=kind):
                self.assertEqual(worker_launch_args(kind, options), expected)
                self.assertEqual(worker_launch_args(kind, expected), expected)

    def test_permission_aliases_normalize_without_duplicate_flags(self):
        cases = (("claude", ["--permission-mode", "bypassPermissions"], "--dangerously-skip-permissions"),
                 ("codex", ["-a", "never", "--sandbox", "danger-full-access"], "--dangerously-bypass-approvals-and-sandbox"),
                 ("grok", ["--permission-mode", "bypassPermissions", "--always-approve"], "--always-approve"))
        for kind, options, expected in cases:
            with self.subTest(kind=kind):
                self.assertEqual(worker_launch_args(kind, options), [expected])

    def test_restrictive_options_cannot_override_yolo(self):
        cases = (("claude", ["--permission-mode", "default"]),
                 ("claude", ["--dangerously-skip-permissions", "--permission-mode", "plan"]),
                 ("codex", ["--full-auto"]), ("codex", ["-a", "on-request"]),
                 ("codex", ["--sandbox", "workspace-write"]),
                 ("codex", ["--dangerously-bypass-approvals-and-sandbox", "-s", "read-only"]),
                 ("grok", ["--permission-mode", "acceptEdits"]))
        for kind, options in cases:
            with self.subTest(kind=kind, options=options), self.assertRaisesRegex(ConfigError, "required YOLO mode"):
                worker_launch_args(kind, options)

    def test_unknown_kind_and_unsupported_arguments_fail(self):
        for kind, options in (("unsupported", []), ("codex", ["--model", "other"]),
                              ("claude", ["--permission-mode"]), ("grok", ["--always-approve", "--always-approve"])):
            with self.subTest(kind=kind), self.assertRaises(ConfigError):
                worker_launch_args(kind, options)

    def test_each_installed_cli_has_exact_flags(self):
        for kind in ("claude", "codex", "grok"):
            tier = select_tier(agent(kind), "reviewer")
            argv = ["/usr/local/bin/" + kind] + launch_flags(kind, tier)
            self.assertEqual(verify_argv(kind, tier, argv)["effort"], "high")

    def test_no_effort_flag_for_haiku(self):
        tier = select_tier(agent(), "developer", "mechanical", mechanical_context())
        self.assertEqual(launch_flags("claude", tier), ["--model", "claude-haiku-4-5"])

    def test_wrong_missing_duplicate_or_transcript_values_do_not_verify(self):
        tier = select_tier(agent(), "reviewer")
        wanted = ["claude", "--model", "opus-5", "--effort", "high"]
        for argv in (" ".join(wanted), [], wanted[:-2], wanted[:-1] + ["xhigh"],
                     wanted + ["--effort", "low"], wanted + ["--resume"],
                     ["echo"] + wanted, ["claude", "--model", "opus-5-extra", "--effort", "high"]):
            with self.subTest(argv=argv), self.assertRaises(HerdrError):
                verify_argv("claude", tier, argv)


class SeatableRoleTest(unittest.TestCase):
    """Only a responsibility whose verification a slice terminates is seated (#434)."""

    def test_a_seat_of_a_seatable_role_passes_through(self):
        for name in ("reviewer#api", "tester#core", "reviewer", "developer", "release"):
            with self.subTest(name=name):
                self.assertEqual(require_seatable(name), name)

    def test_a_seat_whose_slice_cannot_address_it_is_refused(self):
        # A seat is a CLI key. `--brief SEAT=PATH` splits at the first `=`, and
        # `compose-briefs.sh` reads a line-oriented `.roles` key, so a slice
        # carrying a separator, whitespace or nothing at all plans a seat the
        # round cannot address (#434).
        # `=`, `,` and control characters are refused as unaddressable before
        # the slice grammar is reached; both refusals name the same defect.
        for name in ("reviewer#", "reviewer#a b", "reviewer#-lead"):
            with self.subTest(name=name):
                with self.assertRaisesRegex(UsageError, "cannot address it"):
                    require_seatable(name)
        for name in ("reviewer#a=b", "reviewer#a,b", "tester#a\nb"):
            with self.subTest(name=name):
                with self.assertRaisesRegex(UsageError, "cannot be addressed"):
                    require_seatable(name)

    def test_a_role_name_the_cli_keys_cannot_carry_is_refused(self):
        # `plan --roles dev/eloper` emitted an assignment `compose-briefs.sh`
        # then refused, so the same set is refused where the name is read.
        for name in ("dev/eloper", "dev=eloper", "dev,eloper", "dev\neloper"):
            with self.subTest(name=name):
                with self.assertRaisesRegex(UsageError, "cannot be addressed"):
                    require_seatable(name)
        for name in ("developer", "role_v2", "reviewer.v2", "foo..bar", "two words"):
            with self.subTest(name=name):
                self.assertEqual(require_seatable(name), name)

    def test_a_seat_of_a_per_task_counter_role_is_refused(self):
        for name in ("developer#api", "release#core", "judge#api", "lead#x"):
            with self.subTest(name=name):
                with self.assertRaisesRegex(UsageError, "names a seat of"):
                    require_seatable(name)

    def test_the_refusal_names_the_seatable_roles(self):
        with self.assertRaises(UsageError) as caught:
            require_seatable("developer#api")
        for role in SEATABLE_ROLES:
            self.assertIn(role, str(caught.exception))


class RetainedDeEscalationTest(unittest.TestCase):
    """A retained fix round reports the de-escalation its running tier carries (#591)."""

    RISK = {"risk_flags": ["network", "persistence"]}

    def worker(self, kind, fix):
        worker = agent(kind)
        worker.tiers.update(parse_tiers({"fix": fix}, kind))
        return worker

    def planned(self, worker):
        tier = select_tier(worker, "developer", context=self.RISK, fix_round=1, headroom=1.0)
        self.assertTrue(tier["de_escalated"])
        return tier

    def previous(self, model, effort):
        return {"kind": "codex", "model": model, "effort": effort, "multiplier": 3, "effective_multiplier": 3,
                "billing_window": "weekly", "verified": {"model": model, "effort": effort, "source": "launch_argv"}}

    def test_a_kept_effort_that_reaches_the_declined_step_is_not_de_escalated(self):
        worker = self.worker("codex", {"model": "gpt-5.6-sol", "effort": "medium"})
        tier = retained_tier(worker, self.planned(worker), self.previous("gpt-5.6-sol", "xhigh"))
        self.assertEqual((tier["effort"], tier["de_escalated"]), ("xhigh", False))

    def test_a_kept_effort_still_below_the_declined_step_stays_de_escalated(self):
        worker = self.worker("codex", {"model": "gpt-5.6-sol", "effort": "medium"})
        tier = retained_tier(worker, self.planned(worker), self.previous("gpt-5.6-sol", "high"))
        self.assertEqual((tier["effort"], tier["de_escalated"]), ("high", True))

    def test_a_declined_model_switch_stays_de_escalated_whatever_the_effort(self):
        worker = self.worker("claude", {"model": "sonnet-5", "effort": "medium"})
        tier = retained_tier(worker, self.planned(worker), self.previous("sonnet-5", "max"))
        self.assertEqual((tier["model"], tier["effort"], tier["de_escalated"]), ("sonnet-5", "max", True))

    def test_an_unchanged_effort_keeps_the_planned_flag(self):
        worker = self.worker("codex", {"model": "gpt-5.6-sol", "effort": "medium"})
        planned = self.planned(worker)
        self.assertEqual(retained_tier(worker, planned, self.previous("gpt-5.6-sol", "medium")), planned)

    def test_a_plan_without_the_flag_gains_none(self):
        worker = self.worker("codex", {"model": "gpt-5.6-sol", "effort": "medium"})
        planned = {key: value for key, value in self.planned(worker).items() if key != "de_escalated"}
        self.assertNotIn("de_escalated", retained_tier(worker, planned, self.previous("gpt-5.6-sol", "xhigh")))


if __name__ == "__main__":
    unittest.main()

skills

herdr-foreman

tests

__init__.py

fakes.py

test_assign.py

test_attention.py

test_billing.py

test_bounded_run.sh

test_capabilities.py

test_capability_routing.py

test_chronology.py

test_churn.py

test_classify.sh

test_claude_native.py

test_cli.py

test_compose_briefs.sh

test_composer.py

test_composition.py

test_config.py

test_continuity_cli.py

test_cost_report.py

test_diagnostics.py

test_engagement.py

test_entrypoints.py

test_foreman_launcher.sh

test_foreman_queue.py

test_foreman_reset.py

test_foreman_seat.py

test_foreman_tier_check.py

test_freeze.py

test_herdr.py

test_historical.py

test_home.py

test_label_workspaces.sh

test_launch.py

test_legacy_recovery.py

test_lifecycle.py

test_load_set.py

test_measure.py

test_members.py

test_memory.py

test_minimum_adequate.py

test_oracle.py

test_parsers.py

test_partition.py

test_planner.py

test_probe.py

test_provision_worktree.sh

test_prune_remote_branches.sh

test_prune_report_caches.py

test_prune_result.py

test_prune_worktrees.sh

test_recovery_cli.py

test_recovery.py

test_renderable.py

test_report_contract.py

test_report_delivery.py

test_report_gates.py

test_report_verdict.py

test_reset_input_hook.py

test_resolve_gates.sh

test_resolve_policy_paths.py

test_restoration.py

test_retrospective_runtime.py

test_retrospective.py

test_review_package.py

test_role_clear.py

test_roster.sh

test_round_preflight.sh

test_runnable.py

test_scoring.py

test_script_dir_newline.sh

test_seat_holds.py

test_selection.py

test_skill_invocations.sh

test_slice_scope_parity.py

test_specialist_cli.py

test_specialist_delivery.py

test_specialist_recovery.py

test_specialist_retention.py

test_stale_grok_delivery.py

test_start_judge_worker.py

test_state.py

test_successors.py

test_supervision_cli.py

test_supervision_diagnostics.py

test_supervision_gate.py

test_supervision_replay.py

test_supervision.py

test_sweep_worktrees.sh

test_tier_integration.py

test_tiers.py

test_triggers.py

test_typesafe_client.py

test_verdict_gates.py

test_verify_authority.sh

test_wait_report.sh

tier_fixture.py

bounded-run.sh

compose-briefs.sh

config.example.json

foreman-tier-check.py

foreman.sh

label-workspaces.sh

provision-worktree.sh

prune-remote-branches.sh

prune-report-caches.py

prune-worktrees.sh

resolve-gates.sh

resolve-policy-paths.sh

review-package.sh

roster.sh

round-preflight.sh

SKILL.md

start-judge-worker.sh

state-schema.md

sweep-worktrees.sh

verify-authority.sh

wait-report.sh

README.md

tile.json