CtrlK
BlogDocsLog inGet started
Tessl Logo

jbaruch/coding-policy

General-purpose coding policy for Baruch's AI agents

74

Quality

93%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Medium

Suggest reviewing before use

Overview
Quality
Evals
Security
Files

test_supervision_gate.pyskills/herdr-foreman/tests/

"""What the supervision gate suppresses, and everything it must not."""

import os as _os
import sys as _sys
import unittest

_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
if _ROOT not in _sys.path:
    _sys.path.insert(0, _ROOT)

from foreman import supervision_gate as gate
from foreman.errors import UsageError

AT = "2026-09-21T00:00:00+00:00"


def store(events, members=("m1",), acknowledgements=()):
    return {"schema_version": 1,
            "members": [{"id": name, "at": AT, "active": True, "observed": {},
                         "assignment": {}, "refinements": [], "resolution": None}
                        for name in members],
            "events": [{"schema_version": 1, "id": "e{}".format(i), "seq": i, "at": AT,
                        "member": member, "kind": kind, "data": data}
                       for i, (member, kind, data) in enumerate(events, start=1)],
            "acknowledgements": list(acknowledgements)}


def verdicts(result):
    return {row["event"]: (row["kind"], row["reason"]) for row in result["suppressed"]}


class DefaultWakeTest(unittest.TestCase):
    """The safety property: only named cases are suppressed."""

    def test_a_kind_this_module_has_never_seen_wakes_the_lead(self):
        # Recorded history contains none of the failure kinds. A gate that
        # enumerated what to WAKE on would be silent for exactly those.
        for kind in ("watcher_lost", "observation_error_observed", "report_error_observed",
                     "unavailable_observed", "report_unreadable_observed",
                     "some_kind_invented_next_month"):
            with self.subTest(kind=kind):
                result = gate.evaluate(store([("m1", kind, {})]))
                self.assertEqual(result["counts"]["suppressed"], 0)
                self.assertEqual(result["wake"][0]["reason"], "kind is not suppressible")

    def test_the_kinds_that_carry_the_work_always_wake(self):
        for kind, data in (("report_observed", {"present": True, "path": "/r.md"}),
                           ("lifecycle_observed", {"status": "idle"}),
                           ("identity_changed_observed", {"pane_id": "w1:p1"})):
            with self.subTest(kind=kind):
                self.assertEqual(gate.evaluate(store([("m1", kind, data)]))["counts"]["suppressed"], 0)

    def test_an_event_with_no_member_wakes(self):
        result = gate.evaluate(store([(None, "visible_observed", {"sha256": "a" * 64})]))
        self.assertEqual(result["wake"][0]["reason"], "event has no member to join against")


class ScreenHashTest(unittest.TestCase):
    """A screen hash is noise until a report file exists, and signal after."""

    def test_a_screen_hash_before_any_report_is_suppressed(self):
        for lifecycle in ({"status": "working"}, {"status": "idle"}, {"status": "blocked"}):
            with self.subTest(lifecycle=lifecycle):
                result = gate.evaluate(store([
                    ("m1", "lifecycle_observed", lifecycle),
                    ("m1", "visible_observed", {"sha256": "a" * 64}),
                ]))
                self.assertEqual([row["kind"] for row in result["suppressed"]], ["visible_observed"])

    def test_a_screen_hash_after_a_report_file_wakes_the_lead(self):
        # A report FILE is not delivery. Delivery is the file plus the
        # `REPORT: <path>` marker in the worker's final message, and the marker
        # reaches the supervisor only as a screen change. Suppressing these on
        # the recorded history lost 3 deliveries and delayed 5 more.
        result = gate.evaluate(store([
            ("m1", "report_observed", {"present": True, "path": "/r.md"}),
            ("m1", "visible_observed", {"sha256": "b" * 64}),
            ("m1", "visible_observed", {"sha256": "c" * 64}),
        ]))
        self.assertEqual(result["counts"]["suppressed"], 0)
        self.assertEqual([row["kind"] for row in result["wake"]],
                         ["report_observed", "visible_observed", "visible_observed"])

    def test_a_report_that_is_not_present_does_not_count(self):
        result = gate.evaluate(store([
            ("m1", "report_observed", {"present": False, "path": "/r.md"}),
            ("m1", "visible_observed", {"sha256": "d" * 64}),
        ]))
        self.assertEqual([row["kind"] for row in result["suppressed"]], ["visible_observed"])


class DeferralTest(unittest.TestCase):
    """A recheck reads the same state, or it does not."""

    def test_an_unchanged_deferral_is_suppressed(self):
        result = gate.evaluate(store([
            ("m1", "lifecycle_observed", {"status": "working"}),
            ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
        ]))
        self.assertIn("e2", verdicts(result))
        self.assertEqual(verdicts(result)["e2"][1], "observed state unchanged since the deferral")

    def test_a_deferral_whose_state_moved_wakes(self):
        result = gate.evaluate(store([
            ("m1", "lifecycle_observed", {"status": "working"}),
            ("m1", "report_observed", {"present": True, "path": "/r.md"}),
            ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
        ]))
        self.assertEqual(result["counts"]["suppressed"], 0)
        self.assertEqual(result["wake"][-1]["reason"], "observed state changed since the deferral")

    def test_an_unchanging_worker_wakes_the_lead_rather_than_going_quiet_forever(self):
        # Indistinguishable from a stall, and a stall is the foreman's to judge.
        events = [("m1", "lifecycle_observed", {"status": "working"})]
        events += [("m1", "recheck_due", {"event": "e1", "recheck_at": AT})
                   for _ in range(gate.MAX_QUIET_RECHECKS + 2)]
        result = gate.evaluate(store(events))
        self.assertEqual(result["counts"]["suppressed"], gate.MAX_QUIET_RECHECKS)
        self.assertIn("stall is the foreman's to judge", result["wake"][-1]["reason"])

    def test_a_deferral_stays_woken_once_its_state_has_moved_past_the_baseline(self):
        events = [("m1", "lifecycle_observed", {"status": "working"}),
                  ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
                  ("m1", "report_observed", {"present": True, "path": "/r.md"}),
                  ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
                  ("m1", "recheck_due", {"event": "e1", "recheck_at": AT})]
        result = gate.evaluate(store(events))
        # e2 is quiet; e4 and e5 both wake, because e1's baseline never moves.
        # Herdr advances it by deferring the recheck itself, which carries a
        # newer event id and therefore a newer baseline.
        self.assertEqual([row["event"] for row in result["suppressed"]], ["e2"])
        self.assertEqual([row["event"] for row in result["wake"]], ["e1", "e3", "e4", "e5"])


class ReplayTest(unittest.TestCase):
    """A verdict reads the state as of its own event, never a later one."""

    def test_a_verdict_never_reads_the_change_it_is_judging(self):
        # The report lands in the same event being judged: it must still wake,
        # and the NEXT deferral is the one that sees the new state.
        result = gate.evaluate(store([
            ("m1", "lifecycle_observed", {"status": "working"}),
            ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
            ("m1", "report_observed", {"present": True, "path": "/r.md"}),
            ("m1", "recheck_due", {"event": "e1", "recheck_at": AT}),
        ]))
        self.assertEqual([row["event"] for row in result["suppressed"]], ["e2"])

    def test_events_are_judged_in_sequence_order_not_file_order(self):
        data = store([("m1", "lifecycle_observed", {"status": "working"}),
                      ("m1", "recheck_due", {"event": "e1", "recheck_at": AT})])
        data["events"].reverse()
        result = gate.evaluate(data)
        self.assertEqual([row["event"] for row in result["suppressed"]], ["e2"])

    def test_members_are_judged_independently(self):
        result = gate.evaluate(store([
            ("m1", "lifecycle_observed", {"status": "working"}),
            ("m2", "recheck_due", {"event": "e1", "recheck_at": AT}),
        ], members=("m1", "m2")))
        # m2 never observed anything, so m1's sample must not answer for it.
        self.assertEqual(result["counts"]["suppressed"], 0)


class PendingTest(unittest.TestCase):
    """What the foreman is shown: verdicts for unacknowledged events only."""

    def test_only_unacknowledged_events_are_reported(self):
        data = store([("m1", "lifecycle_observed", {"status": "working"}),
                      ("m1", "visible_observed", {"sha256": "a" * 64}),
                      ("m1", "visible_observed", {"sha256": "b" * 64})],
                     acknowledgements=[{"event": "e1"}, {"event": "e2"}])
        result = gate.pending(data)
        self.assertEqual(result["counts"], {"pending": 1, "wake": 0, "suppressed": 1})
        self.assertEqual([row["event"] for row in result["suppressed"]], ["e3"])

    def test_an_acknowledged_event_still_shapes_the_state_a_pending_one_reads(self):
        # e1 is handled, but the report it observed is why e2 must wake.
        data = store([("m1", "report_observed", {"present": True, "path": "/r.md"}),
                      ("m1", "visible_observed", {"sha256": "a" * 64})],
                     acknowledgements=[{"event": "e1"}])
        result = gate.pending(data)
        self.assertEqual([row["event"] for row in result["wake"]], ["e2"])


class InputTest(unittest.TestCase):
    def test_a_store_without_events_is_refused(self):
        for broken in ({}, {"events": "none"}, []):
            with self.subTest(broken=broken):
                with self.assertRaisesRegex(UsageError, "events array"):
                    gate.evaluate(broken)

    def test_counts_add_up(self):
        result = gate.evaluate(store([("m1", "visible_observed", {"sha256": "a" * 64}),
                                      ("m1", "report_observed", {"present": True})]))
        counts = result["counts"]
        self.assertEqual(counts["wake"] + counts["suppressed"], counts["total"])
        self.assertEqual(counts["total"], 2)


if __name__ == "__main__":
    unittest.main()

skills

herdr-foreman

tests

__init__.py

fakes.py

test_assign.py

test_attention.py

test_billing.py

test_bounded_run.sh

test_capabilities.py

test_capability_routing.py

test_chronology.py

test_churn.py

test_classify.sh

test_claude_native.py

test_cli.py

test_compose_briefs.sh

test_composer.py

test_composition.py

test_config.py

test_continuity_cli.py

test_cost_report.py

test_diagnostics.py

test_engagement.py

test_entrypoints.py

test_foreman_launcher.sh

test_foreman_queue.py

test_foreman_reset.py

test_foreman_seat.py

test_foreman_tier_check.py

test_freeze.py

test_herdr.py

test_historical.py

test_home.py

test_label_workspaces.sh

test_launch.py

test_legacy_recovery.py

test_lifecycle.py

test_load_set.py

test_measure.py

test_members.py

test_memory.py

test_minimum_adequate.py

test_oracle.py

test_parsers.py

test_partition.py

test_planner.py

test_probe.py

test_provision_worktree.sh

test_prune_remote_branches.sh

test_prune_report_caches.py

test_prune_result.py

test_prune_worktrees.sh

test_recovery_cli.py

test_recovery.py

test_renderable.py

test_report_contract.py

test_report_delivery.py

test_report_gates.py

test_report_verdict.py

test_reset_input_hook.py

test_resolve_gates.sh

test_resolve_policy_paths.py

test_restoration.py

test_retrospective_runtime.py

test_retrospective.py

test_review_package.py

test_role_clear.py

test_roster.sh

test_round_preflight.sh

test_runnable.py

test_scoring.py

test_script_dir_newline.sh

test_seat_holds.py

test_selection.py

test_skill_invocations.sh

test_slice_scope_parity.py

test_specialist_cli.py

test_specialist_delivery.py

test_specialist_recovery.py

test_specialist_retention.py

test_stale_grok_delivery.py

test_start_judge_worker.py

test_state.py

test_successors.py

test_supervision_cli.py

test_supervision_diagnostics.py

test_supervision_gate.py

test_supervision_replay.py

test_supervision.py

test_sweep_worktrees.sh

test_tier_integration.py

test_tiers.py

test_triggers.py

test_typesafe_client.py

test_verdict_gates.py

test_verify_authority.sh

test_wait_report.sh

tier_fixture.py

bounded-run.sh

compose-briefs.sh

config.example.json

foreman-tier-check.py

foreman.sh

label-workspaces.sh

provision-worktree.sh

prune-remote-branches.sh

prune-report-caches.py

prune-worktrees.sh

resolve-gates.sh

resolve-policy-paths.sh

review-package.sh

roster.sh

round-preflight.sh

SKILL.md

start-judge-worker.sh

state-schema.md

sweep-worktrees.sh

verify-authority.sh

wait-report.sh

README.md

tile.json