{
  "context": "Tests whether the agent correctly classifies and handles each type of review comment (concrete change, question, disagreement, out-of-scope), properly updates §Open Questions with the required three-line format, filters out old comments predating the last revise commit, and produces the correct post-push PR communication artifacts in the right order.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Filter old comments",
      "description": "The revision_plan.md or revised spec treats comment 3001 (created 2026-05-28T08:00:00Z, before the last revise commit at 2026-05-28T10:00:00Z) as already addressed and does NOT act on it",
      "max_score": 8
    },
    {
      "name": "Concrete change applied",
      "description": "The revised spec incorporates the p99 latency change from 500ms to 2s for mobile clients (comment 1001)",
      "max_score": 10
    },
    {
      "name": "Concrete change: Kafka chosen",
      "description": "The revised spec updates §Open Questions or §Design to resolve the queue question in favour of Kafka (comment 1002 — concrete change aligned with project architecture)",
      "max_score": 10
    },
    {
      "name": "Schema column added",
      "description": "The revised spec's data schema includes a `channel` column (or equivalent) to distinguish WebSocket vs mobile push destinations (comment 2001)",
      "max_score": 10
    },
    {
      "name": "Question handled in spec",
      "description": "The APNs credentials ownership question (comment 1003) is answered within the revised spec — either in §Open Questions, §Risks, or a dedicated ownership section — rather than just ignored",
      "max_score": 8
    },
    {
      "name": "Out-of-scope reply, no SPEC change",
      "description": "The out-of-scope Slack notification request (comment 1004) does NOT cause any change to the revised SPEC; the per_comment_replies.md contains a reply for comment 1004 explaining it is out of scope",
      "max_score": 10
    },
    {
      "name": "Open Questions three-line format",
      "description": "Every item in §Open Questions in the revised spec contains exactly three elements: the choice, an alternative, and the cost of being wrong",
      "max_score": 10
    },
    {
      "name": "Resolved OQ removed",
      "description": "The queue implementation open question (resolved by applying Kafka from comment 1002) is removed from §Open Questions in the revised spec",
      "max_score": 8
    },
    {
      "name": "Per-comment reply format",
      "description": "Each entry in per_comment_replies.md for addressed comments includes the phrase 'Addressed in' followed by a commit SHA placeholder and a one-line description of what was done",
      "max_score": 8
    },
    {
      "name": "PR summary comment format",
      "description": "pr_comment_summary.md contains counts of comments addressed, deferred/replied-to, a diff link placeholder, and a count of remaining open questions",
      "max_score": 10
    },
    {
      "name": "Correct label removal order",
      "description": "pr_comment_summary.md or revision_plan.md explicitly states that the `ai:revise-now` label should be removed LAST, after the push and after posting comments",
      "max_score": 8
    }
  ]
}