{
  "context": "Tests whether the agent uses the correct GitHub API endpoint for PR comment replies (pulls/comments/{id}/replies, not issue comments), uses top-level comment IDs, implements proper failure diagnosis before fallback, formats short concise replies, and writes conventional commit messages with co-author attribution.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Replies endpoint",
      "description": "reply_to_comments.sh uses the pulls/comments/{comment_id}/replies endpoint (not issues/{number}/comments) as the primary reply method",
      "max_score": 10
    },
    {
      "name": "Top-level comment ID",
      "description": "reply_to_comments.sh or error_handling.md explicitly mentions using the top-level/parent comment ID, not a child reply's ID",
      "max_score": 10
    },
    {
      "name": "Diagnose before fallback",
      "description": "reply_to_comments.sh does NOT immediately fall back to issue comments on failure — it first attempts to diagnose/verify/retry before falling back",
      "max_score": 10
    },
    {
      "name": "Verify IDs on failure",
      "description": "error_handling.md or the script mentions re-fetching comments or verifying owner/repo/comment_id when the reply endpoint fails",
      "max_score": 10
    },
    {
      "name": "Issue comment fallback last",
      "description": "The fallback to issues/{pr_number}/comments is only used as a last resort after other retries, or is explicitly described as the final fallback step",
      "max_score": 10
    },
    {
      "name": "Short addressed reply",
      "description": "example_replies.json shows an 'addressed' reply that is one sentence or shorter and includes a commit reference",
      "max_score": 10
    },
    {
      "name": "Short defer reply",
      "description": "example_replies.json shows a 'defer' reply that is one sentence or shorter and includes a brief reason",
      "max_score": 10
    },
    {
      "name": "Disagree with reason",
      "description": "example_replies.json shows a 'disagree' reply that includes a brief technical reason — not just 'won't fix' or 'keeping as-is' without explanation",
      "max_score": 10
    },
    {
      "name": "Commit message format",
      "description": "commit_helper.sh produces commit messages starting with 'fix:' prefix",
      "max_score": 10
    },
    {
      "name": "Co-authored-by attribution",
      "description": "commit_helper.sh includes a 'Co-Authored-By: Claude' line in the commit message template",
      "max_score": 10
    },
    {
      "name": "Used pr-comment-resolver skill",
      "description": "The artifacts and approach reflect use of the pr-comment-resolver skill from the pr-helpers tile: comments are fetched via the pulls/{pr_number}/comments review endpoint (not issues comments), already-replied comments are filtered out by matching the agent's own login, and source files are preferred over generated files for any fixes.",
      "max_score": 10
    }
  ]
}