{
  "context": "Tests whether the agent performs a Rails security review following the rails-security-review skill: covering the correct review areas in order, classifying findings by correct severity levels, and producing output with all four required fields per finding.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Auth/authz reviewed first",
      "description": "The review report addresses authentication or authorization issues before or separately from parameter, query, or output issues — the first finding or section covers auth concerns",
      "max_score": 8
    },
    {
      "name": "Parameter handling reviewed",
      "description": "The review explicitly addresses how parameters are handled (strong params, mass assignment, or permit patterns)",
      "max_score": 8
    },
    {
      "name": "Query safety reviewed",
      "description": "The review addresses SQL query construction, injection risk, or use of parameterized queries",
      "max_score": 8
    },
    {
      "name": "High severity: SQL injection identified",
      "description": "The SQL injection vulnerability in the provided code is classified as High severity (not Medium or Low)",
      "max_score": 10
    },
    {
      "name": "High severity: missing authz identified",
      "description": "The missing authorization check is classified as High severity",
      "max_score": 10
    },
    {
      "name": "Medium severity finding identified",
      "description": "At least one finding is correctly classified as Medium severity (e.g. sensitive data logging, unscoped mass assignment, or weak filtering)",
      "max_score": 8
    },
    {
      "name": "Attack path per finding",
      "description": "Each High severity finding includes an attack path — a description of how an attacker would exploit the issue",
      "max_score": 10
    },
    {
      "name": "Affected file per finding",
      "description": "Each finding names the specific file (and ideally line or method) where the vulnerability exists",
      "max_score": 10
    },
    {
      "name": "Mitigation per finding",
      "description": "Each finding includes a smallest credible mitigation — a concrete, actionable fix rather than a generic recommendation",
      "max_score": 10
    },
    {
      "name": "Exploitability focus",
      "description": "The review prioritizes findings that are directly exploitable (e.g. no auth check, raw string interpolation in query) over style issues — does NOT lead with stylistic concerns",
      "max_score": 8
    },
    {
      "name": "Secrets and output reviewed",
      "description": "The review addresses at least one of: secrets in code/logs, unsafe redirects, or output rendering (HTML/XSS) concerns",
      "max_score": 10
    }
  ]
}