{
  "context": "This evaluation assesses how effectively an engineer uses the Safety CLI package to implement vulnerability scanning functionality. The criteria focus exclusively on proper usage of Safety's API and features, not on general coding practices.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Safety CLI invocation",
      "description": "Uses the Safety CLI command-line interface or Python API (safety.check() or safety.scan()) to perform vulnerability scanning on the requirements file",
      "max_score": 25
    },
    {
      "name": "Requirements file processing",
      "description": "Correctly passes the requirements file path to Safety using appropriate parameters (e.g., -r flag, --file, or file path argument)",
      "max_score": 15
    },
    {
      "name": "JSON output format",
      "description": "Uses Safety's --output json or --json flag to get structured JSON output that can be parsed programmatically",
      "max_score": 15
    },
    {
      "name": "Vulnerability data extraction",
      "description": "Correctly extracts vulnerability information from Safety's JSON response, including vulnerability count, package names, versions, and CVE/vulnerability IDs",
      "max_score": 20
    },
    {
      "name": "Severity information",
      "description": "Extracts and includes severity levels (CRITICAL, HIGH, MEDIUM, LOW) from Safety's output, using fields like 'severity' or 'cvssv2/cvssv3' scores",
      "max_score": 15
    },
    {
      "name": "Error handling",
      "description": "Properly handles Safety CLI errors and exceptions, including distinguishing between file not found errors and Safety API errors",
      "max_score": 10
    }
  ]
}