{
  "context": "Tests whether the agent correctly follows the pgvector section of the PostgreSQL skill: installing the extension as 'vector' (not 'pgvector'), choosing HNSW for better recall, selecting the correct opclass for the distance metric used, recommending halfvec for memory efficiency, configuring iterative_scan for filtered ANN queries, tuning maintenance_work_mem for large index builds, and using provider-neutral language about managed services.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Extension named 'vector'",
      "description": "The implementation uses `CREATE EXTENSION vector` (not `CREATE EXTENSION pgvector`) to install the extension",
      "max_score": 15
    },
    {
      "name": "HNSW index chosen",
      "description": "The implementation creates an HNSW index (USING hnsw) rather than IVFFlat as the primary index type, with a justification related to better query performance or recall",
      "max_score": 10
    },
    {
      "name": "Cosine opclass used",
      "description": "The HNSW index uses the cosine distance opclass (vector_cosine_ops or halfvec_cosine_ops) matching the semantic similarity use case",
      "max_score": 12
    },
    {
      "name": "halfvec for memory savings",
      "description": "The implementation uses halfvec type (instead of or as an alternative to vector) OR the deployment notes explicitly recommend halfvec for 4x memory reduction for 1536-dimensional embeddings",
      "max_score": 10
    },
    {
      "name": "Iterative scan for filtered ANN",
      "description": "The implementation or deployment notes include setting hnsw.iterative_scan = strict_order (or equivalent) to handle category-filtered ANN queries correctly",
      "max_score": 13
    },
    {
      "name": "Provider-neutral managed phrasing",
      "description": "Any mention of managed cloud PostgreSQL limitations or pgvector availability is phrased categorically (e.g., 'most managed providers preinstall...') without naming specific cloud providers",
      "max_score": 10
    },
    {
      "name": "One opclass per index",
      "description": "The implementation does NOT create a single index with multiple opclasses, or explicitly notes that each index supports only one distance metric/opclass",
      "max_score": 7
    },
    {
      "name": "ef_search / probes tuning",
      "description": "The deployment notes mention tuning hnsw.ef_search (or ivfflat.probes) to trade recall for latency, and note these are session-scoped GUCs",
      "max_score": 8
    },
    {
      "name": "HNSW build memory",
      "description": "The deployment notes mention raising maintenance_work_mem (and optionally max_parallel_maintenance_workers) when building HNSW indexes on large tables",
      "max_score": 8
    },
    {
      "name": "PG16 baseline SQL",
      "description": "The SQL in implementation.sql is written for PostgreSQL 16 compatibility as the baseline",
      "max_score": 7
    }
  ]
}