{
  "context": "This criteria evaluates how well the engineer uses ConnectorX's batch processing and streaming capabilities to efficiently process large database query results without loading the entire dataset into memory. The focus is on proper usage of the arrow_stream return type, RecordBatchReader iteration, and correct batch size configuration.",
  "type": "weighted_checklist",
  "checklist": [
    {
      "name": "Uses arrow_stream return type",
      "description": "Uses read_sql() with return_type='arrow_stream' to get a RecordBatchReader for streaming batch processing instead of loading all data into memory at once",
      "max_score": 30
    },
    {
      "name": "Configures batch_size parameter",
      "description": "Passes the batch_size parameter to read_sql() to control the number of rows in each batch (e.g., batch_size=5000), matching the requirement for configurable batch sizes",
      "max_score": 15
    },
    {
      "name": "Iterates through RecordBatchReader",
      "description": "Correctly iterates through the RecordBatchReader (RecordBatchReader is an iterator that yields batches), processing each batch individually without materializing the full dataset",
      "max_score": 25
    },
    {
      "name": "Processes each batch",
      "description": "Converts each RecordBatch to a usable format (e.g., to_pandas(), to_pydict()) and extracts/aggregates the required data from each batch",
      "max_score": 15
    },
    {
      "name": "Aggregates across batches",
      "description": "Maintains running totals/counts across all batches by combining results from each batch iteration into final aggregated results",
      "max_score": 15
    }
  ]
}