Python project structure — pyproject.toml, src layout, __init__.py, .gitignore, dependency groups, type hints, py.typed, test structure, entry points, ruff/mypy configuration
91
87%
Does it follow best practices?
Impact
99%
1.03xAverage score across 5 eval scenarios
Passed
No known issues
{
"context": "Tests whether the agent creates a properly structured Python CLI tool package with pyproject.toml entry points, src or flat layout, py.typed marker, dependency groups, type hints, and test structure.",
"type": "weighted_checklist",
"checklist": [
{
"name": "pyproject.toml with build-system",
"description": "pyproject.toml exists with [build-system] section",
"max_score": 8
},
{
"name": "[project.scripts] entry point",
"description": "pyproject.toml has a [project.scripts] section defining a CLI command that points to the entry point function (e.g., log-analyzer = \"package.cli:main\")",
"max_score": 10
},
{
"name": "requires-python >= 3.11",
"description": "pyproject.toml has requires-python >= 3.11 or higher",
"max_score": 5
},
{
"name": "Dependency groups",
"description": "[project.optional-dependencies] has dev or test groups with pytest, ruff, mypy separated from main dependencies",
"max_score": 8
},
{
"name": "Ruff configured",
"description": "[tool.ruff] or [tool.ruff.lint] section exists in pyproject.toml with lint rule selection",
"max_score": 6
},
{
"name": "Mypy strict",
"description": "[tool.mypy] section exists with strict = true",
"max_score": 6
},
{
"name": "Proper package with __init__.py",
"description": "Source code is in a proper Python package directory with __init__.py (either src/pkg/ or pkg/ layout)",
"max_score": 8
},
{
"name": "py.typed marker",
"description": "A py.typed file exists in the package directory (src/pkg/py.typed or pkg/py.typed)",
"max_score": 7
},
{
"name": ".gitignore with venv and pycache",
"description": ".gitignore includes .venv/, __pycache__/, dist/, and .env",
"max_score": 5
},
{
"name": "Type hints on all functions",
"description": "All function definitions have type annotations for parameters and return types",
"max_score": 8
},
{
"name": "Tests directory with conftest.py",
"description": "tests/ directory exists with __init__.py, conftest.py, and at least one test_*.py file",
"max_score": 7
},
{
"name": "No setup.py",
"description": "No setup.py or setup.cfg file is created",
"max_score": 4
},
{
"name": "Separate modules for concerns",
"description": "Code is split into separate modules (cli, parser, analyzer, models) rather than one monolithic file",
"max_score": 6
},
{
"name": "pytest configured in pyproject.toml",
"description": "[tool.pytest.ini_options] section exists in pyproject.toml with testpaths configured",
"max_score": 4
},
{
"name": "Data models defined",
"description": "Data models (LogEntry, AnalysisReport, or similar) are defined using dataclasses or Pydantic in a models module",
"max_score": 4
}
]
}evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
skills
python-project-structure
verifiers