Python project structure — pyproject.toml, src layout, __init__.py, .gitignore, dependency groups, type hints, py.typed, test structure, entry points, ruff/mypy configuration
91
87%
Does it follow best practices?
Impact
99%
1.03xAverage score across 5 eval scenarios
Passed
No known issues
{
"context": "Tests whether the agent creates a properly structured Python library using src layout, py.typed marker, public API in __init__.py, pyproject.toml with build-system, dependency groups, and full type annotations.",
"type": "weighted_checklist",
"checklist": [
{
"name": "src layout used",
"description": "Source code is in src/package_name/ directory (src layout), not flat at project root",
"max_score": 10
},
{
"name": "py.typed marker",
"description": "A py.typed file exists inside the package directory (src/package_name/py.typed) for PEP 561 compliance",
"max_score": 10
},
{
"name": "pyproject.toml with build-system",
"description": "pyproject.toml has [build-system] with a build backend (hatchling, setuptools, etc.)",
"max_score": 8
},
{
"name": "requires-python",
"description": "pyproject.toml has requires-python >= 3.11",
"max_score": 5
},
{
"name": "Dependency groups",
"description": "[project.optional-dependencies] with dev/test groups — pytest, ruff, mypy not in main dependencies",
"max_score": 7
},
{
"name": "Ruff configured",
"description": "[tool.ruff] section in pyproject.toml",
"max_score": 5
},
{
"name": "Mypy strict",
"description": "[tool.mypy] with strict = true in pyproject.toml",
"max_score": 6
},
{
"name": "__init__.py with public API",
"description": "Package __init__.py re-exports the public API (key classes/functions) with __all__ defined",
"max_score": 8
},
{
"name": "__init__.py in all directories",
"description": "All package directories and tests/ have __init__.py",
"max_score": 6
},
{
"name": ".gitignore comprehensive",
"description": ".gitignore includes .venv/, __pycache__/, dist/, *.egg-info/, .env, .mypy_cache/",
"max_score": 5
},
{
"name": "Type hints on all functions",
"description": "All function definitions have parameter type annotations and return type annotations",
"max_score": 8
},
{
"name": "tests/ with conftest and test files",
"description": "tests/ exists with __init__.py, conftest.py, and at least one test_*.py file",
"max_score": 7
},
{
"name": "No setup.py",
"description": "No setup.py or setup.cfg exists",
"max_score": 4
},
{
"name": "Custom exceptions defined",
"description": "Custom exception classes (ValidationError, SchemaError or similar) are defined in a separate exceptions module",
"max_score": 4
},
{
"name": "pytest configured",
"description": "[tool.pytest.ini_options] in pyproject.toml with testpaths",
"max_score": 3
}
]
}evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
skills
python-project-structure
verifiers