Python project structure — pyproject.toml, src layout, __init__.py, .gitignore, dependency groups, type hints, py.typed, test structure, entry points, ruff/mypy configuration
91
87%
Does it follow best practices?
Impact
99%
1.03xAverage score across 5 eval scenarios
Passed
No known issues
{
"context": "Tests whether the agent sets up a FastAPI project with proper pyproject.toml, .gitignore, package structure with __init__.py, dependency groups, type hints, ruff/mypy config, test structure, and app factory pattern.",
"type": "weighted_checklist",
"checklist": [
{
"name": "pyproject.toml with build-system",
"description": "pyproject.toml exists with a [build-system] section specifying a build backend (hatchling, setuptools, etc.)",
"max_score": 8
},
{
"name": "requires-python >= 3.11",
"description": "pyproject.toml has requires-python set to >= 3.11 or >= 3.12",
"max_score": 6
},
{
"name": "No setup.py created",
"description": "No setup.py or setup.cfg file exists — pyproject.toml is the sole project config",
"max_score": 5
},
{
"name": "Dependency groups separated",
"description": "pyproject.toml has [project.optional-dependencies] with dev or test groups containing pytest, ruff, mypy, httpx — not mixed into main [project.dependencies]",
"max_score": 8
},
{
"name": "Ruff configured in pyproject.toml",
"description": "pyproject.toml has a [tool.ruff] or [tool.ruff.lint] section with lint rules configured",
"max_score": 6
},
{
"name": "Mypy configured with strict",
"description": "pyproject.toml has a [tool.mypy] section with strict = true",
"max_score": 6
},
{
"name": ".gitignore includes venv",
"description": ".gitignore exists and includes .venv/ or venv/ to prevent committing virtual environment",
"max_score": 6
},
{
"name": ".gitignore includes __pycache__ and .env",
"description": ".gitignore includes __pycache__/ and .env",
"max_score": 5
},
{
"name": "__init__.py in all packages",
"description": "Every package directory (app/project package, routes/, tests/) has an __init__.py file",
"max_score": 8
},
{
"name": "Config uses pydantic-settings BaseSettings",
"description": "Configuration module uses pydantic_settings.BaseSettings with fields loaded from environment, not hardcoded",
"max_score": 6
},
{
"name": "Routes separated by resource",
"description": "Pet routes and application routes are in separate files in a routes/ directory, each using APIRouter",
"max_score": 7
},
{
"name": "App factory pattern in main.py",
"description": "main.py has a create_app() function that creates and configures the FastAPI instance",
"max_score": 6
},
{
"name": "Type hints on function signatures",
"description": "Function definitions include type hints for parameters and return types (not bare def foo(x):)",
"max_score": 7
},
{
"name": "tests/ with conftest.py",
"description": "A tests/ directory exists with conftest.py containing at least one fixture, and at least one test_*.py file",
"max_score": 8
},
{
"name": "Database logic separated",
"description": "Database/storage logic is in a separate db.py module, not inline in route handlers",
"max_score": 4
},
{
"name": "Custom errors with handler registration",
"description": "errors.py has custom exception classes and a register_error_handlers function called from main.py",
"max_score": 4
}
]
}evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
skills
python-project-structure
verifiers