docs
evals
scenario-1
scenario-10
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
{
"context": "This criteria evaluates how well an engineer uses Git and command execution capabilities to implement a workspace package selector based on Git changes. The focus is on proper integration with Git commands, parsing Git references, and mapping file changes to workspace packages.",
"type": "weighted_checklist",
"checklist": [
{
"name": "Git command execution",
"description": "Uses appropriate Git commands (e.g., `git diff`, `git rev-parse`) via child_process or similar to get changed files between references. Should properly handle command execution and capture output.",
"max_score": 25
},
{
"name": "parseGitFilter implementation",
"description": "Correctly implements parseGitFilter function to extract Git reference from bracket notation (e.g., '[origin/main]' -> 'origin/main'). Should validate format and handle edge cases.",
"max_score": 15
},
{
"name": "getChangedFiles implementation",
"description": "Correctly implements getChangedFiles function to execute Git diff command and return list of changed file paths. Should handle async operations properly and parse Git output correctly.",
"max_score": 25
},
{
"name": "selectChangedPackages implementation",
"description": "Correctly implements selectChangedPackages function to filter workspace packages based on whether they contain changed files. Should properly match file paths to package directories.",
"max_score": 20
},
{
"name": "Error handling",
"description": "Properly handles and throws errors for invalid Git references, non-existent references, and when not in a Git repository. Error messages should be clear and informative.",
"max_score": 10
},
{
"name": "Git reference validation",
"description": "Validates that Git references exist before attempting to diff (e.g., using `git rev-parse --verify`). Should prevent errors from invalid references.",
"max_score": 5
}
]
}