Reviews repositories, pull requests, diffs, and agent-generated code for reward hacking, fake completion, defensive theater, architectural bypasses, weakened guarantees, hidden fallbacks, and misleading abstractions.
98
97%
Does it follow best practices?
Impact
100%
1.09xAverage score across 6 eval scenarios
Passed
No known issues
#!/usr/bin/env python3
"""Collect lightweight implementation-integrity leads from source files."""
from __future__ import annotations
import argparse
import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Iterable
SKIP_DIRS = {
".git",
".hg",
".svn",
".venv",
"venv",
"node_modules",
"dist",
"build",
"coverage",
"__pycache__",
}
SOURCE_SUFFIXES = {
".py",
".js",
".jsx",
".ts",
".tsx",
".go",
".rs",
".java",
".kt",
".cs",
".rb",
".php",
".sh",
}
@dataclass(frozen=True)
class Rule:
category: str
severity: str
confidence: str
pattern: re.Pattern[str]
rationale: str
@dataclass(frozen=True)
class IntegritySignal:
category: str
severity: str
confidence: str
evidence: str
rationale: str
file: str
line: int
RULES = [
Rule(
"Fake Completion",
"High",
"Medium",
re.compile(r"\breturn\s+(True|true|\{\s*success\s*:\s*true\s*\})\s*(#.*)?$"),
"Hardcoded success return may hide missing work or skipped side effects.",
),
Rule(
"Reward Hacking",
"Critical",
"Medium",
re.compile(r"\b(test|fixture|snapshot|golden|pytest|unittest)\b.*\b(return|if|case)\b", re.I),
"Code appears to branch on test or fixture concepts.",
),
Rule(
"Defensive Theater",
"High",
"Medium",
re.compile(r"\bexcept\s+(Exception|BaseException)\b|\bcatch\s*\([^)]*\)\s*\{?\s*$"),
"Broad exception handling can hide failures unless it reports and preserves semantics.",
),
Rule(
"Silent Degradation",
"High",
"Medium",
re.compile(r"\b(pass|return\s+None|return\s+null|return\s+undefined)\b\s*(#\s*(ignore|fallback|optional|best effort))?", re.I),
"Silent no-op or null fallback may weaken caller guarantees.",
),
Rule(
"Weakened Validation",
"High",
"Medium",
re.compile(r"\b(skip[_-]?validation|validate\s*=\s*False|validation\s*:\s*false|validate\s*:\s*false)\b", re.I),
"Validation appears to be disabled or bypassed.",
),
Rule(
"Placeholder Implementation",
"Medium",
"Low",
re.compile(r"\b(TODO|FIXME|stub|placeholder|not implemented|temporary)\b", re.I),
"Placeholder language can indicate incomplete production behavior.",
),
]
def iter_source_files(root: Path) -> Iterable[Path]:
if root.is_file():
if root.suffix in SOURCE_SUFFIXES:
yield root
return
for path in root.rglob("*"):
if any(part in SKIP_DIRS for part in path.parts):
continue
if path.is_file() and path.suffix in SOURCE_SUFFIXES:
yield path
def scan_file(path: Path) -> list[IntegritySignal]:
try:
lines = path.read_text(encoding="utf-8").splitlines()
except UnicodeDecodeError:
return []
signals: list[IntegritySignal] = []
for line_number, line in enumerate(lines, start=1):
stripped = line.strip()
for rule in RULES:
if rule.pattern.search(stripped):
signals.append(
IntegritySignal(
category=rule.category,
severity=rule.severity,
confidence=rule.confidence,
evidence=stripped[:240],
rationale=rule.rationale,
file=str(path),
line=line_number,
)
)
return signals
def collect(root: Path) -> list[IntegritySignal]:
signals: list[IntegritySignal] = []
for path in iter_source_files(root):
signals.extend(scan_file(path))
return signals
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("path", nargs="?", default=".", help="Repository path to scan")
parser.add_argument(
"--format",
choices=("json", "text"),
default="json",
help="Output format",
)
args = parser.parse_args()
signals = collect(Path(args.path))
if args.format == "json":
print(json.dumps([asdict(signal) for signal in signals], indent=2))
else:
for signal in signals:
print(
f"{signal.file}:{signal.line}: {signal.severity} "
f"{signal.category}: {signal.evidence}"
)
return 1 if signals else 0
if __name__ == "__main__":
raise SystemExit(main())