CtrlK
BlogDocsLog inGet started
Tessl Logo

evilissimo/implementation-integrity-review

Reviews repositories, pull requests, diffs, and agent-generated code for reward hacking, fake completion, defensive theater, architectural bypasses, weakened guarantees, hidden fallbacks, and misleading abstractions.

98

1.09x
Quality

97%

Does it follow best practices?

Impact

100%

1.09x

Average score across 6 eval scenarios

SecuritybySnyk

Passed

No known issues

Overview
Quality
Evals
Security
Files

collect-signals.pyscripts/

#!/usr/bin/env python3
"""Collect lightweight implementation-integrity leads from source files."""

from __future__ import annotations

import argparse
import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Iterable


SKIP_DIRS = {
    ".git",
    ".hg",
    ".svn",
    ".venv",
    "venv",
    "node_modules",
    "dist",
    "build",
    "coverage",
    "__pycache__",
}

SOURCE_SUFFIXES = {
    ".py",
    ".js",
    ".jsx",
    ".ts",
    ".tsx",
    ".go",
    ".rs",
    ".java",
    ".kt",
    ".cs",
    ".rb",
    ".php",
    ".sh",
}


@dataclass(frozen=True)
class Rule:
    category: str
    severity: str
    confidence: str
    pattern: re.Pattern[str]
    rationale: str


@dataclass(frozen=True)
class IntegritySignal:
    category: str
    severity: str
    confidence: str
    evidence: str
    rationale: str
    file: str
    line: int


RULES = [
    Rule(
        "Fake Completion",
        "High",
        "Medium",
        re.compile(r"\breturn\s+(True|true|\{\s*success\s*:\s*true\s*\})\s*(#.*)?$"),
        "Hardcoded success return may hide missing work or skipped side effects.",
    ),
    Rule(
        "Reward Hacking",
        "Critical",
        "Medium",
        re.compile(r"\b(test|fixture|snapshot|golden|pytest|unittest)\b.*\b(return|if|case)\b", re.I),
        "Code appears to branch on test or fixture concepts.",
    ),
    Rule(
        "Defensive Theater",
        "High",
        "Medium",
        re.compile(r"\bexcept\s+(Exception|BaseException)\b|\bcatch\s*\([^)]*\)\s*\{?\s*$"),
        "Broad exception handling can hide failures unless it reports and preserves semantics.",
    ),
    Rule(
        "Silent Degradation",
        "High",
        "Medium",
        re.compile(r"\b(pass|return\s+None|return\s+null|return\s+undefined)\b\s*(#\s*(ignore|fallback|optional|best effort))?", re.I),
        "Silent no-op or null fallback may weaken caller guarantees.",
    ),
    Rule(
        "Weakened Validation",
        "High",
        "Medium",
        re.compile(r"\b(skip[_-]?validation|validate\s*=\s*False|validation\s*:\s*false|validate\s*:\s*false)\b", re.I),
        "Validation appears to be disabled or bypassed.",
    ),
    Rule(
        "Placeholder Implementation",
        "Medium",
        "Low",
        re.compile(r"\b(TODO|FIXME|stub|placeholder|not implemented|temporary)\b", re.I),
        "Placeholder language can indicate incomplete production behavior.",
    ),
]


def iter_source_files(root: Path) -> Iterable[Path]:
    if root.is_file():
        if root.suffix in SOURCE_SUFFIXES:
            yield root
        return

    for path in root.rglob("*"):
        if any(part in SKIP_DIRS for part in path.parts):
            continue
        if path.is_file() and path.suffix in SOURCE_SUFFIXES:
            yield path


def scan_file(path: Path) -> list[IntegritySignal]:
    try:
        lines = path.read_text(encoding="utf-8").splitlines()
    except UnicodeDecodeError:
        return []

    signals: list[IntegritySignal] = []
    for line_number, line in enumerate(lines, start=1):
        stripped = line.strip()
        for rule in RULES:
            if rule.pattern.search(stripped):
                signals.append(
                    IntegritySignal(
                        category=rule.category,
                        severity=rule.severity,
                        confidence=rule.confidence,
                        evidence=stripped[:240],
                        rationale=rule.rationale,
                        file=str(path),
                        line=line_number,
                    )
                )
    return signals


def collect(root: Path) -> list[IntegritySignal]:
    signals: list[IntegritySignal] = []
    for path in iter_source_files(root):
        signals.extend(scan_file(path))
    return signals


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("path", nargs="?", default=".", help="Repository path to scan")
    parser.add_argument(
        "--format",
        choices=("json", "text"),
        default="json",
        help="Output format",
    )
    args = parser.parse_args()

    signals = collect(Path(args.path))
    if args.format == "json":
        print(json.dumps([asdict(signal) for signal in signals], indent=2))
    else:
        for signal in signals:
            print(
                f"{signal.file}:{signal.line}: {signal.severity} "
                f"{signal.category}: {signal.evidence}"
            )

    return 1 if signals else 0


if __name__ == "__main__":
    raise SystemExit(main())

SKILL.md

tile.json