Collect and normalize agent logs, discover installed verifiers, and dispatch LLM judges to evaluate adherence. Produces per-session verdicts and aggregated reports.
91
90%
Does it follow best practices?
Impact
96%
3.09xAverage score across 3 eval scenarios
Passed
No known issues
#!/usr/bin/env python3
"""
Validate verifier JSON files against the checklist-based schema.
Checks:
- Required top-level fields (instruction, relevant_when, context, checklist)
- Optional sources array structure (omit when embedded in a skill dir)
- Checklist item structure (name, rule, relevant_when)
- Unique checklist names within each file
- File naming conventions
No external dependencies.
"""
import json
import sys
from pathlib import Path
REQUIRED_TOP = {"instruction", "relevant_when", "context", "checklist"}
REQUIRED_SOURCE = {"type"}
VALID_SOURCE_TYPES = {"file", "user"}
REQUIRED_CHECK = {"name", "rule", "relevant_when"}
def validate_file(path: Path) -> list[str]:
"""Validate a single verifier file. Returns list of error strings."""
errors = []
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as e:
return [f"Invalid JSON: {e}"]
except OSError as e:
return [f"Cannot read: {e}"]
if not isinstance(data, dict):
return ["Top level must be a JSON object"]
# Required fields
for field in REQUIRED_TOP:
if field not in data:
errors.append(f"Missing required field: {field}")
# Type checks
for field in ("instruction", "relevant_when", "context"):
if field in data and not isinstance(data[field], str):
errors.append(f"{field} must be a string")
# Sources (optional — omit when verifier is embedded in a skill directory)
sources = data.get("sources")
if sources is None:
sources = [] # skip validation
elif not isinstance(sources, list):
errors.append("sources must be an array")
else:
for i, src in enumerate(sources):
if not isinstance(src, dict):
errors.append(f"sources[{i}] must be an object")
continue
for field in REQUIRED_SOURCE:
if field not in src:
errors.append(f"sources[{i}] missing: {field}")
src_type = src.get("type")
if src_type and src_type not in VALID_SOURCE_TYPES:
errors.append(
f"sources[{i}].type '{src_type}' not in {VALID_SOURCE_TYPES}"
)
if src_type == "file" and "filename" not in src:
errors.append(f"sources[{i}] type=file but missing filename")
# Checklist
checklist = data.get("checklist", [])
if not isinstance(checklist, list):
errors.append("checklist must be an array")
else:
names_seen = set()
for i, item in enumerate(checklist):
if not isinstance(item, dict):
errors.append(f"checklist[{i}] must be an object")
continue
for field in REQUIRED_CHECK:
if field not in item:
errors.append(f"checklist[{i}] missing: {field}")
for field in REQUIRED_CHECK:
if field in item and not isinstance(item[field], str):
errors.append(f"checklist[{i}].{field} must be a string")
name = item.get("name", "")
if name:
if name in names_seen:
errors.append(f"checklist[{i}] duplicate name: {name}")
names_seen.add(name)
return errors
def validate_dir(verifiers_dir: Path) -> dict:
"""Validate all verifier files in a directory. Returns summary."""
results = {"total": 0, "valid": 0, "invalid": 0, "errors": {}}
json_files = sorted(verifiers_dir.glob("*.json"))
if not json_files:
print(f"No JSON files found in {verifiers_dir}")
return results
for path in json_files:
if path.name.startswith("_"):
continue
results["total"] += 1
errors = validate_file(path)
if errors:
results["invalid"] += 1
results["errors"][path.name] = errors
else:
results["valid"] += 1
return results
def main():
if len(sys.argv) < 2:
print("Usage: validate_verifiers.py <verifiers-dir>")
print(" Validates all JSON files in the directory against the verifier schema.")
sys.exit(1)
verifiers_dir = Path(sys.argv[1])
if not verifiers_dir.is_dir():
print(f"Error: {verifiers_dir} is not a directory")
sys.exit(1)
results = validate_dir(verifiers_dir)
print(f"\nValidation: {results['total']} files checked")
print(f" Valid: {results['valid']}")
print(f" Invalid: {results['invalid']}")
if results["errors"]:
print("\nErrors:")
for filename, errors in results["errors"].items():
print(f"\n {filename}:")
for error in errors:
print(f" - {error}")
sys.exit(1)
else:
print("\nAll verifier files are valid.")
if __name__ == "__main__":
main()