CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl-labs/audit-logs

Collect and normalize agent logs, discover installed verifiers, and dispatch LLM judges to evaluate adherence. Produces per-session verdicts and aggregated reports.

91

3.09x
Quality

90%

Does it follow best practices?

Impact

96%

3.09x

Average score across 3 eval scenarios

SecuritybySnyk

Passed

No known issues

Overview
Quality
Evals
Security
Files

validate_verifiers.pyskills/create-verifiers/scripts/

#!/usr/bin/env python3
"""
Validate verifier JSON files against the checklist-based schema.

Checks:
  - Required top-level fields (instruction, relevant_when, context, checklist)
  - Optional sources array structure (omit when embedded in a skill dir)
  - Checklist item structure (name, rule, relevant_when)
  - Unique checklist names within each file
  - File naming conventions

No external dependencies.
"""

import json
import sys
from pathlib import Path

REQUIRED_TOP = {"instruction", "relevant_when", "context", "checklist"}
REQUIRED_SOURCE = {"type"}
VALID_SOURCE_TYPES = {"file", "user"}
REQUIRED_CHECK = {"name", "rule", "relevant_when"}


def validate_file(path: Path) -> list[str]:
    """Validate a single verifier file. Returns list of error strings."""
    errors = []
    try:
        data = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError as e:
        return [f"Invalid JSON: {e}"]
    except OSError as e:
        return [f"Cannot read: {e}"]

    if not isinstance(data, dict):
        return ["Top level must be a JSON object"]

    # Required fields
    for field in REQUIRED_TOP:
        if field not in data:
            errors.append(f"Missing required field: {field}")

    # Type checks
    for field in ("instruction", "relevant_when", "context"):
        if field in data and not isinstance(data[field], str):
            errors.append(f"{field} must be a string")

    # Sources (optional — omit when verifier is embedded in a skill directory)
    sources = data.get("sources")
    if sources is None:
        sources = []  # skip validation
    elif not isinstance(sources, list):
        errors.append("sources must be an array")
    else:
        for i, src in enumerate(sources):
            if not isinstance(src, dict):
                errors.append(f"sources[{i}] must be an object")
                continue
            for field in REQUIRED_SOURCE:
                if field not in src:
                    errors.append(f"sources[{i}] missing: {field}")
            src_type = src.get("type")
            if src_type and src_type not in VALID_SOURCE_TYPES:
                errors.append(
                    f"sources[{i}].type '{src_type}' not in {VALID_SOURCE_TYPES}"
                )
            if src_type == "file" and "filename" not in src:
                errors.append(f"sources[{i}] type=file but missing filename")

    # Checklist
    checklist = data.get("checklist", [])
    if not isinstance(checklist, list):
        errors.append("checklist must be an array")
    else:
        names_seen = set()
        for i, item in enumerate(checklist):
            if not isinstance(item, dict):
                errors.append(f"checklist[{i}] must be an object")
                continue
            for field in REQUIRED_CHECK:
                if field not in item:
                    errors.append(f"checklist[{i}] missing: {field}")
            for field in REQUIRED_CHECK:
                if field in item and not isinstance(item[field], str):
                    errors.append(f"checklist[{i}].{field} must be a string")
            name = item.get("name", "")
            if name:
                if name in names_seen:
                    errors.append(f"checklist[{i}] duplicate name: {name}")
                names_seen.add(name)

    return errors


def validate_dir(verifiers_dir: Path) -> dict:
    """Validate all verifier files in a directory. Returns summary."""
    results = {"total": 0, "valid": 0, "invalid": 0, "errors": {}}

    json_files = sorted(verifiers_dir.glob("*.json"))
    if not json_files:
        print(f"No JSON files found in {verifiers_dir}")
        return results

    for path in json_files:
        if path.name.startswith("_"):
            continue
        results["total"] += 1
        errors = validate_file(path)
        if errors:
            results["invalid"] += 1
            results["errors"][path.name] = errors
        else:
            results["valid"] += 1

    return results


def main():
    if len(sys.argv) < 2:
        print("Usage: validate_verifiers.py <verifiers-dir>")
        print("  Validates all JSON files in the directory against the verifier schema.")
        sys.exit(1)

    verifiers_dir = Path(sys.argv[1])
    if not verifiers_dir.is_dir():
        print(f"Error: {verifiers_dir} is not a directory")
        sys.exit(1)

    results = validate_dir(verifiers_dir)

    print(f"\nValidation: {results['total']} files checked")
    print(f"  Valid:   {results['valid']}")
    print(f"  Invalid: {results['invalid']}")

    if results["errors"]:
        print("\nErrors:")
        for filename, errors in results["errors"].items():
            print(f"\n  {filename}:")
            for error in errors:
                print(f"    - {error}")
        sys.exit(1)
    else:
        print("\nAll verifier files are valid.")


if __name__ == "__main__":
    main()

tile.json