CtrlK
BlogDocsLog inGet started
Tessl Logo

try-tessl/agent-quality

Analyze agent sessions against verifier checklists, detect friction points, and create structured verifiers from skills and docs. Produces per-session verdicts and aggregated quality reports.

88

2.93x
Quality

86%

Does it follow best practices?

Impact

97%

2.93x

Average score across 3 eval scenarios

SecuritybySnyk

Passed

No known issues

Overview
Quality
Evals
Security
Files

discover_verifiers.pyskills/analyze-sessions/scripts/

#!/usr/bin/env python3
"""
Discover verifiers in installed tiles.

Scans .tessl/tiles/<org>/<name>/ for verifiers/ directories anywhere in the
tile tree — at the root, inside skill directories, or any other subdirectory.

    my-tile/
      tile.json
      verifiers/                    # root-level verifiers
        use-tailwind.json
      skills/
        my-skill/
          verifiers/                # skill-level verifiers
            run-tests.json

No external dependencies.

Usage:
    python3 discover_verifiers.py [--cwd <path>] [--tiles-dir <path>]

When --tiles-dir is not given, searches both <cwd>/.tessl/tiles and
~/.tessl/tiles, preferring local tiles when the same tile name appears in both.
"""

from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path


def _find_verifier_dirs(tile_dir: Path) -> list[Path]:
    """Find all verifiers/ directories anywhere under a tile root."""
    dirs = []
    for dirpath, dirnames, _ in os.walk(tile_dir):
        # Skip hidden directories
        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
        p = Path(dirpath)
        if p.name == "verifiers" and p != tile_dir:
            dirs.append(p)
            # Don't recurse into verifiers/ itself
            dirnames.clear()
    return sorted(dirs)


def discover_tiles(tiles_dir: Path) -> list[dict]:
    """Walk .tessl/tiles/<org>/<name>/ looking for verifiers/ directories.

    Searches the entire tile tree, not just the root — verifiers can live
    at the tile root, inside skill subdirectories, or anywhere else.

    Trust boundary: tiles are user-installed via `tessl install` from the
    registry, not fetched from arbitrary third-party sources at runtime.
    """
    tiles = []
    if not tiles_dir.exists():
        return tiles

    for org_dir in sorted(tiles_dir.iterdir()):
        if not org_dir.is_dir() or org_dir.name.startswith("."):
            continue
        for tile_dir in sorted(org_dir.iterdir()):
            if not tile_dir.is_dir() or tile_dir.name.startswith("."):
                continue

            verifier_dirs = _find_verifier_dirs(tile_dir)
            if not verifier_dirs:
                continue

            # Read tile name from tile.json, fall back to dir path
            name = f"{org_dir.name}/{tile_dir.name}"
            tile_json = tile_dir / "tile.json"
            if tile_json.exists():
                try:
                    data = json.loads(tile_json.read_text())
                    name = data.get("name", name)
                except (json.JSONDecodeError, OSError):
                    pass

            # Collect verifier files from all verifiers/ directories
            all_verifier_files = []
            for vdir in verifier_dirs:
                all_verifier_files.extend(
                    f for f in sorted(vdir.glob("*.json"))
                    if not f.name.startswith("_")
                )

            if not all_verifier_files:
                continue

            verifier_count = 0
            with_checklist = 0
            empty_checklist = 0
            for vf in all_verifier_files:
                try:
                    vdata = json.loads(vf.read_text())
                    # Handle both dict format and list-of-verifiers format
                    if isinstance(vdata, list):
                        items = vdata
                    else:
                        items = [vdata]
                    for item in items:
                        if not isinstance(item, dict):
                            continue
                        verifier_count += 1
                        cl = item.get("checklist", [])
                        if isinstance(cl, list) and len(cl) > 0:
                            with_checklist += 1
                        else:
                            empty_checklist += 1
                except (json.JSONDecodeError, OSError):
                    empty_checklist += 1

            tiles.append({
                "name": name,
                "verifier_dirs": [str(d.resolve()) for d in verifier_dirs],
                "verifier_count": verifier_count,
                "with_checklist": with_checklist,
                "empty_checklist": empty_checklist,
            })

    return tiles


def resolve_tile_dirs(explicit: str | None = None, cwd: Path | None = None) -> list[Path]:
    """Return tile directories to search, in priority order.

    When --tiles-dir is given explicitly, use only that.  Otherwise check
    both the local project dir and the global ~/.tessl/tiles, preferring
    local (so local tiles shadow global ones with the same name).
    """
    if explicit:
        p = Path(explicit).resolve()
        return [p] if p.exists() else []

    cwd = cwd or Path(os.getcwd()).resolve()
    candidates = [
        cwd / ".tessl" / "tiles",
        Path.home() / ".tessl" / "tiles",
    ]
    return [p for p in candidates if p.exists()]


def discover_all(tiles_dirs: list[Path]) -> list[dict]:
    """Discover tiles across multiple tile directories, deduplicating by name (first wins)."""
    seen: set[str] = set()
    all_tiles: list[dict] = []
    for td in tiles_dirs:
        for tile in discover_tiles(td):
            if tile["name"] not in seen:
                seen.add(tile["name"])
                all_tiles.append(tile)
    return all_tiles


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Discover verifiers in installed tiles")
    parser.add_argument("--cwd", default=os.getcwd(),
                        help="Project root (default: cwd)")
    parser.add_argument("--tiles-dir",
                        help="Override tiles directory (default: auto-detect local + global)")
    args = parser.parse_args()

    cwd = Path(args.cwd).resolve()
    tiles_dirs = resolve_tile_dirs(args.tiles_dir, cwd)

    if not tiles_dirs:
        searched = args.tiles_dir or f"{cwd / '.tessl' / 'tiles'} and {Path.home() / '.tessl' / 'tiles'}"
        print(f"No tiles directory found. Searched: {searched}", file=sys.stderr)
        sys.exit(1)

    tiles = discover_all(tiles_dirs)

    manifest = {
        "tiles_dirs": [str(d) for d in tiles_dirs],
        "tiles": tiles,
        "total_verifiers": sum(t["verifier_count"] for t in tiles),
        "total_tiles": len(tiles),
    }

    print(json.dumps(manifest, indent=2))

    # Summary to stderr
    if tiles:
        print(f"\nTiles with verifiers:", file=sys.stderr)
        for t in tiles:
            detail = f"{t['with_checklist']} with checklists"
            if t["empty_checklist"]:
                detail += f", {t['empty_checklist']} empty"
            print(f"  {t['name']} — {t['verifier_count']} verifiers ({detail})",
                  file=sys.stderr)
        total = sum(t["verifier_count"] for t in tiles)
        print(f"\nTotal: {total} verifiers across {len(tiles)} tile(s)",
              file=sys.stderr)
    else:
        print("\nNo tiles with verifiers found.", file=sys.stderr)
        print(
            "\nTo create verifiers, use the create-verifiers skill "
            "included in this tile.",
            file=sys.stderr,
        )


if __name__ == "__main__":
    main()

README.md

tile.json