Analyze agent sessions against verifier checklists, detect friction points, and create structured verifiers from skills and docs. Produces per-session verdicts and aggregated quality reports.
88
86%
Does it follow best practices?
Impact
97%
2.93xAverage score across 3 eval scenarios
Passed
No known issues
#!/usr/bin/env python3
"""
Discover verifiers in installed tiles.
Scans .tessl/tiles/<org>/<name>/ for verifiers/ directories anywhere in the
tile tree — at the root, inside skill directories, or any other subdirectory.
my-tile/
tile.json
verifiers/ # root-level verifiers
use-tailwind.json
skills/
my-skill/
verifiers/ # skill-level verifiers
run-tests.json
No external dependencies.
Usage:
python3 discover_verifiers.py [--cwd <path>] [--tiles-dir <path>]
When --tiles-dir is not given, searches both <cwd>/.tessl/tiles and
~/.tessl/tiles, preferring local tiles when the same tile name appears in both.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
def _find_verifier_dirs(tile_dir: Path) -> list[Path]:
"""Find all verifiers/ directories anywhere under a tile root."""
dirs = []
for dirpath, dirnames, _ in os.walk(tile_dir):
# Skip hidden directories
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
p = Path(dirpath)
if p.name == "verifiers" and p != tile_dir:
dirs.append(p)
# Don't recurse into verifiers/ itself
dirnames.clear()
return sorted(dirs)
def discover_tiles(tiles_dir: Path) -> list[dict]:
"""Walk .tessl/tiles/<org>/<name>/ looking for verifiers/ directories.
Searches the entire tile tree, not just the root — verifiers can live
at the tile root, inside skill subdirectories, or anywhere else.
Trust boundary: tiles are user-installed via `tessl install` from the
registry, not fetched from arbitrary third-party sources at runtime.
"""
tiles = []
if not tiles_dir.exists():
return tiles
for org_dir in sorted(tiles_dir.iterdir()):
if not org_dir.is_dir() or org_dir.name.startswith("."):
continue
for tile_dir in sorted(org_dir.iterdir()):
if not tile_dir.is_dir() or tile_dir.name.startswith("."):
continue
verifier_dirs = _find_verifier_dirs(tile_dir)
if not verifier_dirs:
continue
# Read tile name from tile.json, fall back to dir path
name = f"{org_dir.name}/{tile_dir.name}"
tile_json = tile_dir / "tile.json"
if tile_json.exists():
try:
data = json.loads(tile_json.read_text())
name = data.get("name", name)
except (json.JSONDecodeError, OSError):
pass
# Collect verifier files from all verifiers/ directories
all_verifier_files = []
for vdir in verifier_dirs:
all_verifier_files.extend(
f for f in sorted(vdir.glob("*.json"))
if not f.name.startswith("_")
)
if not all_verifier_files:
continue
verifier_count = 0
with_checklist = 0
empty_checklist = 0
for vf in all_verifier_files:
try:
vdata = json.loads(vf.read_text())
# Handle both dict format and list-of-verifiers format
if isinstance(vdata, list):
items = vdata
else:
items = [vdata]
for item in items:
if not isinstance(item, dict):
continue
verifier_count += 1
cl = item.get("checklist", [])
if isinstance(cl, list) and len(cl) > 0:
with_checklist += 1
else:
empty_checklist += 1
except (json.JSONDecodeError, OSError):
empty_checklist += 1
tiles.append({
"name": name,
"verifier_dirs": [str(d.resolve()) for d in verifier_dirs],
"verifier_count": verifier_count,
"with_checklist": with_checklist,
"empty_checklist": empty_checklist,
})
return tiles
def resolve_tile_dirs(explicit: str | None = None, cwd: Path | None = None) -> list[Path]:
"""Return tile directories to search, in priority order.
When --tiles-dir is given explicitly, use only that. Otherwise check
both the local project dir and the global ~/.tessl/tiles, preferring
local (so local tiles shadow global ones with the same name).
"""
if explicit:
p = Path(explicit).resolve()
return [p] if p.exists() else []
cwd = cwd or Path(os.getcwd()).resolve()
candidates = [
cwd / ".tessl" / "tiles",
Path.home() / ".tessl" / "tiles",
]
return [p for p in candidates if p.exists()]
def discover_all(tiles_dirs: list[Path]) -> list[dict]:
"""Discover tiles across multiple tile directories, deduplicating by name (first wins)."""
seen: set[str] = set()
all_tiles: list[dict] = []
for td in tiles_dirs:
for tile in discover_tiles(td):
if tile["name"] not in seen:
seen.add(tile["name"])
all_tiles.append(tile)
return all_tiles
def main() -> None:
parser = argparse.ArgumentParser(
description="Discover verifiers in installed tiles")
parser.add_argument("--cwd", default=os.getcwd(),
help="Project root (default: cwd)")
parser.add_argument("--tiles-dir",
help="Override tiles directory (default: auto-detect local + global)")
args = parser.parse_args()
cwd = Path(args.cwd).resolve()
tiles_dirs = resolve_tile_dirs(args.tiles_dir, cwd)
if not tiles_dirs:
searched = args.tiles_dir or f"{cwd / '.tessl' / 'tiles'} and {Path.home() / '.tessl' / 'tiles'}"
print(f"No tiles directory found. Searched: {searched}", file=sys.stderr)
sys.exit(1)
tiles = discover_all(tiles_dirs)
manifest = {
"tiles_dirs": [str(d) for d in tiles_dirs],
"tiles": tiles,
"total_verifiers": sum(t["verifier_count"] for t in tiles),
"total_tiles": len(tiles),
}
print(json.dumps(manifest, indent=2))
# Summary to stderr
if tiles:
print(f"\nTiles with verifiers:", file=sys.stderr)
for t in tiles:
detail = f"{t['with_checklist']} with checklists"
if t["empty_checklist"]:
detail += f", {t['empty_checklist']} empty"
print(f" {t['name']} — {t['verifier_count']} verifiers ({detail})",
file=sys.stderr)
total = sum(t["verifier_count"] for t in tiles)
print(f"\nTotal: {total} verifiers across {len(tiles)} tile(s)",
file=sys.stderr)
else:
print("\nNo tiles with verifiers found.", file=sys.stderr)
print(
"\nTo create verifiers, use the create-verifiers skill "
"included in this tile.",
file=sys.stderr,
)
if __name__ == "__main__":
main()