Automated pipeline that takes a company name and produces a custom Tessl skill plus an eval report showing per-scenario lift (baseline agent vs with-skill agent). A1 MVP cell of the produce/consume × personalization 2x2.
88
86%
Does it follow best practices?
Impact
89%
1.45xAverage score across 13 eval scenarios
Advisory
Suggest reviewing before use
#!/usr/bin/env python3
"""Validate a selection.json artifact against its linked discovery.json.
Reads a selection file path as the first argument; loads the linked discovery
file referenced by `discovery_path`; checks `selected_target_id` (when present)
references a real entry in `discovery.skill_targets[]`. Prints a JSON
validation report; exits 0 on success, 1 on validation failure, 2 on usage error.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
ALLOWED_STATUS = {"selected", "skipped"}
def fail(msg: str) -> None:
print(f"validate-selection.py: {msg}", file=sys.stderr)
sys.exit(2)
def load_json(path: Path) -> dict[str, Any]:
if not path.is_file():
fail(f"file not found: {path}")
try:
return json.loads(path.read_text())
except json.JSONDecodeError as exc:
fail(f"invalid JSON in {path}: {exc}")
def collect_errors(sel: dict[str, Any]) -> list[str]:
errors: list[str] = []
if sel.get("schema_version") != 1:
errors.append(f"schema_version must be 1, got {sel.get('schema_version')!r}")
status = sel.get("selection_status")
if status not in ALLOWED_STATUS:
errors.append(f"selection_status must be in {sorted(ALLOWED_STATUS)}, got {status!r}")
if status == "selected":
if not sel.get("selected_target_id"):
errors.append("selection_status=selected requires selected_target_id")
elif status == "skipped":
if sel.get("selected_target_id"):
errors.append("selection_status=skipped must not have selected_target_id")
if not sel.get("selection_rationale"):
errors.append("selection_status=skipped requires selection_rationale")
if not sel.get("discovery_path"):
errors.append("missing discovery_path")
if not sel.get("selected_at"):
errors.append("missing selected_at")
return errors
def main() -> int:
if len(sys.argv) < 2:
fail("usage: validate-selection.py <selection.json>")
sel_path = Path(sys.argv[1])
sel = load_json(sel_path)
errors = collect_errors(sel)
discovery_path_str = sel.get("discovery_path")
target_ids: set[str] = set()
if discovery_path_str:
disc_path = Path(discovery_path_str)
if not disc_path.is_file():
errors.append(f"discovery_path references missing file: {disc_path}")
else:
try:
disc = json.loads(disc_path.read_text())
except json.JSONDecodeError as exc:
errors.append(f"discovery_path file is not valid JSON: {exc}")
else:
target_ids = {
t.get("id")
for t in (disc.get("skill_targets") or [])
if t.get("id")
}
if sel.get("selection_status") == "selected":
target_id = sel.get("selected_target_id")
if target_id and target_id not in target_ids:
errors.append(
f"selected_target_id {target_id!r} not in discovery.skill_targets[]; "
f"available: {sorted(target_ids)}"
)
report = {
"ok": not errors,
"errors": errors,
"selection_status": sel.get("selection_status"),
"selected_target_id": sel.get("selected_target_id"),
}
print(json.dumps(report, indent=2))
return 0 if not errors else 1
if __name__ == "__main__":
sys.exit(main())evals
scenario-1
scenario-2
scenario-3
scenario-4
scenario-5
scenario-6
scenario-7
scenario-8
scenario-9
scenario-10
scenario-11
scenario-12
scenario-13
skills
batch-driver
build-and-evaluate
company-list-filter
discovery
discovery-produce
select-target