CtrlK
BlogDocsLog inGet started
Tessl Logo
PluginPublic

tessl/factory-setup

Set up, understand, and change a Tessl factory on a repository: pick automations that fit how you work, such as shipping tickets as pull requests, answering reviews and fixing checks on those pull requests, code review, and scheduled jobs; prepare the repository for agents; write issues for the factory to build; then see what the factory has done, find out why a run failed or never started, and change or extend it.

Quality

91%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Medium

Suggest reviewing before use

Overview
Quality
Evals
Security
Files

build-report.mjsskills/prepare-repo/scripts/

#!/usr/bin/env node
// Builds the agent-readiness report from the structured assessment JSON.
//
// The agent discovers the repo's apps and, for each criterion, judges how many
// of the applicable apps satisfy it (passingApps / applicableApps). Every
// aggregate — per-category pass rates, the app-weighted overall score, the
// maturity level, and the summary counts — is computed here so the numbers are
// always arithmetically consistent with the per-app judgements, never eyeballed.
//
// Usage: node build-report.mjs [input.json] [output.md]
//   input  defaults to agent-readiness.json
//   output defaults to agent-readiness-report.md; it must be a file name in the
//   current directory
// Prints the one-line summary (level + overall score) to stdout.

import { lstatSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
import {
  basename,
  dirname,
  isAbsolute,
  join,
  relative,
  resolve,
  sep,
} from 'node:path';
import { fileURLToPath } from 'node:url';

// The published rubric (references/rubric.md): each category and the exact
// criteria it scores. An assessment must judge every criterion once, by these
// names, so a score cannot rise by dropping a failure or adding a pass.
export const RUBRIC = {
  'Style & Validation': [
    'Linter Configured',
    'Code Formatter',
    'Type Checker / Strict Typing',
    'Pre-commit Hooks',
    'Dead Code Detection',
  ],
  'Build System': [
    'Single Command Setup',
    'Dependencies Pinned',
    'Build Command Documented',
    'Task / Monorepo Tooling',
    'CI Configured',
  ],
  Testing: [
    'Unit Tests Exist',
    'Tests Runnable',
    'Integration / E2E Tests',
    'Test File Conventions',
    'Coverage Thresholds',
  ],
  Documentation: [
    'README',
    'Agent Context File',
    'Architecture Docs',
    'API / Schema Docs',
    'Documentation Freshness',
  ],
  'Development Environment': [
    'Environment Template',
    'Local Services Setup',
    'Database Schema & Migrations',
    'Dev Container',
  ],
  Observability: [
    'Structured Logging',
    'Health Checks',
    'Metrics / Telemetry',
    'Error Tracking',
    'Distributed Tracing',
  ],
  Security: [
    'Secret Scanning',
    'Dependency Update Automation',
    'Gitignore Comprehensive',
    'Secrets Management',
    'CODEOWNERS',
  ],
  'Agent Readiness': [
    'Agent Context Quality',
    'Skills / Reusable Context',
    'MCP Configuration',
    'Editor / Agent Rules',
    'Task Discovery',
  ],
};

const CATEGORY_NAMES = Object.keys(RUBRIC);

// Validation failures throw; main() converts them to a clean exit(1). Throwing
// (rather than exiting inline) keeps parseAssessment testable.
function fail(message) {
  throw new Error(message);
}

const isNonNegInt = (n) => Number.isInteger(n) && n >= 0;

/**
 * A criterion's status derives from its per-app counts:
 * - `skipped`  — no app the criterion applies to (excluded from scores)
 * - `passed`   — every applicable app satisfies it
 * - `failed`   — some or no applicable apps satisfy it
 */
export function statusOf(criterion) {
  if (criterion.applicableApps === 0) return 'skipped';
  return criterion.passingApps === criterion.applicableApps
    ? 'passed'
    : 'failed';
}

/**
 * Throw unless `category` judges each of its rubric criteria exactly once, by
 * the rubric's name, and nothing else.
 */
function checkCriterionSet(category) {
  const expected = RUBRIC[category.name];
  const names = category.criteria.map((criterion) => criterion.name);
  const duplicated = names.filter((name, i) => names.indexOf(name) !== i);
  const unknown = names.filter((name) => !expected.includes(name));
  const missing = expected.filter((name) => !names.includes(name));
  const problems = [
    ['missing', missing],
    ['unknown', unknown],
    ['duplicated', [...new Set(duplicated)]],
  ]
    .filter(([, list]) => list.length > 0)
    .map(
      ([label, list]) =>
        `${label} ${list.map((n) => JSON.stringify(n)).join(', ')}`,
    );
  if (problems.length > 0) {
    fail(
      `category "${category.name}" must judge each rubric criterion exactly once: ${problems.join('; ')}`,
    );
  }
}

/** Parse and structurally validate the assessment JSON. Throws on bad input. */
export function parseAssessment(raw) {
  let data;
  try {
    data = JSON.parse(raw);
  } catch (err) {
    fail(`input is not valid JSON: ${err.message}`);
  }
  if (
    !data ||
    !Array.isArray(data.apps) ||
    data.apps.length === 0 ||
    !data.apps.every((a) => typeof a === 'string' && a.length > 0)
  ) {
    fail('input must have a non-empty "apps" array of app names');
  }
  if (!Array.isArray(data.categories)) {
    fail('input must have a "categories" array');
  }
  const appCount = data.apps.length;
  if (new Set(data.apps).size !== appCount) {
    fail('"apps" must contain unique app names');
  }
  const categoryNames = data.categories.map((category) => category?.name);
  if (
    categoryNames.length !== CATEGORY_NAMES.length ||
    new Set(categoryNames).size !== CATEGORY_NAMES.length ||
    CATEGORY_NAMES.some((name) => !categoryNames.includes(name))
  ) {
    fail('input must contain each readiness category exactly once');
  }
  for (const category of data.categories) {
    if (!CATEGORY_NAMES.includes(category?.name)) {
      fail(`unknown category name: ${JSON.stringify(category?.name)}`);
    }
    if (!Array.isArray(category.criteria) || category.criteria.length === 0) {
      fail(`category "${category.name}" has no criteria`);
    }
    for (const criterion of category.criteria) {
      if (typeof criterion?.name !== 'string' || criterion.name.length === 0) {
        fail(`a criterion in "${category.name}" is missing a name`);
      }
    }
    checkCriterionSet(category);
    for (const criterion of category.criteria) {
      if (
        !isNonNegInt(criterion.applicableApps) ||
        !isNonNegInt(criterion.passingApps) ||
        criterion.applicableApps > appCount ||
        criterion.passingApps > criterion.applicableApps
      ) {
        fail(
          `criterion "${criterion.name}" needs 0 <= passingApps <= applicableApps <= ${appCount}`,
        );
      }
      if (
        typeof criterion.rationale !== 'string' ||
        criterion.rationale.length === 0
      ) {
        fail(`criterion "${criterion.name}" is missing a rationale`);
      }
      if (
        statusOf(criterion) === 'failed' &&
        (typeof criterion.action !== 'string' || criterion.action.length === 0)
      ) {
        fail(`failed criterion "${criterion.name}" is missing an action`);
      }
    }
  }
  if (data.recommendations !== undefined) {
    if (!Array.isArray(data.recommendations)) {
      fail('"recommendations" must be an array when present');
    }
    for (const rec of data.recommendations) {
      if (
        typeof rec?.title !== 'string' ||
        rec.title.length === 0 ||
        typeof rec?.detail !== 'string' ||
        rec.detail.length === 0
      ) {
        fail('each recommendation needs a non-empty title and detail');
      }
    }
  }
  return data;
}

// null when no app is applicable — that is "not applicable", not "0% passing".
const ratio = (passing, applicable) =>
  applicable === 0 ? null : passing / applicable;

function levelFor(overallScore) {
  if (overallScore < 0.4) return 1;
  if (overallScore < 0.6) return 2;
  if (overallScore < 0.8) return 3;
  if (overallScore < 0.92) return 4;
  return 5;
}

/**
 * Compute every aggregate from the per-app criterion counts. Scores are
 * app-weighted: a criterion satisfied by 7 of 12 apps contributes 7/12, so
 * partial adoption across a monorepo pulls the total down.
 */
export function computeRollup(assessment) {
  let totalPassing = 0;
  let totalApplicable = 0;
  const summary = { total: 0, passed: 0, failed: 0, skipped: 0 };

  const categories = assessment.categories.map((category) => {
    let passing = 0;
    let applicable = 0;
    for (const criterion of category.criteria) {
      passing += criterion.passingApps;
      applicable += criterion.applicableApps;
      summary.total += 1;
      summary[statusOf(criterion)] += 1;
    }
    totalPassing += passing;
    totalApplicable += applicable;
    return {
      name: category.name,
      criteria: category.criteria,
      passRate: ratio(passing, applicable),
    };
  });

  const overallScore = ratio(totalPassing, totalApplicable);
  return {
    level: overallScore === null ? null : levelFor(overallScore),
    overallScore,
    appsAssessed: assessment.apps.length,
    categories,
    summary,
  };
}

const pct = (fraction) =>
  fraction === null ? 'n/a' : `${Math.round(fraction * 100)}%`;
const levelLabel = (level) => (level === null ? 'n/a' : `${level}/5`);
const STATUS_LABEL = { passed: 'Passed', failed: 'Failed', skipped: 'Skipped' };
// Flatten a value into a single markdown table cell.
const cell = (text) => text.replaceAll('\n', ' ').replaceAll('|', '\\|');
// "7 / 12", or "n/a" when the criterion applies to no app.
const coverage = (crit) =>
  crit.applicableApps === 0
    ? 'n/a'
    : `${crit.passingApps} / ${crit.applicableApps}`;

export function renderReport(assessment, rollup, generatedAt) {
  const repo = assessment.repo ?? 'repository';
  const lines = [
    `# Agent Readiness Report: ${repo}`,
    '',
    `**Level:** ${levelLabel(rollup.level)}`,
    `**Overall Score:** ${pct(rollup.overallScore)}`,
    `**Apps assessed:** ${rollup.appsAssessed} (${assessment.apps.map(cell).join(', ')})`,
    `**Generated:** ${generatedAt}`,
    '',
    'Coverage is the number of applicable apps that satisfy each criterion. The',
    'overall score is app-weighted, so a capability adopted in only some apps',
    'scores partially.',
    '',
    '## Summary',
    '',
    '| Metric | Value |',
    '|--------|-------|',
    `| Total Criteria | ${rollup.summary.total} |`,
    `| Passed (all apps) | ${rollup.summary.passed} |`,
    `| Failed (some/no apps) | ${rollup.summary.failed} |`,
    `| Skipped (n/a) | ${rollup.summary.skipped} |`,
    '',
    '## Pass Rate by Category',
    '',
    '| Category | Pass Rate |',
    '|----------|-----------|',
    ...rollup.categories.map((c) => `| ${c.name} | ${pct(c.passRate)} |`),
  ];

  for (const category of rollup.categories) {
    lines.push(
      '',
      `## ${category.name}`,
      '',
      '| Criterion | Coverage | Status | Rationale |',
      '|-----------|----------|--------|-----------|',
      ...category.criteria.map(
        (crit) =>
          `| ${cell(crit.name)} | ${coverage(crit)} | ${STATUS_LABEL[statusOf(crit)]} | ${cell(crit.rationale)} |`,
      ),
    );
  }

  const recommendations = assessment.recommendations ?? [];
  if (recommendations.length > 0) {
    lines.push('', '## Recommended actions', '');
    recommendations.forEach((rec, i) => {
      lines.push(`${i + 1}. **${rec.title}** — ${rec.detail}`);
    });
  }

  const gaps = rollup.categories.flatMap((category) =>
    category.criteria
      .filter((crit) => statusOf(crit) === 'failed')
      .map((crit) => ({ category: category.name, crit })),
  );
  if (gaps.length > 0) {
    lines.push(
      '',
      '## Gaps → actions',
      '',
      '| Category | Criterion | Coverage | Action |',
      '|----------|-----------|----------|--------|',
      ...gaps.map(
        ({ category, crit }) =>
          `| ${cell(category)} | ${cell(crit.name)} | ${coverage(crit)} | ${cell(crit.action)} |`,
      ),
    );
  }

  lines.push('');
  return lines.join('\n');
}

export function resolveOutputPath(outputPath, cwd = process.cwd()) {
  if (isAbsolute(outputPath)) {
    fail('output path must be relative to the current working directory');
  }

  const resolvedCwd = resolve(cwd);
  const resolvedOutputPath = resolve(resolvedCwd, outputPath);
  const relativeOutputPath = relative(resolvedCwd, resolvedOutputPath);
  if (
    relativeOutputPath === '..' ||
    relativeOutputPath.startsWith(`..${sep}`) ||
    isAbsolute(relativeOutputPath)
  ) {
    fail('output path must remain within the current working directory');
  }

  let outputIsSymlink = false;
  try {
    outputIsSymlink = lstatSync(resolvedOutputPath).isSymbolicLink();
  } catch (err) {
    if (err.code !== 'ENOENT') throw err;
  }

  let canonicalOutputPath;
  try {
    canonicalOutputPath = realpathSync(resolvedOutputPath);
  } catch (err) {
    if (err.code !== 'ENOENT') throw err;
    if (outputIsSymlink) {
      fail('output path must not be a dangling symbolic link');
    }
    canonicalOutputPath = resolve(
      realpathSync(dirname(resolvedOutputPath)),
      basename(resolvedOutputPath),
    );
  }

  const canonicalCwd = realpathSync(resolvedCwd);
  const relativeCanonicalOutputPath = relative(
    canonicalCwd,
    canonicalOutputPath,
  );
  if (
    relativeCanonicalOutputPath === '..' ||
    relativeCanonicalOutputPath.startsWith(`..${sep}`) ||
    isAbsolute(relativeCanonicalOutputPath)
  ) {
    fail('output path must remain within the current working directory');
  }

  if (
    canonicalOutputPath === canonicalCwd ||
    dirname(canonicalOutputPath) !== canonicalCwd
  ) {
    fail('output path must be a file name in the current working directory');
  }

  // Rebuild from the trusted root and a single name segment so the result is
  // not the command-line argument itself.
  return join(canonicalCwd, basename(canonicalOutputPath));
}

function main() {
  const inputPath = process.argv[2] ?? 'agent-readiness.json';
  const outputPath = process.argv[3] ?? 'agent-readiness-report.md';
  const resolvedOutputPath = resolveOutputPath(outputPath);

  let raw;
  try {
    raw = readFileSync(inputPath, 'utf8');
  } catch (err) {
    throw new Error(`cannot read ${inputPath}: ${err.message}`);
  }

  const assessment = parseAssessment(raw);
  const rollup = computeRollup(assessment);
  const report = renderReport(assessment, rollup, new Date().toISOString());
  writeFileSync(resolvedOutputPath, report);

  console.log(
    `Level ${levelLabel(rollup.level)} — Overall ${pct(rollup.overallScore)} across ${rollup.appsAssessed} apps (${rollup.summary.passed} passed, ${rollup.summary.failed} failed, ${rollup.summary.skipped} skipped of ${rollup.summary.total} criteria). Report written to ${outputPath}.`,
  );
}

// Run only as a CLI, so the pure functions above can be imported by tests.
// Compare real paths: an installed skill is often reached through a symlink,
// and Node resolves import.meta.url to the real path but not argv[1].
function isEntryPoint() {
  try {
    return (
      realpathSync(fileURLToPath(import.meta.url)) ===
      realpathSync(process.argv[1])
    );
  } catch {
    return false;
  }
}

if (process.argv[1] && isEntryPoint()) {
  try {
    main();
  } catch (err) {
    console.error(`build-report: ${err.message}`);
    process.exit(1);
  }
}

README.md

tile.json