CtrlK
BlogDocsLog inGet started
Tessl Logo
PluginPublic

tessl-labs/factory

Skills that Tessl factory loops run: implement a Linear or GitHub issue as a pull request, shepherd that pull request through review and checks, wake the shepherd for pull requests that a push to their base branch left in conflict, and keep the work in step when a person stops, ends, answers or closes it.

Quality

88%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Low

Low-risk findings worth noting

Overview
Quality
Evals
Security
Files

trigger-shepherd-rounds.mjsskills/merge-conflict-sweep/scripts/

#!/usr/bin/env node
// Starts one round of the fixer action the listing names for each
// conflicting factory pull request the agent decided to wake, and writes the run's `launch-result.json` by joining
// the listing's fact sheets, the agent's decisions and the trigger outcomes.
// Reads no pull request or issue; its only writes are `tessl action trigger`
// calls and the result. `main` runs when the file is the entry point.

import { existsSync, readFileSync, readdirSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

import {
  DEFAULT_DIR,
  NOT_REACHED_REASON,
  boundsFrom,
  flagValue,
  parseJson,
  parseUntil,
  readJson,
  realClock,
  runCommand,
  writeJson,
} from './sweep-common.mjs';

export const ISSUE_SOURCES = ['branch', 'description'];
export const DECISIONS = ['trigger', 'skip'];

/** Why a PR whose trigger calls the end time cut off before its own retries ran out is skipped. */
export function cutShortReason(reason) {
  return `${reason}; the script's end time came before its retries ran out, and the next sweep takes it up`;
}

/** One request per PR head: the trigger route records an id once. */
export function requestId(number, headSha) {
  return `merge-conflict:${number}@${headSha}`;
}

/**
 * The per-PR key, held at limit 1, that serialises the fixer's rounds on a PR.
 * The request holds it itself, so its round waits behind those rounds.
 */
export function prConcurrencyKey(number) {
  return `github:pr:${number}`;
}

/**
 * The arguments of the `tessl action trigger` call that wakes `action` for one
 * PR. The link starts the round's run carrying the PR, so a lookup by PR finds
 * sweep-started rounds.
 */
export function triggerArgs(pr, action) {
  return [
    'action',
    'trigger',
    action,
    '--json',
    '--input',
    `GITHUB_PR_NUMBER=${pr.number}`,
    '--request-id',
    requestId(pr.number, pr.headRefOid),
    '--concurrency-key',
    `${prConcurrencyKey(pr.number)}=1`,
    '--link',
    `references:github_pr=${pr.url}`,
  ];
}

/**
 * What is wrong with the agent's decision for a fact sheet, as a list of
 * problems, empty when it is well formed. This checks the decision's shape and
 * that its evidence names what the fact sheet holds, not whether the decision
 * is the right one.
 */
export function decisionProblems(sheet, decision) {
  const at = `decisions/${sheet.number}.json`;
  if (decision === undefined) return [`${at} is missing`];
  if (decision === null || typeof decision !== 'object')
    return [`${at} is not a JSON object`];
  const problems = [];
  if (decision.number !== sheet.number)
    problems.push(`${at} names number ${decision.number}`);
  if (!DECISIONS.includes(decision.decision))
    problems.push(`${at} decision is not one of ${DECISIONS.join(', ')}`);
  if (typeof decision.reason !== 'string' || !decision.reason.trim())
    problems.push(`${at} has no reason`);
  else if (/[\r\n]/.test(decision.reason))
    problems.push(`${at} reason is not one line`);
  if (decision.issue === null) {
    if (decision.decision === 'trigger')
      problems.push(`${at} triggers with no issue`);
    if (decision.issueSource !== null || decision.issueSourceText !== null)
      problems.push(`${at} has no issue but names a source`);
    return problems;
  }
  if (
    typeof decision.issue !== 'string' ||
    !Object.hasOwn(sheet.issues, decision.issue)
  )
    problems.push(`${at} issue ${decision.issue} is not in the fact sheet`);
  if (!ISSUE_SOURCES.includes(decision.issueSource))
    problems.push(
      `${at} issueSource is not one of ${ISSUE_SOURCES.join(', ')}`,
    );
  const quoted = decision.issueSourceText;
  const source =
    decision.issueSource === 'branch' ? [sheet.branch] : sheet.descriptionLines;
  if (typeof quoted !== 'string' || !quoted.trim())
    problems.push(`${at} has no issueSourceText`);
  else if (
    ISSUE_SOURCES.includes(decision.issueSource) &&
    !source.some((text) => text.includes(quoted))
  )
    problems.push(
      `${at} issueSourceText is not quoted from the fact sheet's ${decision.issueSource}`,
    );
  else if (
    typeof decision.issue === 'string' &&
    !(
      namesIssue(quoted, decision.issue) &&
      source.some(
        (text) => text.includes(quoted) && namesIssue(text, decision.issue),
      )
    )
  )
    problems.push(`${at} issueSourceText does not name ${decision.issue}`);
  if (
    decision.issueSource === 'branch' &&
    !isBranchSuffixFor(sheet.branchSuffix, decision.issue)
  )
    problems.push(
      `${at} issue ${decision.issue} is not the branch's whole suffix`,
    );
  if (
    typeof decision.issue === 'string' &&
    !decision.issue.startsWith('#') &&
    /^issue-[1-9][0-9]*$/i.test(sheet.branchSuffix) &&
    decision.issueSource === 'branch'
  )
    problems.push(
      `${at} branch ${sheet.branch} names GitHub issue #${sheet.branchSuffix.slice('issue-'.length)}, not ${decision.issue}`,
    );
  return problems;
}

/**
 * Whether quoted text names the issue: a Linear identifier anywhere in it,
 * ignoring case, or a GitHub issue `#12` as `#12` or as a branch's `issue-12`.
 */
export function namesIssue(text, issue) {
  const github = /^#([1-9][0-9]*)$/.exec(issue);
  if (!github) return text.toUpperCase().includes(issue);
  return new RegExp(`(?:#|\\bissue-)${github[1]}(?![0-9])`, 'i').test(text);
}

/** Whether a branch suffix is the whole name of the issue: `abc-45` for `ABC-45`, `issue-12` for `#12`. */
export function isBranchSuffixFor(suffix, issue) {
  const github = /^#([1-9][0-9]*)$/.exec(issue);
  if (github) return suffix.toLowerCase() === `issue-${github[1]}`;
  return suffix.toUpperCase() === issue;
}

/**
 * One decision's evidence for the result: the agent's choice, with the
 * issue's state, delegate id and entry time taken from the fact sheet.
 */
export function evidence(sheet, decision) {
  const facts = decision.issue === null ? null : sheet.issues[decision.issue];
  return {
    number: sheet.number,
    headSha: sheet.headSha,
    issue: decision.issue,
    issueSource: decision.issue === null ? null : decision.issueSource,
    issueSourceText: decision.issue === null ? null : decision.issueSourceText,
    state: facts?.state ?? null,
    delegateId: facts?.delegateId ?? null,
    stateEnteredAt: facts?.stateEnteredAt ?? null,
    decision: decision.decision,
    reason: decision.reason,
  };
}

/**
 * Triggers each item, and retries each one whose call failed with its own
 * backoff, for `forMs` from its first attempt, until the end time. Calls
 * run one at a time, so while retries are due and PRs are still untried the
 * two take turns, and the PRs not yet tried still reach their first attempt.
 * Among due retries the one whose window ends first goes, and it takes an
 * untried PR's turn when its window would close before its next turn, so a PR
 * failing early still gets retries in its window under load. `call` returns
 * the item with `triggered` true or false. A call starts only when, run to its
 * timeout, it would end inside its item's window and by the end time. An item
 * is failed when its own window ran out without success, `cutShort` when the
 * end time came first, and `notReached` when no call for it could start.
 */
export async function triggerAll(
  items,
  call,
  policy,
  { clock, deadline, callMs },
) {
  const results = new Map();
  const untried = [...items];
  const retrying = [];
  // Whether a call starting at `at` would end inside the item's window and the end time.
  const fits = (entry, at) =>
    at + callMs <= entry.firstAt + policy.forMs && at + callMs <= deadline;
  // Stops retrying an item: failed when its own window is spent, cut short otherwise.
  const stop = (entry, at) => {
    retrying.splice(retrying.indexOf(entry), 1);
    const failed = { ...entry.result, attempts: entry.attempts };
    if (at + callMs <= entry.firstAt + policy.forMs) failed.cutShort = true;
    results.set(entry.item.number, failed);
  };
  const attempt = (entry) => {
    entry.attempts += 1;
    entry.result = call(entry.item);
    if (entry.result.triggered !== false) {
      results.set(entry.item.number, entry.result);
      return;
    }
    if (!retrying.includes(entry)) retrying.push(entry);
    entry.nextAt = clock.now() + entry.delay;
    entry.delay = Math.min(entry.delay * 2, policy.maxDelayMs);
    if (!fits(entry, entry.nextAt)) stop(entry, entry.nextAt);
  };
  let retriedLast = false;
  for (;;) {
    const now = clock.now();
    const due = retrying
      .filter((entry) => entry.nextAt <= now)
      .sort((a, b) => a.firstAt - b.firstAt)[0];
    const canStart = untried.length > 0 && now + callMs <= deadline;
    // Whether this retry would no longer fit after an untried PR's call run
    // to its timeout.
    const closing = due && !fits(due, now + callMs);
    if (due && (closing || !(canStart && retriedLast))) {
      retriedLast = true;
      if (fits(due, now)) {
        retrying.splice(retrying.indexOf(due), 1);
        attempt(due);
      } else stop(due, now);
    } else if (canStart) {
      retriedLast = false;
      const item = untried.shift();
      attempt({ item, attempts: 0, delay: policy.firstDelayMs, firstAt: now });
    } else if (retrying.length > 0) {
      await clock.sleep(
        Math.min(...retrying.map((entry) => entry.nextAt)) - now,
      );
    } else break;
  }
  return items.map((item) => {
    const result = results.get(item.number);
    if (!result) return { ...item, triggered: false, notReached: true };
    if (result.triggered !== false) return result;
    const { attempts, ...failed } = result;
    return attempts > 1
      ? { ...failed, reason: `${failed.reason} (${attempts} attempts)` }
      : failed;
  });
}

/**
 * The run's `launch-result.json`: succeeded with the output when every
 * trigger call succeeded, failed naming each PR whose call failed otherwise.
 */
export function launchResult({ output, failed, action }) {
  const counts = englishList([
    `decided ${output.decisions.length} conflicting factory pull requests`,
    ...triggeredClauses(output, action),
    `skipped ${output.skipped.length} of ${output.openPullRequests} open pull requests`,
  ]);
  if (failed.length === 0) {
    return {
      status: 'succeeded',
      summary: `The sweep ${counts}.`,
      output,
    };
  }
  return {
    status: 'failed',
    summary: `The sweep ${counts}; ${failed.length} trigger calls failed.`,
    reason: failed.map((item) => `#${item.number}: ${item.reason}`).join('; '),
  };
}

/**
 * The summary clauses for the sweep's `triggered` entries. A dry run made no
 * trigger call, so it gives only the number it would trigger. A real run
 * counts the trigger route's answers: `alreadyRecorded: false` is a new
 * request, `true` resends one the route already recorded for that head, and an
 * entry whose response carried no boolean `alreadyRecorded` is counted apart.
 */
function triggeredClauses({ dryRun, triggered }, action) {
  if (dryRun) return [`would trigger ${triggered.length}`];
  const count = (alreadyRecorded) =>
    triggered.filter((item) => item.alreadyRecorded === alreadyRecorded).length;
  const requests = (n, kind = '') =>
    `${n} ${kind}${n === 1 ? 'request' : 'requests'}`;
  const unknown = count(undefined);
  return [
    `made ${requests(count(false), `new ${action} `)}`,
    `resent ${requests(count(true))} already recorded for the same head commit`,
    ...(unknown > 0
      ? [
          `sent ${requests(unknown)} whose answer did not say whether it was new`,
        ]
      : []),
  ];
}

/** `a and b`, or `a, b, and c`. */
function englishList(clauses) {
  if (clauses.length <= 2) return clauses.join(' and ');
  return `${clauses.slice(0, -1).join(', ')}, and ${clauses.at(-1)}`;
}

/**
 * The triggers and the result, from what the listing wrote and the agent
 * decided. Throws, before any trigger call, when a decision is missing or
 * malformed, naming every problem. A dry run makes no trigger call. Nothing
 * runs past `until`.
 */
export async function triggerRounds({
  listing,
  listingSkipped,
  factSheets,
  decisions,
  trigger,
  clock,
  until,
  bounds,
}) {
  const problems = factSheets.flatMap((sheet) =>
    decisionProblems(sheet, decisions.get(sheet.number)),
  );
  const extra = [...decisions.keys()].filter(
    (number) => !factSheets.some((sheet) => sheet.number === number),
  );
  for (const number of extra)
    problems.push(`decisions/${number}.json has no fact sheet`);
  if (problems.length > 0)
    throw new Error(`the decisions are not ready: ${problems.join('; ')}`);

  const decided = factSheets.map((sheet) =>
    evidence(sheet, decisions.get(sheet.number)),
  );
  const toTrigger = decided
    .filter((item) => item.decision === 'trigger')
    .map(({ number, issue, headSha }) => ({ number, issue, headSha }));
  const urls = new Map(factSheets.map((sheet) => [sheet.number, sheet.url]));
  const action = listing.fixerAction;
  const results = listing.dryRun
    ? toTrigger
    : await triggerAll(
        toTrigger,
        (item) => {
          const call = trigger({ ...item, url: urls.get(item.number) }, action);
          if (!call.ok)
            return { ...item, triggered: false, reason: call.reason };
          return {
            ...item,
            triggered: true,
            requestId: requestId(item.number, item.headSha),
            ...(typeof call.response?.alreadyRecorded === 'boolean'
              ? { alreadyRecorded: call.response.alreadyRecorded }
              : {}),
          };
        },
        bounds.triggerRetry,
        { clock, callMs: bounds.callTimeoutMs, deadline: until },
      );
  const unfinished = results
    .filter((item) => item.notReached || item.cutShort)
    .map(({ number, issue, reason, notReached }) => ({
      number,
      issue,
      reason: notReached ? NOT_REACHED_REASON : cutShortReason(reason),
    }));
  const decidedSkips = decided
    .filter((item) => item.decision === 'skip')
    .map(({ number, issue, reason }) => ({
      number,
      ...(issue === null ? {} : { issue }),
      reason,
    }));
  const output = {
    collectedAt: listing.collectedAt,
    dryRun: listing.dryRun,
    baseBranch: listing.baseBranch,
    repositoryId: listing.repositoryId,
    openPullRequests: listing.counts.openPullRequests,
    counts: listing.counts,
    linearRequests: listing.linearRequests,
    githubIssueRequests: listing.githubIssueRequests,
    decisions: decided,
    triggered: results.filter((item) => item.triggered !== false),
    skipped: [...listingSkipped, ...decidedSkips, ...unfinished].sort(
      (a, b) => a.number - b.number,
    ),
  };
  return launchResult({
    output,
    action,
    failed: results.filter(
      (item) => item.triggered === false && !item.notReached && !item.cutShort,
    ),
  });
}

/** One `tessl action trigger` call, run to its timeout. */
export function commandTrigger(bounds) {
  return (item, action) => {
    const result = runCommand(
      'tessl',
      triggerArgs(
        {
          number: item.number,
          url: item.url,
          headRefOid: item.headSha,
        },
        action,
      ),
      { timeoutMs: bounds.callTimeoutMs },
    );
    if (!result.ok) return { ok: false, reason: result.reason };
    return { ok: true, response: parseJson(result.stdout) };
  };
}

/** Every JSON file in `dir`, keyed by its `<number>.json` name. */
function readNumbered(dir) {
  if (!existsSync(dir)) return new Map();
  return new Map(
    readdirSync(dir)
      .filter((name) => /^[0-9]+\.json$/.test(name))
      .map((name) => [
        Number(name.slice(0, -'.json'.length)),
        // A file that is not JSON reads as a string, which the checks name.
        parseJson(readFileSync(join(dir, name), 'utf8')) ?? 'not JSON',
      ]),
  );
}

/** What the listing wrote into `dir`, or an error when it wrote no listing. */
export function readListing(dir) {
  const listing = readJson(join(dir, 'listing.json'));
  if (!listing)
    throw new Error(`no listing in ${dir}: run list-conflicting-prs.mjs first`);
  if (typeof listing.fixerAction !== 'string' || !listing.fixerAction)
    throw new Error(`the listing in ${dir} names no fixer action`);
  const sheets = readNumbered(join(dir, 'facts'));
  const factSheets = listing.factSheets.map((number) => {
    const sheet = sheets.get(number);
    if (!sheet) throw new Error(`facts/${number}.json is missing from ${dir}`);
    if (typeof sheet.url !== 'string' || !sheet.url)
      throw new Error(`facts/${number}.json has no pull request url`);
    return sheet;
  });
  return {
    listing,
    listingSkipped: readJson(join(dir, 'listing-skipped.json')) ?? [],
    factSheets,
    decisions: readNumbered(join(dir, 'decisions')),
  };
}

async function main() {
  const until = parseUntil(flagValue('--until'));
  const bounds = boundsFrom(process.env.MERGE_CONFLICT_SWEEP_BOUNDS);
  const root = runCommand('git', ['rev-parse', '--show-toplevel']);
  if (!root.ok) throw new Error(`not in a clone: ${root.reason}`);
  const resultName = basename(flagValue('--result') ?? 'launch-result.json');
  const resultPath = join(root.stdout.trim(), resultName);
  const dir = DEFAULT_DIR;
  const result = await triggerRounds({
    ...readListing(dir),
    trigger: commandTrigger(bounds),
    clock: realClock,
    until,
    bounds,
  });
  writeJson(resultPath, result);
  process.stdout.write(
    `${result.status}: ${result.summary} Wrote ${resultPath}.\n`,
  );
  return result.status === 'succeeded' ? 0 : 1;
}

if (
  process.argv[1] &&
  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
  main().then(
    (code) => process.exit(code),
    (error) => {
      process.stdout.write(`failed: ${error.message}\n`);
      process.exit(2);
    },
  );
}

README.md

tile.json