Skills that Tessl factory loops run: implement a Linear or GitHub issue as a pull request, shepherd that pull request through review and checks, wake the shepherd for pull requests that a push to their base branch left in conflict, and keep the work in step when a person stops, ends, answers or closes it.
88%
Does it follow best practices?
Run evals on this skill
Adds up to 20 points to the overall score
Low
Low-risk findings worth noting
#!/usr/bin/env node
// Starts one round of the fixer action the listing names for each
// conflicting factory pull request the agent decided to wake, and writes the run's `launch-result.json` by joining
// the listing's fact sheets, the agent's decisions and the trigger outcomes.
// Reads no pull request or issue; its only writes are `tessl action trigger`
// calls and the result. `main` runs when the file is the entry point.
import { existsSync, readFileSync, readdirSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
DEFAULT_DIR,
NOT_REACHED_REASON,
boundsFrom,
flagValue,
parseJson,
parseUntil,
readJson,
realClock,
runCommand,
writeJson,
} from './sweep-common.mjs';
export const ISSUE_SOURCES = ['branch', 'description'];
export const DECISIONS = ['trigger', 'skip'];
/** Why a PR whose trigger calls the end time cut off before its own retries ran out is skipped. */
export function cutShortReason(reason) {
return `${reason}; the script's end time came before its retries ran out, and the next sweep takes it up`;
}
/** One request per PR head: the trigger route records an id once. */
export function requestId(number, headSha) {
return `merge-conflict:${number}@${headSha}`;
}
/**
* The per-PR key, held at limit 1, that serialises the fixer's rounds on a PR.
* The request holds it itself, so its round waits behind those rounds.
*/
export function prConcurrencyKey(number) {
return `github:pr:${number}`;
}
/**
* The arguments of the `tessl action trigger` call that wakes `action` for one
* PR. The link starts the round's run carrying the PR, so a lookup by PR finds
* sweep-started rounds.
*/
export function triggerArgs(pr, action) {
return [
'action',
'trigger',
action,
'--json',
'--input',
`GITHUB_PR_NUMBER=${pr.number}`,
'--request-id',
requestId(pr.number, pr.headRefOid),
'--concurrency-key',
`${prConcurrencyKey(pr.number)}=1`,
'--link',
`references:github_pr=${pr.url}`,
];
}
/**
* What is wrong with the agent's decision for a fact sheet, as a list of
* problems, empty when it is well formed. This checks the decision's shape and
* that its evidence names what the fact sheet holds, not whether the decision
* is the right one.
*/
export function decisionProblems(sheet, decision) {
const at = `decisions/${sheet.number}.json`;
if (decision === undefined) return [`${at} is missing`];
if (decision === null || typeof decision !== 'object')
return [`${at} is not a JSON object`];
const problems = [];
if (decision.number !== sheet.number)
problems.push(`${at} names number ${decision.number}`);
if (!DECISIONS.includes(decision.decision))
problems.push(`${at} decision is not one of ${DECISIONS.join(', ')}`);
if (typeof decision.reason !== 'string' || !decision.reason.trim())
problems.push(`${at} has no reason`);
else if (/[\r\n]/.test(decision.reason))
problems.push(`${at} reason is not one line`);
if (decision.issue === null) {
if (decision.decision === 'trigger')
problems.push(`${at} triggers with no issue`);
if (decision.issueSource !== null || decision.issueSourceText !== null)
problems.push(`${at} has no issue but names a source`);
return problems;
}
if (
typeof decision.issue !== 'string' ||
!Object.hasOwn(sheet.issues, decision.issue)
)
problems.push(`${at} issue ${decision.issue} is not in the fact sheet`);
if (!ISSUE_SOURCES.includes(decision.issueSource))
problems.push(
`${at} issueSource is not one of ${ISSUE_SOURCES.join(', ')}`,
);
const quoted = decision.issueSourceText;
const source =
decision.issueSource === 'branch' ? [sheet.branch] : sheet.descriptionLines;
if (typeof quoted !== 'string' || !quoted.trim())
problems.push(`${at} has no issueSourceText`);
else if (
ISSUE_SOURCES.includes(decision.issueSource) &&
!source.some((text) => text.includes(quoted))
)
problems.push(
`${at} issueSourceText is not quoted from the fact sheet's ${decision.issueSource}`,
);
else if (
typeof decision.issue === 'string' &&
!(
namesIssue(quoted, decision.issue) &&
source.some(
(text) => text.includes(quoted) && namesIssue(text, decision.issue),
)
)
)
problems.push(`${at} issueSourceText does not name ${decision.issue}`);
if (
decision.issueSource === 'branch' &&
!isBranchSuffixFor(sheet.branchSuffix, decision.issue)
)
problems.push(
`${at} issue ${decision.issue} is not the branch's whole suffix`,
);
if (
typeof decision.issue === 'string' &&
!decision.issue.startsWith('#') &&
/^issue-[1-9][0-9]*$/i.test(sheet.branchSuffix) &&
decision.issueSource === 'branch'
)
problems.push(
`${at} branch ${sheet.branch} names GitHub issue #${sheet.branchSuffix.slice('issue-'.length)}, not ${decision.issue}`,
);
return problems;
}
/**
* Whether quoted text names the issue: a Linear identifier anywhere in it,
* ignoring case, or a GitHub issue `#12` as `#12` or as a branch's `issue-12`.
*/
export function namesIssue(text, issue) {
const github = /^#([1-9][0-9]*)$/.exec(issue);
if (!github) return text.toUpperCase().includes(issue);
return new RegExp(`(?:#|\\bissue-)${github[1]}(?![0-9])`, 'i').test(text);
}
/** Whether a branch suffix is the whole name of the issue: `abc-45` for `ABC-45`, `issue-12` for `#12`. */
export function isBranchSuffixFor(suffix, issue) {
const github = /^#([1-9][0-9]*)$/.exec(issue);
if (github) return suffix.toLowerCase() === `issue-${github[1]}`;
return suffix.toUpperCase() === issue;
}
/**
* One decision's evidence for the result: the agent's choice, with the
* issue's state, delegate id and entry time taken from the fact sheet.
*/
export function evidence(sheet, decision) {
const facts = decision.issue === null ? null : sheet.issues[decision.issue];
return {
number: sheet.number,
headSha: sheet.headSha,
issue: decision.issue,
issueSource: decision.issue === null ? null : decision.issueSource,
issueSourceText: decision.issue === null ? null : decision.issueSourceText,
state: facts?.state ?? null,
delegateId: facts?.delegateId ?? null,
stateEnteredAt: facts?.stateEnteredAt ?? null,
decision: decision.decision,
reason: decision.reason,
};
}
/**
* Triggers each item, and retries each one whose call failed with its own
* backoff, for `forMs` from its first attempt, until the end time. Calls
* run one at a time, so while retries are due and PRs are still untried the
* two take turns, and the PRs not yet tried still reach their first attempt.
* Among due retries the one whose window ends first goes, and it takes an
* untried PR's turn when its window would close before its next turn, so a PR
* failing early still gets retries in its window under load. `call` returns
* the item with `triggered` true or false. A call starts only when, run to its
* timeout, it would end inside its item's window and by the end time. An item
* is failed when its own window ran out without success, `cutShort` when the
* end time came first, and `notReached` when no call for it could start.
*/
export async function triggerAll(
items,
call,
policy,
{ clock, deadline, callMs },
) {
const results = new Map();
const untried = [...items];
const retrying = [];
// Whether a call starting at `at` would end inside the item's window and the end time.
const fits = (entry, at) =>
at + callMs <= entry.firstAt + policy.forMs && at + callMs <= deadline;
// Stops retrying an item: failed when its own window is spent, cut short otherwise.
const stop = (entry, at) => {
retrying.splice(retrying.indexOf(entry), 1);
const failed = { ...entry.result, attempts: entry.attempts };
if (at + callMs <= entry.firstAt + policy.forMs) failed.cutShort = true;
results.set(entry.item.number, failed);
};
const attempt = (entry) => {
entry.attempts += 1;
entry.result = call(entry.item);
if (entry.result.triggered !== false) {
results.set(entry.item.number, entry.result);
return;
}
if (!retrying.includes(entry)) retrying.push(entry);
entry.nextAt = clock.now() + entry.delay;
entry.delay = Math.min(entry.delay * 2, policy.maxDelayMs);
if (!fits(entry, entry.nextAt)) stop(entry, entry.nextAt);
};
let retriedLast = false;
for (;;) {
const now = clock.now();
const due = retrying
.filter((entry) => entry.nextAt <= now)
.sort((a, b) => a.firstAt - b.firstAt)[0];
const canStart = untried.length > 0 && now + callMs <= deadline;
// Whether this retry would no longer fit after an untried PR's call run
// to its timeout.
const closing = due && !fits(due, now + callMs);
if (due && (closing || !(canStart && retriedLast))) {
retriedLast = true;
if (fits(due, now)) {
retrying.splice(retrying.indexOf(due), 1);
attempt(due);
} else stop(due, now);
} else if (canStart) {
retriedLast = false;
const item = untried.shift();
attempt({ item, attempts: 0, delay: policy.firstDelayMs, firstAt: now });
} else if (retrying.length > 0) {
await clock.sleep(
Math.min(...retrying.map((entry) => entry.nextAt)) - now,
);
} else break;
}
return items.map((item) => {
const result = results.get(item.number);
if (!result) return { ...item, triggered: false, notReached: true };
if (result.triggered !== false) return result;
const { attempts, ...failed } = result;
return attempts > 1
? { ...failed, reason: `${failed.reason} (${attempts} attempts)` }
: failed;
});
}
/**
* The run's `launch-result.json`: succeeded with the output when every
* trigger call succeeded, failed naming each PR whose call failed otherwise.
*/
export function launchResult({ output, failed, action }) {
const counts = englishList([
`decided ${output.decisions.length} conflicting factory pull requests`,
...triggeredClauses(output, action),
`skipped ${output.skipped.length} of ${output.openPullRequests} open pull requests`,
]);
if (failed.length === 0) {
return {
status: 'succeeded',
summary: `The sweep ${counts}.`,
output,
};
}
return {
status: 'failed',
summary: `The sweep ${counts}; ${failed.length} trigger calls failed.`,
reason: failed.map((item) => `#${item.number}: ${item.reason}`).join('; '),
};
}
/**
* The summary clauses for the sweep's `triggered` entries. A dry run made no
* trigger call, so it gives only the number it would trigger. A real run
* counts the trigger route's answers: `alreadyRecorded: false` is a new
* request, `true` resends one the route already recorded for that head, and an
* entry whose response carried no boolean `alreadyRecorded` is counted apart.
*/
function triggeredClauses({ dryRun, triggered }, action) {
if (dryRun) return [`would trigger ${triggered.length}`];
const count = (alreadyRecorded) =>
triggered.filter((item) => item.alreadyRecorded === alreadyRecorded).length;
const requests = (n, kind = '') =>
`${n} ${kind}${n === 1 ? 'request' : 'requests'}`;
const unknown = count(undefined);
return [
`made ${requests(count(false), `new ${action} `)}`,
`resent ${requests(count(true))} already recorded for the same head commit`,
...(unknown > 0
? [
`sent ${requests(unknown)} whose answer did not say whether it was new`,
]
: []),
];
}
/** `a and b`, or `a, b, and c`. */
function englishList(clauses) {
if (clauses.length <= 2) return clauses.join(' and ');
return `${clauses.slice(0, -1).join(', ')}, and ${clauses.at(-1)}`;
}
/**
* The triggers and the result, from what the listing wrote and the agent
* decided. Throws, before any trigger call, when a decision is missing or
* malformed, naming every problem. A dry run makes no trigger call. Nothing
* runs past `until`.
*/
export async function triggerRounds({
listing,
listingSkipped,
factSheets,
decisions,
trigger,
clock,
until,
bounds,
}) {
const problems = factSheets.flatMap((sheet) =>
decisionProblems(sheet, decisions.get(sheet.number)),
);
const extra = [...decisions.keys()].filter(
(number) => !factSheets.some((sheet) => sheet.number === number),
);
for (const number of extra)
problems.push(`decisions/${number}.json has no fact sheet`);
if (problems.length > 0)
throw new Error(`the decisions are not ready: ${problems.join('; ')}`);
const decided = factSheets.map((sheet) =>
evidence(sheet, decisions.get(sheet.number)),
);
const toTrigger = decided
.filter((item) => item.decision === 'trigger')
.map(({ number, issue, headSha }) => ({ number, issue, headSha }));
const urls = new Map(factSheets.map((sheet) => [sheet.number, sheet.url]));
const action = listing.fixerAction;
const results = listing.dryRun
? toTrigger
: await triggerAll(
toTrigger,
(item) => {
const call = trigger({ ...item, url: urls.get(item.number) }, action);
if (!call.ok)
return { ...item, triggered: false, reason: call.reason };
return {
...item,
triggered: true,
requestId: requestId(item.number, item.headSha),
...(typeof call.response?.alreadyRecorded === 'boolean'
? { alreadyRecorded: call.response.alreadyRecorded }
: {}),
};
},
bounds.triggerRetry,
{ clock, callMs: bounds.callTimeoutMs, deadline: until },
);
const unfinished = results
.filter((item) => item.notReached || item.cutShort)
.map(({ number, issue, reason, notReached }) => ({
number,
issue,
reason: notReached ? NOT_REACHED_REASON : cutShortReason(reason),
}));
const decidedSkips = decided
.filter((item) => item.decision === 'skip')
.map(({ number, issue, reason }) => ({
number,
...(issue === null ? {} : { issue }),
reason,
}));
const output = {
collectedAt: listing.collectedAt,
dryRun: listing.dryRun,
baseBranch: listing.baseBranch,
repositoryId: listing.repositoryId,
openPullRequests: listing.counts.openPullRequests,
counts: listing.counts,
linearRequests: listing.linearRequests,
githubIssueRequests: listing.githubIssueRequests,
decisions: decided,
triggered: results.filter((item) => item.triggered !== false),
skipped: [...listingSkipped, ...decidedSkips, ...unfinished].sort(
(a, b) => a.number - b.number,
),
};
return launchResult({
output,
action,
failed: results.filter(
(item) => item.triggered === false && !item.notReached && !item.cutShort,
),
});
}
/** One `tessl action trigger` call, run to its timeout. */
export function commandTrigger(bounds) {
return (item, action) => {
const result = runCommand(
'tessl',
triggerArgs(
{
number: item.number,
url: item.url,
headRefOid: item.headSha,
},
action,
),
{ timeoutMs: bounds.callTimeoutMs },
);
if (!result.ok) return { ok: false, reason: result.reason };
return { ok: true, response: parseJson(result.stdout) };
};
}
/** Every JSON file in `dir`, keyed by its `<number>.json` name. */
function readNumbered(dir) {
if (!existsSync(dir)) return new Map();
return new Map(
readdirSync(dir)
.filter((name) => /^[0-9]+\.json$/.test(name))
.map((name) => [
Number(name.slice(0, -'.json'.length)),
// A file that is not JSON reads as a string, which the checks name.
parseJson(readFileSync(join(dir, name), 'utf8')) ?? 'not JSON',
]),
);
}
/** What the listing wrote into `dir`, or an error when it wrote no listing. */
export function readListing(dir) {
const listing = readJson(join(dir, 'listing.json'));
if (!listing)
throw new Error(`no listing in ${dir}: run list-conflicting-prs.mjs first`);
if (typeof listing.fixerAction !== 'string' || !listing.fixerAction)
throw new Error(`the listing in ${dir} names no fixer action`);
const sheets = readNumbered(join(dir, 'facts'));
const factSheets = listing.factSheets.map((number) => {
const sheet = sheets.get(number);
if (!sheet) throw new Error(`facts/${number}.json is missing from ${dir}`);
if (typeof sheet.url !== 'string' || !sheet.url)
throw new Error(`facts/${number}.json has no pull request url`);
return sheet;
});
return {
listing,
listingSkipped: readJson(join(dir, 'listing-skipped.json')) ?? [],
factSheets,
decisions: readNumbered(join(dir, 'decisions')),
};
}
async function main() {
const until = parseUntil(flagValue('--until'));
const bounds = boundsFrom(process.env.MERGE_CONFLICT_SWEEP_BOUNDS);
const root = runCommand('git', ['rev-parse', '--show-toplevel']);
if (!root.ok) throw new Error(`not in a clone: ${root.reason}`);
const resultName = basename(flagValue('--result') ?? 'launch-result.json');
const resultPath = join(root.stdout.trim(), resultName);
const dir = DEFAULT_DIR;
const result = await triggerRounds({
...readListing(dir),
trigger: commandTrigger(bounds),
clock: realClock,
until,
bounds,
});
writeJson(resultPath, result);
process.stdout.write(
`${result.status}: ${result.summary} Wrote ${resultPath}.\n`,
);
return result.status === 'succeeded' ? 0 : 1;
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
main().then(
(code) => process.exit(code),
(error) => {
process.stdout.write(`failed: ${error.message}\n`);
process.exit(2);
},
);
}