Redesign an existing website to make it better. Built on top of impeccable.
88%
Does it follow best practices?
Run evals on this skill
Adds up to 20 points to the overall score
Low
Low-risk findings worth noting
#!/usr/bin/env node
/**
* spec-inventory.mjs — S1 inventory: robots.txt + sitemaps (index or urlset, nested) → the URL list of the
* scope path, plus every other sitemap (locales, microsites) counted for the multi-language inventory.
*
* node spec-inventory.mjs [--config <file>] [--max <n>] [--headed | --archive <YYYY-MM-DD>]
*
* Reads config: origin, scopePath, sitemaps? (default: robots.txt Sitemap lines, else /sitemap.xml), maxPages?.
* Stops when the origin redirects elsewhere (set `origin` to the final one). Without sitemap URLs in scope, crawls
* same-origin links from the scope root (breadth first, up to the cap). Above --max / maxPages, keeps an even
* sample per first path segment (every section at least one URL).
* Writes <work>/inventory/urls.txt (the URLs every later stage reads, sorted), urls-all.txt (when sampled),
* summary.json ({ total, kept, source: sitemap|crawl, sampled }), sitemaps.json (every sitemap: url count, path
* roots, hreflang count, lastmod years), robots.txt. Exit 0; 1 on a redirected origin; 2 on usage; 3 when the origin
* turns this client away (a bot challenge on robots, sitemaps or pages, classified by the diff skill's
* live-session.mjs): ask the user, then re-run with --headed (live-session's stealth real-Chrome tier) or --archive
* <date> (Internet Archive captures since <date>), the same choice S2 then makes.
*/
import { arg, flag, helpAndExit, loadBrowser, loadConfig, log, writeJSON, writeText, pool } from './lib.mjs';
helpAndExit(import.meta.url);
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0 Safari/537.36 stardust-spec/1.0';
/** <loc> values and whether the document is a sitemap index. Pure. */
export function parseSitemap(xml) {
const locs = [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1].replace(/&/g, '&'));
return { isIndex: /<sitemapindex/i.test(xml), locs, hreflang: (xml.match(/hreflang=/g) || []).length, lastmod: [...xml.matchAll(/<lastmod>(\d{4})/g)].map((m) => m[1]) };
}
/** First two path segments, e.g. /de/de or /en/home. Pure. */
export const rootOf = (u) => { try { return `/${new URL(u).pathname.split('/').filter(Boolean).slice(0, 2).join('/')}`; } catch { return '/'; } };
// how documents are read: live (default), live-session's stealth Chrome (--headed) or the Internet Archive (--archive)
const MODE = { archive: arg('archive', null), headed: flag('headed'), blocked: 0, ctx: null, ls: null };
async function get(url) {
if (MODE.archive) {
const cdx = await fetch(`https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(url)}&output=json&from=${MODE.archive.replace(/-/g, '')}&filter=statuscode:200&fl=timestamp`, { headers: { 'user-agent': UA } }).then((r) => (r.ok ? r.json() : [])).catch(() => []);
const ts = cdx.length > 1 ? cdx[cdx.length - 1][0] : null;
return ts ? fetch(`https://web.archive.org/web/${ts}id_/${url}`, { headers: { 'user-agent': UA } }).then((r) => (r.ok ? r.text() : '')).catch(() => '') : '';
}
if (!MODE.ls) MODE.ls = await loadBrowser();
if (MODE.headed) {
if (!MODE.ctx) { const { openBrowser, newLiveContext } = MODE.ls; MODE.ctx = await newLiveContext(await openBrowser({ mode: 'headed' })); }
const p = await MODE.ctx.newPage();
try { const r = await MODE.ls.gotoLive(p, url, { solveWindow: true, httpError: 'measure', settleMs: 300 }); return r.ok() ? await r.text().catch(() => '') : ''; } catch (e) { if (e.name === 'BotChallengeError') MODE.blocked += 1; return ''; } finally { await p.close(); }
}
const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/xml,text/xml,*/*' }, redirect: 'follow' }).catch(() => null);
if (!r) return '';
const h = Object.fromEntries(r.headers);
if (MODE.ls.isChallengeResponse({ status: () => r.status, headers: () => h, url: () => r.url }) || r.status === 403 || r.status === 429) { MODE.blocked += 1; await r.arrayBuffer().catch(() => {}); return ''; }
return r.ok ? r.text() : '';
}
/** Even sample of `max` URLs, allocated per first path segment under the scope (each section ≥ 1). Pure. */
export function sampleBySection(urls, max, scopePath) {
if (urls.length <= max) return urls;
const sec = (u) => { try { return new URL(u).pathname.slice(scopePath.length).split('/').filter(Boolean)[0] || '(root)'; } catch { return '(root)'; } };
const groups = {}; urls.forEach((u) => { (groups[sec(u)] = groups[sec(u)] || []).push(u); });
const keys = Object.keys(groups).sort((a, b) => groups[b].length - groups[a].length);
const quota = Object.fromEntries(keys.map((k) => [k, 1]));
let left = max - keys.length;
if (left < 0) return keys.slice(0, max).map((k) => groups[k][0]);
keys.forEach((k) => { const q = Math.floor(((groups[k].length - 1) / (urls.length - keys.length)) * left); quota[k] += Math.min(q, groups[k].length - 1); });
left = max - Object.values(quota).reduce((a, b) => a + b, 0);
for (const k of keys) { if (left <= 0) break; if (quota[k] < groups[k].length) { quota[k] += 1; left -= 1; } }
const out = [];
for (const k of keys) { const g = groups[k].sort(); const step = g.length / quota[k]; for (let i = 0; i < quota[k]; i += 1) out.push(g[Math.floor(i * step)]); }
return out.sort();
}
const inScopeOf = (cfg) => (l) => { try { const x = new URL(l); return `${x.origin}` === cfg.origin && (x.pathname === cfg.scopePath || x.pathname.startsWith(`${cfg.scopePath.replace(/\/$/, '')}/`) || x.pathname.startsWith(cfg.scopePath)); } catch { return false; } };
/** Links that are files, not pages: the crawl reads every href, so <link> icons, fonts and manifests reach it too. */
export const NOT_PAGE = /\.(pdf|jpe?g|png|gif|svg|webp|avif|ico|zip|docx?|xlsx?|pptx?|mp4|webm|mov|mp3|css|js|xml|json|txt|webmanifest|woff2?|ttf|otf|eot)(\?|$)/i;
/** Breadth-first same-origin crawl from the scope root, for sites without a usable sitemap. */
async function crawl(cfg, max) {
const ok = inScopeOf(cfg); const seen = new Set([`${cfg.origin}${cfg.scopePath}`]); let frontier = [...seen];
const skip = NOT_PAGE;
while (frontier.length && seen.size < max * 2) {
const pages = await pool(frontier.slice(0, 200), 4, get); const next = [];
pages.forEach((html, i) => { for (const m of (html || '').matchAll(/href="([^"#]+)"/g)) { let u; try { u = new URL(m[1].replace(/&/g, '&'), frontier[i]); } catch { continue; } u.hash = ''; u.search = ''; const h = u.href; if (ok(h) && !skip.test(h) && !seen.has(h)) { seen.add(h); next.push(h); } } });
frontier = next;
}
return [...seen];
}
async function main() {
const cfg = loadConfig();
const max = Number(arg('max', cfg.maxPages || 0)) || Infinity;
const probe = await fetch(`${cfg.origin}${cfg.scopePath}`, { headers: { 'user-agent': UA }, redirect: 'follow' }).catch((e) => ({ url: '', error: e.message }));
const finalOrigin = probe.url ? new URL(probe.url).origin : null;
if (finalOrigin && finalOrigin !== cfg.origin) { console.error(`origin ${cfg.origin} redirects to ${finalOrigin}: set "origin": "${finalOrigin}" in spec.config.json`); process.exit(1); }
const robots = await get(`${cfg.origin}/robots.txt`);
writeText(cfg.w('inventory', 'robots.txt'), robots || '# none');
let roots = cfg.sitemaps || [...robots.matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((m) => m[1]);
if (!roots.length) roots = [`${cfg.origin}/sitemap.xml`];
const seen = new Set(); const docs = {}; let queue = [...roots];
while (queue.length) {
const batch = queue.filter((u) => !seen.has(u)); queue = [];
batch.forEach((u) => seen.add(u));
const xmls = await pool(batch, 4, get);
batch.forEach((u, i) => {
const s = parseSitemap(xmls[i] || '');
if (s.isIndex) queue.push(...s.locs);
else docs[u] = s;
});
}
const inScope = new Set(); const summary = {}; const ok = inScopeOf(cfg);
for (const [u, s] of Object.entries(docs)) {
const roots2 = {}; s.locs.forEach((l) => { const r = rootOf(l); roots2[r] = (roots2[r] || 0) + 1; });
const years = {}; s.lastmod.forEach((y) => { years[y] = (years[y] || 0) + 1; });
summary[u] = { urls: s.locs.length, roots: roots2, hreflang: s.hreflang, lastmodYears: years };
s.locs.filter(ok).forEach((l) => inScope.add(l));
}
let all = [...inScope].sort(); let source = 'sitemap';
if (!all.length) { source = 'crawl'; all = (await crawl(cfg, max === Infinity ? 5000 : max)).sort(); }
if (MODE.ctx) await MODE.ctx.browser().close();
if (MODE.blocked && all.length <= 1) {
console.error(`BLOCKED ORIGIN: ${MODE.blocked} request(s) for robots, sitemaps or pages were turned away${MODE.headed ? ' even headed' : ''}. Ask the user how to proceed: allow-list the crawler (user agent "stardust-spec/1.0") and re-run; --headed (live-session's stealth real-Chrome tier; it waits out a JS challenge); or --archive <YYYY-MM-DD> (Internet Archive captures: stale, and the spec must say so). Use the same choice for S2.`);
process.exit(3);
}
if (MODE.archive) source = `${source} (archive since ${MODE.archive})`;
const urls = sampleBySection(all, max, cfg.scopePath);
writeText(cfg.w('inventory', 'urls.txt'), urls.join('\n'));
if (urls.length < all.length) writeText(cfg.w('inventory', 'urls-all.txt'), all.join('\n'));
writeJSON(cfg.w('inventory', 'summary.json'), { total: all.length, kept: urls.length, source, sampled: urls.length < all.length });
writeJSON(cfg.w('inventory', 'sitemaps.json'), summary);
log(`${Object.keys(docs).length} sitemaps, ${Object.values(summary).reduce((a, s) => a + s.urls, 0)} urls total, ${all.length} in scope ${cfg.scopePath} (${source})${urls.length < all.length ? `, sampled ${urls.length}` : ''}`);
}
if (import.meta.url === `file://${process.argv[1]}`) main().catch((e) => { console.error(e.message); process.exit(1); });.tessl-plugin
skills
audit
reference
deploy
reference
scripts
test
diff
direct
dynamics
extract
migrate
prepare-migration
prototype
assets
motion
fixtures
reference
qa
replica
reference
scripts
test
reskin
rollout
spec
stardust
uplift
reference