CtrlK
BlogDocsLog inGet started
Tessl Logo
PluginPublic

adobe/stardust

Redesign an existing website to make it better. Built on top of impeccable.

Quality

88%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Low

Low-risk findings worth noting

Overview
Quality
Evals
Security
Files

dynamics-detect.mjsskills/dynamics/scripts/

#!/usr/bin/env node
/**
 * dynamics-detect.mjs — stardust:dynamics Phase 1 + 2 (detect, classify).
 *
 * Records, per probed page, everything the live page renders from JavaScript,
 * a service or a data source, then classifies each finding into the dynamic
 * classes (reference/classes-and-signals.md). Evidence only: no decision is
 * taken here.
 *
 *   node dynamics-detect.mjs --urls <url,url,…> [--out stardust/current]
 *        [--from-state stardust/state.json]   one URL per page type + the home page, from extract's inventory
 *        [--reach stardust/current]           roll per-page `dynamic` sections (extract --dynamics) into feature reach
 *        [--settle 5000] [--width 1440] [--headed]
 *
 * Writes (under --out, default stardust/current):
 *   _dynamics.json                  per-page evidence + classified findings (+ `reach` with --reach)
 *   dynamic-features.generated.md   the findings as a table, one row per feature
 * Progress lines go to stderr. Exit 0 on completion, 2 on usage or a setup error.
 *
 * Probes the SOURCE site. No auth header is sent (the source is public); the
 * target-host probe lives in dynamics-plan.mjs.
 */
/* eslint-disable no-await-in-loop, no-restricted-syntax, max-len */
import { readdirSync, existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import {
  CLASS_NAMES, arg, flag, list, readJSON, writeJSON, writeText, provenance, openBrowser, newContext, goto, stopOnSetupError, vendorFor, registrable, sameSite, pathPattern, settlePage, slug,
} from './lib.mjs';

// --help prints this file's usage header, so an agent never reads the source to learn the flags.
if (process.argv.includes('--help') || process.argv.includes('-h')) {
  const src = readFileSync(new URL(import.meta.url), 'utf8');
  const header = src.match(/\/\*\*[\s\S]*?\*\//);
  console.log(header ? header[0].replace(/^\/\*\*\s*|\s*\*\/$/g, '').replace(/^\s*\* ?/gm, '').trim() : 'no usage header');
  process.exit(0);
}

const OUT = arg('out', 'stardust/current');
const SETTLE = Number(arg('settle', 5000));
const WIDTH = Number(arg('width', 1440));
let URLS = list(arg('urls', ''));
if (!URLS.length && arg('from-state')) {
  const st = readJSON(arg('from-state'));
  const byType = new Map();
  for (const p of st.pages || []) { const t = p.type || 'untyped'; if (!byType.has(t)) byType.set(t, p.url); }
  URLS = [...new Set([st.site?.url || st.site?.origin, ...byType.values()].filter(Boolean))];
}
if (!URLS.length) { console.error('usage: dynamics-detect.mjs --urls <url,…> | --from-state stardust/state.json [--out dir]'); process.exit(2); }

const API_PATH = /\/(api|graphql|ajax|json|search|autocomplete|typeahead|suggest|client\/|webservices|_next\/data|wp-json|\.rest|odata)/i;

/* ------------------------------------------------------ in-page capture -- */
function domCapture() {
  const norm = (t) => (t || '').replace(/\s+/g, ' ').trim();
  const cn = (el) => (el.getAttribute && el.getAttribute('class')) || '';
  const abs = (h) => { try { return new URL(h, location.href).href; } catch { return h; } };
  const inChrome = (el) => !!el.closest('header, nav, footer, [role=navigation], [role=banner], [role=contentinfo], [class*="header" i], [class*="footer" i], [class*="cookie" i], [id*="onetrust" i]');
  const vis = (el) => { const r = el.getBoundingClientRect(); return r.width > 1 && r.height > 1; };

  // forms — keyed on name + field signature, action optional (JS-wired forms have none)
  const forms = [...document.querySelectorAll('form')].map((f) => {
    const inputs = [...f.querySelectorAll('input, select, textarea')].filter((i) => !['hidden', 'submit', 'button', 'reset'].includes((i.type || '').toLowerCase()));
    const sig = inputs.map((i) => `${i.tagName === 'SELECT' ? 'select' : i.type || 'text'}:${i.name || i.id || ''}`).slice(0, 12);
    return {
      name: f.getAttribute('name') || f.id || cn(f).split(' ')[0] || '', action: f.getAttribute('action') ? abs(f.getAttribute('action')) : null,
      method: (f.getAttribute('method') || 'get').toLowerCase(), role: f.getAttribute('role') || '', inChrome: inChrome(f), fields: inputs.length, signature: sig,
      hidden: [...f.querySelectorAll('input[type=hidden]')].map((i) => i.name).filter(Boolean).slice(0, 12),
      tokens: [...f.querySelectorAll('input')].map((i) => i.name || '').filter((n) => /token|csrf|antibot|nonce|honeypot|captcha|verification/i.test(n)),
      search: f.getAttribute('role') === 'search' || !!f.querySelector('input[type=search], input[name="q"], input[name="s"], input[name*="query" i], input[name*="search" i]'),
      inlineHandler: !!(f.getAttribute('onsubmit') || f.querySelector('[onclick], [onsubmit]')),
    };
  });
  // form-less control groups (modern CMS front ends render forms without <form>)
  const loose = [...document.querySelectorAll('input, select, textarea')].filter((el) => !el.closest('form') && !inChrome(el) && !['hidden', 'submit', 'button', 'password'].includes((el.type || '').toLowerCase()) && vis(el));
  const groups = new Map();
  for (const el of loose) {
    const root = el.closest('section, article, [class*="form" i], [data-component], main > div, main') || document.body;
    const key = root === document.body ? 'body' : `${root.tagName.toLowerCase()}.${cn(root).split(' ')[0]}`;
    if (!groups.has(key)) groups.set(key, { container: key, controls: 0, names: [], submit: null });
    const g = groups.get(key); g.controls += 1; if (el.name && g.names.length < 12) g.names.push(el.name);
    if (!g.submit) { const b = [...root.querySelectorAll('button, [role=button], input[type=submit]')].find((x) => /submit|send|continue|enquir|subscribe|sign ?up|request|apply|pay|search/i.test(norm(x.textContent) || x.value || '')); if (b) g.submit = norm(b.textContent) || b.value; }
  }
  const controlGroups = [...groups.values()].filter((g) => g.controls >= 2);

  // trigger → dialog → content graph
  const dialogs = [...document.querySelectorAll('dialog, [role=dialog], [aria-modal=true], [class*="modal" i]:not(a):not(button):not(script)')];
  const contentSig = (t) => t && { role: t.getAttribute('role') || t.tagName.toLowerCase(), heading: norm(t.querySelector('h1,h2,h3,[class*="title" i]')?.textContent).slice(0, 80), textLen: norm(t.textContent).length, hasForm: !!t.querySelector('form, input, select, textarea'), hasIframe: !!t.querySelector('iframe'), hasVideo: !!t.querySelector('video, video-js, [data-video-id], iframe[src*="player" i]') };
  const triggers = [...document.querySelectorAll('a, button, [role=button]')].filter((el) => !/close|dismiss|vjs-|carousel|slider/i.test(cn(el) + (el.getAttribute('aria-label') || ''))).map((el) => {
    const attrs = [...el.attributes].map((a) => a.name);
    const marker = (cn(el).match(/[\w-]*(modal|dialog|lightbox|popup|overlay)[\w-]*/i) || [])[0] || attrs.find((n) => /modal|dialog|lightbox|popup|micromodal/i.test(n)) || (el.getAttribute('aria-haspopup') === 'dialog' ? 'aria-haspopup=dialog' : null);
    const byId = attrs.map((n) => el.getAttribute(n)).find((v) => v && /^[\w-]+$/.test(v) && document.getElementById(v));
    const controls = (el.getAttribute('aria-controls') && document.getElementById(el.getAttribute('aria-controls'))) || (byId && document.getElementById(byId));
    const controlsDialog = controls && (controls.getAttribute('role') === 'dialog' || controls.tagName === 'DIALOG');
    if (!marker && !controlsDialog) return null;
    const titleAttr = attrs.find((n) => /title/i.test(n) && /modal|dialog/i.test(n));
    return { text: norm(el.textContent).slice(0, 60), href: el.getAttribute('href') ? abs(el.getAttribute('href')) : null, marker: marker || 'aria-controls→dialog', inChrome: inChrome(el), titleOnTrigger: titleAttr ? el.getAttribute(titleAttr).slice(0, 80) : null, target: contentSig(controls || (el.getAttribute('data-target') && document.querySelector(el.getAttribute('data-target'))) || null) };
  }).filter(Boolean).slice(0, 40);
  const dialogSigs = dialogs.map(contentSig).filter((d) => d && d.textLen > 0).slice(0, 12);

  // media players + ids (ids live only in the live DOM)
  const media = [...document.querySelectorAll('video, audio, video-js, iframe, [data-video-id], [data-account], [data-player], a[href*="player" i]')].map((el) => ({
    tag: el.tagName.toLowerCase(), src: (el.getAttribute('src') || el.getAttribute('href') || el.getAttribute('data-src') || '').slice(0, 200), account: el.getAttribute('data-account'), player: el.getAttribute('data-player'), videoId: el.getAttribute('data-video-id') || el.getAttribute('data-videoid'), inDialog: !!el.closest('dialog, [role=dialog], [class*="modal" i]'),
  })).filter((m) => m.src || m.videoId);
  // iframes: a missing src means a runtime-injected embed
  const iframes = [...document.querySelectorAll('iframe')].map((f) => ({ src: f.getAttribute('src') || null, title: f.getAttribute('title') || null, w: f.getBoundingClientRect().width | 0, h: f.getBoundingClientRect().height | 0 }));
  // tag-manager-proxied third parties: mount divs with vendor config attributes, zero script tags
  const mounts = [...document.querySelectorAll('[data-businessunit-id], [data-template-id], [data-widget-id], [class*="widget" i], [data-pr-component], [data-bv-show], [id^="hs-"], [data-hs-forms-root], [id*="mkto" i], [class*="embed" i][data-src]')]
    .filter((el) => !inChrome(el) && [...el.attributes].some((a) => a.name.startsWith('data-')))
    .map((el) => ({ cls: cn(el).split(' ').slice(0, 2).join(' '), attrs: [...el.attributes].filter((a) => a.name.startsWith('data-')).map((a) => a.name).slice(0, 6) })).slice(0, 20);

  const hrefs = [...document.querySelectorAll('a[href]')].map((a) => abs(a.getAttribute('href')));
  const auth = hrefs.filter((h) => /login|sign-?in|signin|account|register|logout|b2clogin|oauth|sso|myaccount/i.test(h)).slice(0, 8);
  const commerce = { cart: !!document.querySelector('[class*="cart" i], [href*="/cart"], [href*="checkout"]'), prices: (document.body.innerText.match(/[£$€]\s?\d[\d,.]*/g) || []).length };
  const locale = { lang: document.documentElement.lang || null, hreflang: [...document.querySelectorAll('link[rel=alternate][hreflang]')].map((l) => l.getAttribute('hreflang')), switcher: hrefs.filter((h) => { try { const p = new URL(h).pathname; return /^\/(?:[a-z]{2}(?:-[a-z]{2})?)(?:\/|$)/i.test(p) && !location.pathname.startsWith(p.split('/').slice(0, 2).join('/')); } catch { return false; } }).slice(0, 5) };
  const cms = {}; for (const k of ['drupalSettings', 'wp', 'wpApiSettings', 'Shopify', 'Sitecore', 'digitalData', 'dataLayer', 'adobeDataLayer', 'utag_data', 'appConfig', '__NEXT_DATA__', '__NUXT__', '__INITIAL_STATE__', '__PRELOADED_STATE__', '__APOLLO_STATE__', 'Granite', 'CQ']) { try { if (window[k] !== undefined) cms[k] = window[k] && typeof window[k] === 'object' ? Object.keys(window[k]).slice(0, 30) : typeof window[k]; } catch { /* cross-origin getter */ } }
  const framework = document.querySelector('#__next') ? 'next' : document.querySelector('#___gatsby') ? 'gatsby' : document.querySelector('#__nuxt') ? 'nuxt' : document.querySelector('[data-reactroot], [data-react-helmet]') ? 'react' : document.querySelector('[data-v-app], [data-server-rendered]') ? 'vue' : document.querySelector('[ng-version]') ? 'angular' : document.querySelector('[data-sveltekit-preload-data]') ? 'sveltekit' : document.querySelector('astro-island') ? 'astro' : null;
  const globals = ['adobe', '_satellite', 'OneTrust', 'Optanon', 'google_tag_manager', 'gtag', 'Munchkin', 'videojs', 'bc', 'YT', 'Vimeo', 'Intercom', 'drift', 'zE', 'hj', 'clarity', 'optimizely', 'utag', 'dtrum', 'Drupal', 'Shopify', 'algoliasearch', 'grecaptcha', 'hbspt', 'MktoForms2', 'Trustpilot'].filter((g) => { try { return window[g] !== undefined; } catch { return false; } });
  const clientRendered = [...document.querySelectorAll('[class*="placeholder" i], [class*="skeleton" i], [class*="js-" i], [data-lazy], [data-load], [data-src-url], [data-endpoint], [data-api], [data-url], [data-component]')].filter((el) => el.children.length && norm(el.textContent).length > 20 && !inChrome(el)).map((el) => ({ cls: cn(el).split(' ').slice(0, 3).join(' '), attrs: [...el.attributes].filter((a) => /^data-(endpoint|api|url|src-url|load|lazy|params|component)/.test(a.name)).map((a) => `${a.name}=${a.value.slice(0, 60)}`), text: norm(el.textContent).slice(0, 60) })).slice(0, 12);
  const listingCandidates = [...document.querySelectorAll('main ul, main ol, main div, main section')].filter((c) => !inChrome(c)).map((c) => { const kids = [...c.children]; if (kids.length < 3) return null; const cards = kids.map((k) => k.querySelector('a[href]')).filter(Boolean); if (cards.length < 3 || cards.length !== kids.length) return null; const same = cards.filter((a) => abs(a.getAttribute('href')).startsWith(location.origin)).length; if (same < 3) return null; return { cls: cn(c).split(' ').slice(0, 2).join(' ') || c.tagName.toLowerCase(), cards: kids.length, sameSite: same, hasDate: /\b(20\d\d|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/.test(c.textContent) }; }).filter(Boolean).slice(0, 10);
  // client-compute signature: inline script + controls + (network judged outside)
  const inline = [...document.querySelectorAll('script:not([src])')].filter((s) => !/json/i.test(s.type || ''));
  const inlineScripts = inline.length;
  const inlineFetch = inline.some((s) => /fetch\(|XMLHttpRequest|\.ajax\(|axios|sendBeacon/.test(s.textContent || ''));
  const mainText = norm(document.querySelector('main')?.innerText || document.body.innerText).length;
  return { title: document.title, forms, controlGroups, triggers, dialogs: dialogSigs, dialogCount: dialogs.length, media, iframes, mounts, auth, commerce, locale, cms, framework, globals, clientRendered, listingCandidates, inlineScripts, inlineFetch, mainText, domAddedAfterLoad: window.__sdDomAfterLoad || 0 };
}

/* ------------------------------------------------------------ classify -- */
function classify(page, path, add) {
  const host = page.host;
  const firstParty = (h) => sameSite(h, host);
  // third-party hosts (from any request), by vendor role
  for (const h of Object.keys(page.hosts)) {
    if (!h || h === host) continue;
    const v = vendorFor(h) || vendorFor(page.scripts.find((s) => s.includes(h)) || '');
    if (v && v.class === '-') continue;
    if (firstParty(h)) { if (v && v.class === 'T') add({ class: 'T', feature: `${v.role} (first-party subdomain — needs a CNAME on the new host)`, page: path, evidence: [h], hint: 'tags' }); continue; }
    const xhr = page.thirdPartyXhr.some((x) => x.includes(h));
    if (v) add({ class: v.class, feature: v.role, role: v.role, page: path, evidence: [h], hint: v.class === 'T' ? 'tags' : v.class === 'F' ? 'forms' : v.class === 'V' ? 'media' : v.class === 'S' ? 'search' : v.class === 'X' ? 'decided-out' : 'inspect' });
    else add({ class: xhr ? 'A' : 'T', feature: `unknown third-party host ${h}`, page: path, evidence: [h], hint: 'inspect' });
  }
  for (const a of page.firstPartyApi) {
    if (!API_PATH.test(a.path) && a.contentType !== 'application/json') continue;
    const isSearch = /search|autocomplete|typeahead|suggest/i.test(a.path);
    const isData = /\.json$/.test(a.path) && a.method === 'GET' && !API_PATH.test(a.path.replace(/\.json$/, ''));
    add({ class: isSearch ? 'S' : isData ? 'D' : 'A', feature: `first-party ${isData ? 'data file' : 'API'} ${a.method} ${a.path}`, page: path, evidence: [`${a.method} ${a.path}${a.query.length ? `?${a.query.join(',')}` : ''} → ${a.status}`], api: { method: a.method, path: a.path }, hint: isSearch ? 'search' : isData ? 'data' : 'api' });
  }
  for (const f of page.forms) {
    if (f.search) { add({ class: 'S', feature: `site search form → ${f.action ? new URL(f.action).pathname : '(JS-submitted)'}`, page: path, evidence: [f.signature.join(',')], hint: 'search' }); continue; }
    if (f.fields === 0) continue;
    const target = f.action ? (firstParty(new URL(f.action).host) ? `origin ${new URL(f.action).pathname}` : `third-party ${new URL(f.action).host}`) : 'no action (JS-wired)';
    add({ class: 'F', feature: `form "${f.name || f.signature.slice(0, 2).join('+')}" → ${target} (${f.fields} fields${f.tokens.length ? `, tokens: ${f.tokens.join(',')}` : ''})`, page: path, evidence: [f.signature.join(','), f.hidden.join(',')].filter(Boolean), signature: f.signature, hint: !f.action && !page.inlineFetch && page.inlineScripts > 0 ? 'client-compute?' : 'forms' });
  }
  for (const g of page.controlGroups) add({ class: 'F', feature: `form-less control group in ${g.container} (${g.controls} controls${g.submit ? `, submit "${g.submit}"` : ''})`, page: path, evidence: [g.names.join(',')].filter(Boolean), hint: !page.inlineFetch && page.inlineScripts > 0 ? 'client-compute?' : 'forms' });
  const byMarker = new Map();
  for (const t of page.triggers) { const k = t.marker; const row = byMarker.get(k) || { n: 0, ex: [], targets: new Set(), titles: 0, chrome: 0 }; row.n += 1; if (row.ex.length < 4) row.ex.push(t.href || t.text); if (t.target) row.targets.add(`${t.target.role}:${t.target.hasForm ? 'form' : t.target.hasVideo ? 'video' : t.target.hasIframe ? 'iframe' : 'content'}`); if (t.titleOnTrigger) row.titles += 1; if (t.inChrome) row.chrome += 1; byMarker.set(k, row); }
  for (const [marker, r] of byMarker) add({ class: 'M', feature: `modal trigger ${marker}${r.chrome === r.n ? ' (chrome only)' : ''} → ${[...r.targets].join('/') || 'target outside DOM at capture'}`, page: path, evidence: [...r.ex, r.titles ? `${r.titles} triggers carry the title (data-*title)` : null].filter(Boolean), hint: r.chrome === r.n ? 'chrome-interaction' : 'modal' });
  for (const m of page.media) { const v = vendorFor(m.src || ''); if ((v && v.class === 'V') || m.videoId || m.tag === 'video-js') add({ class: 'V', feature: v ? v.role : `player element <${m.tag}>${m.inDialog ? ' in a dialog' : ''}`, page: path, evidence: [m.videoId ? `${m.account || '?'}/${m.player || 'default'}/${m.videoId}` : m.src], hint: 'media' }); }
  for (const f of page.iframes) if (!f.src) add({ class: 'V', feature: 'iframe without src (runtime-injected embed)', page: path, evidence: [f.title || `${f.w}×${f.h}`], hint: 'embed-runtime' });
  for (const mnt of page.mounts) add({ class: 'T', feature: `third-party mount <div ${mnt.attrs[0] || mnt.cls}> (tag-manager-injected widget)`, page: path, evidence: [mnt.attrs.join(' ') || mnt.cls], hint: 'tags' });
  if (page.auth.length) add({ class: 'X', feature: 'sign-in / account links', page: path, evidence: page.auth.slice(0, 4), hint: 'decided-out' });
  if (page.commerce.cart || page.commerce.prices > 8) add({ class: 'X', feature: `commerce signals (cart: ${page.commerce.cart}, prices: ${page.commerce.prices})`, page: path, hint: 'decided-out' });
  if (page.locale.hreflang.length || page.locale.switcher.length) add({ class: 'I18N', feature: `locale variants ${page.locale.hreflang.join(',') || page.locale.switcher.map((h) => new URL(h).pathname.split('/')[1]).join(',')}`, page: path, evidence: page.locale.switcher.slice(0, 3), hint: 'locale' });
  for (const c of page.clientRendered) add({ class: 'CR', feature: `client-rendered slot ${c.cls}`, page: path, evidence: c.attrs.length ? c.attrs : [c.text], hint: 'client-rendered' });
  if (page.mainEmptyAtLoad) add({ class: 'CR', feature: 'main empty at load, filled after (client-rendered page)', page: path, evidence: [`text at load ${page.textAtLoad} → settled ${page.mainText}`], hint: 'client-rendered-page' });
  for (const l of page.listingCandidates.filter((x) => x.hasDate || x.cards >= 6)) add({ class: 'L', feature: `listing candidate ${l.cls} (${l.cards} cards)`, page: path, hint: 'listings' });
  for (const [k, keys] of Object.entries(page.cms)) if (Array.isArray(keys)) { const named = keys.filter((x) => /endpoint|api|url|marketo|eloqua|hubspot|antibot|search|campaign|schema|ddl|analytics|consent|token|form/i.test(x)); add({ class: 'A', feature: `CMS / app settings object ${k}`, page: path, evidence: named.slice(0, 8), hint: 'settings' }); }
  if (page.framework) add({ class: 'CR', feature: `client framework ${page.framework}`, page: path, hint: 'framework' });
}

/* ---------------------------------------------------------------- main -- */
const browser = await openBrowser({ mode: 'plain', launchOptions: { headless: !flag('headed'), args: ['--disable-blink-features=AutomationControlled'] } }).catch(stopOnSetupError);
const report = { _provenance: provenance('detect', { settleMs: SETTLE, width: WIDTH, urls: URLS }), pages: {}, findings: [] };
const findingsByKey = new Map();
const add = (f) => {
  const key = `${f.class}|${f.feature}`;
  let x = findingsByKey.get(key);
  if (!x) { x = { id: slug(`${f.class}-${f.feature}`), class: f.class, feature: f.feature, role: f.role, api: f.api, signature: f.signature, hint: f.hint, evidence: [], pages: [] }; findingsByKey.set(key, x); report.findings.push(x); }
  if (!x.pages.includes(f.page)) x.pages.push(f.page);
  if (f.evidence) x.evidence = [...new Set([...x.evidence, ...f.evidence.filter(Boolean)])].slice(0, 12);
};

for (const url of URLS) {
  const { host, origin } = new URL(url);
  const path = new URL(url).pathname || '/';
  const ctx = await newContext(browser, { viewport: { width: WIDTH, height: 900 }, locale: 'en-US' });
  await ctx.addInitScript(() => { window.__sdDomAfterLoad = 0; let loaded = false; window.addEventListener('load', () => setTimeout(() => { loaded = true; }, 300)); document.addEventListener('DOMContentLoaded', () => new MutationObserver((ms) => { if (loaded) for (const m of ms) window.__sdDomAfterLoad += m.addedNodes.length; }).observe(document.documentElement, { childList: true, subtree: true })); });
  const page = await ctx.newPage();
  const hosts = {}; const scripts = new Set(); const firstPartyApi = new Map(); const thirdPartyXhr = new Set(); const postBodies = [];
  page.on('request', (r) => { if (r.method() === 'POST' && ['xhr', 'fetch'].includes(r.resourceType()) && postBodies.length < 12) postBodies.push({ url: r.url().slice(0, 200), body: (r.postData() || '').slice(0, 400) }); });
  page.on('response', (resp) => {
    try {
      const req = resp.request(); const u = new URL(resp.url()); const type = req.resourceType();
      if (!/^https?:$/.test(u.protocol)) return;
      hosts[u.host] = (hosts[u.host] || 0) + 1;
      if (type === 'script') scripts.add(resp.url().slice(0, 200));
      const ct = (resp.headers()['content-type'] || '').split(';')[0].trim();
      const dataLike = ['xhr', 'fetch', 'eventsource'].includes(type) || /json|graphql/.test(ct);
      if (!dataLike || type === 'document') return;
      if (sameSite(u.host, host)) { const key = `${req.method()} ${u.host}${pathPattern(u.pathname)}`; if (!firstPartyApi.has(key)) firstPartyApi.set(key, { method: req.method(), host: u.host, path: pathPattern(u.pathname), query: [...new Set([...u.searchParams.keys()])].sort(), status: resp.status(), contentType: ct, example: `${u.origin}${u.pathname}` }); } else thirdPartyXhr.add(`${req.method()} ${u.host}${u.pathname.slice(0, 80)}`);
    } catch { /* evidence only */ }
  });
  let status = 0; let textAtLoad = 0;
  try {
    const resp = await goto(page, url, { waitUntil: 'domcontentloaded', timeout: 60000 });
    status = resp?.status() || 0;
    textAtLoad = await page.evaluate(() => (document.querySelector('main') || document.body)?.innerText.replace(/\s+/g, ' ').trim().length || 0);
    await settlePage(page, { settleMs: SETTLE });
  } catch (e) { report.pages[path] = { url, error: String(e.message).slice(0, 160) }; console.error(`[dynamics] FAIL ${path} ${e.message.slice(0, 100)}`); await ctx.close(); continue; }
  const dom = await page.evaluate(domCapture);
  const pageRec = {
    url, status, host, ...dom, hosts, thirdPartyHosts: Object.keys(hosts).filter((h) => h && !sameSite(h, host)), scripts: [...scripts].slice(0, 60),
    firstPartyApi: [...firstPartyApi.values()], thirdPartyXhr: [...thirdPartyXhr].slice(0, 40), postBodies, textAtLoad, mainEmptyAtLoad: textAtLoad < 200 && dom.mainText > 600,
  };
  report.pages[path] = pageRec;
  classify(pageRec, path, add);
  console.error(`[dynamics] ${status} ${path} · 3rd-party hosts ${pageRec.thirdPartyHosts.length} · 1st-party api ${pageRec.firstPartyApi.length} · forms ${dom.forms.length}+${dom.controlGroups.length} · triggers ${dom.triggers.length} · media ${dom.media.length} · client-rendered ${dom.clientRendered.length}${pageRec.mainEmptyAtLoad ? ' · MAIN-EMPTY-AT-LOAD' : ''}`);
  await ctx.close();
}
await browser.close().catch(() => {});

/* --------------------------------------------- reach from extract pages -- */
if (arg('reach') && arg('reach') !== true) {
  const dir = join(arg('reach'), 'pages');
  if (existsSync(dir)) {
    const files = readdirSync(dir).filter((f) => f.endsWith('.json'));
    let withDyn = 0; const endpointPages = new Map(); const searchPages = new Set(); const formPages = new Set(); const triggerPages = new Map();
    for (const f of files) {
      const rec = readJSON(join(dir, f), {}); const d = rec.dynamic; if (!d) continue; withDyn += 1;
      for (const e of d.endpoints || []) { const k = `${e.method} ${e.host}${e.path}`; endpointPages.set(k, (endpointPages.get(k) || 0) + 1); }
      if ((d.summary?.searchForms || 0) > 0) searchPages.add(rec.slug);
      if ((d.forms || []).some((x) => !x.search)) formPages.add(rec.slug);
      for (const t of d.triggers || []) triggerPages.set(t.marker, (triggerPages.get(t.marker) || 0) + 1);
    }
    report.reach = { pagesWithEvidence: withDyn, of: files.length, endpoints: Object.fromEntries(endpointPages), searchFormPages: searchPages.size, formPages: formPages.size, triggerPages: Object.fromEntries(triggerPages) };
    for (const fnd of report.findings) {
      if (fnd.api) { const n = [...endpointPages.entries()].filter(([k]) => k.startsWith(`${fnd.api.method} `) && k.endsWith(fnd.api.path)).reduce((sum, [, v]) => sum + v, 0); if (n) fnd.reach = { pages: n, of: files.length }; }
      if (fnd.class === 'S' && /site search form/.test(fnd.feature)) fnd.reach = { pages: searchPages.size, of: files.length };
      if (fnd.class === 'M') { const marker = (fnd.feature.match(/modal trigger (\S+)/) || [])[1]; if (marker && triggerPages.has(marker)) fnd.reach = { pages: triggerPages.get(marker), of: files.length }; }
    }
    report._provenance.reachSource = dir;
  }
}

/* -------------------------------------------------------------- output -- */
report.findings.sort((a, b) => a.class.localeCompare(b.class) || b.pages.length - a.pages.length);
writeJSON(join(OUT, '_dynamics.json'), report);
const md = [
  `# Dynamic features — detected (${report._provenance.writtenAt})`, '',
  `Pages probed: ${Object.keys(report.pages).join(', ')} · settle ${SETTLE} ms · width ${WIDTH}${report.reach ? ` · reach from ${report.reach.pagesWithEvidence}/${report.reach.of} crawled pages` : ''}`, '',
  'Evidence only. Every row must receive a disposition in `stardust/dynamic-features.md` (`dynamics-plan.mjs` drafts it).', '',
  '| id | class | feature | pages | reach | evidence | hint |', '|---|---|---|---|---|---|---|',
  ...report.findings.map((f) => `| ${f.id} | ${f.class} ${CLASS_NAMES[f.class] || ''} | ${f.feature.replace(/\|/g, '/')} | ${f.pages.length}/${Object.keys(report.pages).length} | ${f.reach ? `${f.reach.pages}/${f.reach.of}` : ''} | ${(f.evidence || []).slice(0, 3).join('<br>').replace(/\|/g, '/')} | ${f.hint || ''} |`),
];
writeText(join(OUT, 'dynamic-features.generated.md'), md.join('\n'));
console.error(`[dynamics] ${report.findings.length} findings → ${join(OUT, '_dynamics.json')}, dynamic-features.generated.md`);
setTimeout(() => process.exit(0), 200).unref();

skills

README.md

tile.json