import subprocess
import sys
import time
from pathlib import Path

import requests
from bs4 import BeautifulSoup

from library_paths import locations_for

# Plain URLs. The briefing wrapped these in Markdown links, which are not valid Python strings.
BASE_URL = "https://www.gov.uk"
SEARCH_URL = "https://www.gov.uk/api/search.json"
PAGE_SIZE = 100
PAUSE_SECONDS = 0.2

HEADERS = {
    "User-Agent": "gds-assessment-plugin/1.0 (public assessment-report ingestion)"
}
CURATED = {
    "discovery-lead-stop.md",
    "alpha-tech-spikes.md",
    "beta-ur-fail-assistive-tech.md",
    "beta-tech-pass.md",
}


def report_links(session):
    """Every published service standard report, via the GOV.UK search API.

    The HTML index is paginated at 50 per page and the first version of this
    script kept only the first 15 links on page 1. The index currently lists
    about 500 reports.
    """
    links = []
    start = 0
    total = None
    while total is None or start < total:
        resp = session.get(
            SEARCH_URL,
            params={
                "filter_format": "service_standard_report",
                "count": PAGE_SIZE,
                "start": start,
                "fields": "link",
                "order": "-public_timestamp",
            },
            timeout=30,
        )
        resp.raise_for_status()
        data = resp.json()
        total = data["total"]
        batch = data.get("results") or []
        if not batch:
            break
        for item in batch:
            link = item.get("link")
            if link and link not in links:
                links.append(link)
        start += len(batch)
    return links


def filename_for(path):
    slug = path.rstrip("/").split("/")[-1]
    return f"{slug}.md"


def remove_truncated_scrapes(reports_dir, kept_names):
    """Drop the earlier 50-character title files so they do not sit beside the full set."""
    if not reports_dir.is_dir():
        return
    for path in reports_dir.iterdir():
        name = path.name
        if not path.is_file() or not name.endswith(".md") or name in kept_names or name in CURATED:
            continue
        head = path.read_text(encoding="utf-8")[:300]
        if "Source: https://www.gov.uk/service-standard-reports/" in head:
            path.unlink()
            print(f"Removed truncated scrape: {name}")


def scrape(reports_dir, catalogue_dir):
    reports_dir.mkdir(parents=True, exist_ok=True)
    session = requests.Session()
    session.headers.update(HEADERS)

    links = report_links(session)
    print(f"Found {len(links)} published reports")
    ingested = 0

    for index, path in enumerate(links, start=1):
        report_url = f"{BASE_URL}{path}" if path.startswith("/") else path
        filename = filename_for(path)
        if filename in CURATED:
            print(f"Skipped curated exemplar name: {report_url}")
            continue

        try:
            r_resp = session.get(report_url, timeout=30)
            r_resp.raise_for_status()
        except requests.RequestException as exc:
            print(f"Skipped {report_url}: {exc}")
            time.sleep(PAUSE_SECONDS)
            continue

        r_soup = BeautifulSoup(r_resp.text, "html.parser")
        title = r_soup.find("h1").get_text(strip=True) if r_soup.find("h1") else "unknown"
        body = r_soup.find("div", class_="gem-c-govspeak")
        if not body:
            print(f"Skipped {report_url}: no report body")
            time.sleep(PAUSE_SECONDS)
            continue

        content = f"# {title}\nSource: {report_url}\n\n{body.get_text()}"
        (reports_dir / filename).write_text(content, encoding="utf-8")
        ingested += 1
        print(f"Ingested {index}/{len(links)}: {title}")
        time.sleep(PAUSE_SECONDS)

    remove_truncated_scrapes(reports_dir, {filename_for(path) for path in links})
    print(f"Done. Wrote {ingested} reports to {reports_dir}")
    catalogue = Path(__file__).resolve().parent / "build_past_assessment_catalogue.py"
    subprocess.check_call(
        [sys.executable, str(catalogue), "--reports", str(reports_dir), "--catalogue", str(catalogue_dir)]
    )


def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Download published GOV.UK service assessment reports into a project library."
    )
    parser.add_argument(
        "library",
        nargs="?",
        help="Folder to create. Default: docs/past-assessments in the current directory, or context/ in this plugin checkout.",
    )
    args = parser.parse_args()
    reports_dir, catalogue_dir = locations_for(args.library, Path(__file__))
    scrape(reports_dir, catalogue_dir)


if __name__ == "__main__":
    main()