#!/usr/bin/env python3
"""Find high-view videos on a topic, without Apify.

app4 replaced a $229/mo Apify ad-library scraper with a self-hosted reader and
got 100% recall at $0. The transferable part is not the browser — it is WHAT it
reads: the page's own embedded JSON, not the DOM, not a vendor's API.

  YouTube   search page embeds ytInitialData with viewCountText per result.
            Plain fetch. No API key, no quota, no browser.
  Instagram i.instagram.com/api/v1/users/web_profile_info returns recent media
            with video_play_count, given the public X-IG-App-ID header. Per
            ACCOUNT, not search — so you curate whose numbers matter, which is
            better signal than hashtag roulette anyway.

Usage:
  find_viral.py --yt "ai agent ecommerce ads" "shopify automation" --min-views 500000
  find_viral.py --ig shopify klaviyo --min-views 200000
  find_viral.py --yt "ai analytics" --lang ru --out backlog/candidates.json
  find_viral.py --selftest
"""
import argparse, json, re, sys, urllib.error, urllib.parse, urllib.request

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
IG_APP_ID = "936619743392459"          # public web client id, not a credential


def get(url, headers=None, timeout=25):
    req = urllib.request.Request(url, headers={"User-Agent": UA, **(headers or {})})
    return urllib.request.urlopen(req, timeout=timeout).read().decode("utf-8", "replace")


def parse_views(text):
    """'1,234,567 views' / '1.2M views' / '12 тыс. просмотров' -> int or None."""
    if not text:
        return None
    t = text.replace(" ", " ").lower()
    m = re.search(r"([\d.,\s]+)\s*([kmб]|тыс|млн|m|k)?", t)
    if not m:
        return None
    num = m.group(1).replace(" ", "").replace(",", "")
    try:
        v = float(num)
    except ValueError:
        return None
    mult = {"k": 1e3, "тыс": 1e3, "m": 1e6, "млн": 1e6, "б": 1e9}.get(m.group(2) or "", 1)
    return int(v * mult)


def youtube(query, lang="en", limit=20):
    """Search results with view counts, read from the page's own ytInitialData."""
    url = "https://www.youtube.com/results?" + urllib.parse.urlencode(
        {"search_query": query, "sp": "CAMSAhAB"})     # sort: view count, video only
    html = get(url, {"Accept-Language": f"{lang}-{lang.upper()},{lang};q=0.9"})
    m = re.search(r"var ytInitialData = (\{.*?\});</script>", html)
    if not m:
        return []
    def walk(o):
        if isinstance(o, dict):
            if "videoRenderer" in o:
                yield o["videoRenderer"]
            for v in o.values():
                yield from walk(v)
        elif isinstance(o, list):
            for v in o:
                yield from walk(v)
    out = []
    for v in list(walk(json.loads(m.group(1))))[:limit]:
        try:
            out.append({
                "platform": "youtube", "query": query, "lang": lang,
                "title": v["title"]["runs"][0]["text"],
                "channel": v.get("ownerText", {}).get("runs", [{}])[0].get("text", ""),
                "url": f"https://youtube.com/watch?v={v['videoId']}",
                "views": parse_views(v.get("viewCountText", {}).get("simpleText")),
                "published": v.get("publishedTimeText", {}).get("simpleText", ""),
            })
        except (KeyError, IndexError):
            continue
    return out


def instagram(username, limit=12):
    """Recent media for one account, with play counts. Public endpoint."""
    url = ("https://i.instagram.com/api/v1/users/web_profile_info/?"
           + urllib.parse.urlencode({"username": username}))
    try:
        d = json.loads(get(url, {"X-IG-App-ID": IG_APP_ID}))
    except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError) as e:
        print(f"  instagram/{username}: unavailable ({e})", file=sys.stderr)
        return []
    user = (d.get("data") or {}).get("user") or {}
    out = []
    for e in (user.get("edge_owner_to_timeline_media") or {}).get("edges", [])[:limit]:
        n = e["node"]
        if not n.get("is_video"):
            continue
        cap = (n.get("edge_media_to_caption") or {}).get("edges") or []
        out.append({
            "platform": "instagram", "query": f"@{username}", "lang": "",
            "title": (cap[0]["node"]["text"][:110] if cap else "(no caption)"),
            "channel": username,
            "url": f"https://instagram.com/p/{n['shortcode']}/",
            "views": n.get("video_play_count") or n.get("video_view_count"),
            "published": "",
        })
    return out


def selftest():
    assert parse_views("1,234,567 views") == 1234567
    assert parse_views("1.2M views") == 1200000
    assert parse_views("500K views") == 500000
    assert parse_views("12 тыс. просмотров") == 12000
    assert parse_views("3,4 млн просмотров") is not None
    assert parse_views("No views") is None and parse_views("") is None
    assert parse_views(None) is None
    # the threshold must not silently admit an unknown count
    rows = [{"views": 600000}, {"views": None}, {"views": 100}]
    assert [r for r in rows if (r["views"] or 0) >= 500000] == [{"views": 600000}]
    print("ok")


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--yt", nargs="*", default=[], help="YouTube search queries")
    ap.add_argument("--ig", nargs="*", default=[], help="Instagram usernames")
    ap.add_argument("--lang", default="en")
    ap.add_argument("--min-views", type=int, default=500000)
    ap.add_argument("--out")
    ap.add_argument("--selftest", action="store_true")
    a = ap.parse_args()
    if a.selftest:
        return selftest()
    if not a.yt and not a.ig:
        ap.error("give --yt queries and/or --ig usernames")

    rows = []
    for q in a.yt:
        r = youtube(q, a.lang)
        print(f"  youtube {q!r} ({a.lang}): {len(r)} results", file=sys.stderr)
        rows += r
    for u in a.ig:
        r = instagram(u)
        print(f"  instagram @{u}: {len(r)} videos", file=sys.stderr)
        rows += r

    # an unknown view count is never assumed to clear the bar
    hits = sorted((r for r in rows if (r["views"] or 0) >= a.min_views),
                  key=lambda r: -(r["views"] or 0))
    seen, uniq = set(), []
    for r in hits:
        if r["url"] not in seen:
            seen.add(r["url"]); uniq.append(r)
    print(f"\n{len(uniq)} of {len(rows)} over {a.min_views:,} views\n")
    for r in uniq:
        print(f"  {r['views']:>12,}  {r['platform']:9} {r['channel'][:20]:20} {r['title'][:58]}")
        print(f"                {r['url']}")
    if a.out:
        json.dump(uniq, open(a.out, "w"), indent=2, ensure_ascii=False)
        print(f"\n-> {a.out}", file=sys.stderr)


if __name__ == "__main__":
    main()