#!/usr/bin/env python3
"""
clean_scrape.py — distil an X (Twitter) thread/article scrape down to ONE author's prose.

The reference-author corpora under corpus/<name>/_raw/ are raw thread exports. They
interleave the target author's article + tweets with OTHER people's replies and a whole
"## Top Comments" section, plus engagement scaffolding (Likes:/Author:/Posted:/URL:). Fed
straight into profiler.py that measures ~7 people + export metadata, not one voice — which
is why the first Tariq card showed colon_per_k≈98 (timestamps + "Likes:"/"URL:") and
pull_quotes≈84 (every line is a '>' quote).

What this keeps, per file:
  - the leading ARTICLE body  (the target author's long-form piece / thread opener — all
    raw articles are pure '>'-quoted prose with no internal '#' headers, so the scaffolding
    markers below are unambiguous and never collide with real content)
  - any "### N. Thread Post" block whose Author URL is the target handle
What it drops:
  - everything under "## Top Comments", every non-target reply, and all
    Likes:/Author:/Posted:/URL: + "### N."/"## Thread" header scaffolding
Then it unwraps '>' blockquotes and HTML entities so punctuation/structure features
reflect prose, not export noise.

Attribution is by the handle in each block's Author *URL* (x.com/<handle>/status/…),
not the display name — names are unreliable, the URL is canonical.

Usage:
  python clean_scrape.py --handle trq212 --raw corpus/tariq/_raw --out corpus/tariq
"""
from __future__ import annotations

import argparse
import glob
import html
import os
import re

SECTION_HDR = re.compile(r"^\s*##\s+(Thread|Top\s+Comments)\b", re.I)   # section label lines
COMMENTS = re.compile(r"^\s*##\s+Top\s+Comments\b", re.I)               # start of replies-by-others
BLOCK_HDR = re.compile(r"^\s*###\s+\d+\.")                              # "### 2. Thread Post" / "### 1. @x"
AUTHOR_LINE = re.compile(r"^\s*Author\s*:", re.I)
SCAFFOLD = re.compile(r"^\s*(Likes|Retweets|Author|Posted|URL)\s*:", re.I)
ANY_HEADER = re.compile(r"^\s*#{1,6}\s")
HANDLE_IN_URL = re.compile(r"x\.com/([A-Za-z0-9_\\]+)/status", re.I)


def author_of(line: str) -> str | None:
    """The real posting handle, from 'Author: … URL: x.com/<handle>/status/…'."""
    m = HANDLE_IN_URL.search(line)
    return m.group(1).replace("\\", "").lower() if m else None


def clean_line(line: str) -> str:
    line = re.sub(r"^\s*>\s?", "", line)            # unwrap blockquote marker
    line = html.unescape(line)                       # &gt; &amp; &lt; -> > & <
    line = re.sub(r"\\([.\-_>&#*`~])", r"\1", line)  # undo the scrape's backslash-escapes
    return line.rstrip()


def distil(text: str, handle: str) -> str:
    """Return only `handle`'s prose from one raw scrape file."""
    handle = handle.lstrip("@").lower()
    lines = text.splitlines()
    kept: list[str] = []
    in_comments = False
    keep_block = True   # pre-first-block content (the article / thread opener) is the author's

    i = 0
    while i < len(lines):
        line = lines[i]

        if COMMENTS.match(line):           # everything from here on is other people
            in_comments = True
            i += 1
            continue
        if SECTION_HDR.match(line):        # bare "## Thread" label — drop, keep attribution
            i += 1
            continue
        if BLOCK_HDR.match(line):          # new tweet/comment — attribute by its Author URL
            keep_block = False
            j = i + 1
            while j < len(lines) and j < i + 4 and not AUTHOR_LINE.match(lines[j]):
                j += 1
            if j < len(lines) and AUTHOR_LINE.match(lines[j]):
                keep_block = (not in_comments) and (author_of(lines[j]) == handle)
            i += 1
            continue

        if in_comments or not keep_block:                 # someone else's text
            i += 1
            continue
        if SCAFFOLD.match(line) or ANY_HEADER.match(line):  # leftover metadata / headers
            i += 1
            continue

        kept.append(clean_line(line))
        i += 1

    out = re.sub(r"\n{3,}", "\n\n", "\n".join(kept)).strip()
    return out + "\n" if out else ""


def main(argv=None):
    ap = argparse.ArgumentParser(description="Distil an X thread/article scrape to one author's prose.")
    ap.add_argument("--handle", required=True, help="target author handle, e.g. trq212 or @trq212")
    ap.add_argument("--raw", required=True, help="dir of raw scrape *.md files")
    ap.add_argument("--out", required=True, help="dir to write cleaned post_*.md files")
    ap.add_argument("--glob", default="*.md", help="filename pattern within --raw (default *.md)")
    a = ap.parse_args(argv)

    os.makedirs(a.out, exist_ok=True)
    files = sorted(glob.glob(os.path.join(a.raw, a.glob)))
    if not files:
        raise SystemExit(f"No files matched {a.glob} in {a.raw}")

    kept_files, total_words = 0, 0
    for f in files:
        raw = open(f, encoding="utf-8").read()
        prose = distil(raw, a.handle)
        name = os.path.basename(f)
        wc = len(re.findall(r"[A-Za-z']+", prose))
        if wc < 1:
            print(f"  skip  {name:14} (no @{a.handle.lstrip('@')} prose)")
            continue
        open(os.path.join(a.out, name), "w", encoding="utf-8").write(prose)
        kept_files += 1
        total_words += wc
        print(f"  write {name:14} {wc:>5} words")
    print(f"\n{kept_files} files, {total_words:,} words of @{a.handle.lstrip('@')} prose -> {a.out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())