"""Raqib — polite catalogue collector and price monitor.

Collects every product of https://books.toscrape.com (a sandbox built for scraping practice),
cleans and validates the records, and writes CSV, JSON and a formatted Excel report.
Run again later and pass --previous to get a change report (new items, removed items, price changes).

    python3 raqib.py --out ../data                     # full crawl
    python3 raqib.py --out ../data --previous old.json # crawl + diff against an earlier snapshot

Design choices: limited concurrency, a request delay, retries with backoff, a descriptive
User-Agent, and no login, CAPTCHA or paywall bypass. Only use on sites whose terms allow it.
"""
from __future__ import annotations

import argparse
import concurrent.futures as cf
import csv
import json
import re
import sys
import time
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

BASE = "https://books.toscrape.com/"
UA = "RaqibDemo/1.0 (+https://cadrienstudio.com; portfolio demo, polite crawler)"
RATING = {"One": 1, "Two": 2, "Three": 3, "Four": 4, "Five": 5}


@dataclass
class Book:
    upc: str
    title: str
    category: str
    price_gbp: float
    rating: int
    in_stock: int
    url: str
    description_words: int


class Fetcher:
    def __init__(self, delay: float = 0.15, retries: int = 3):
        self.s = requests.Session()
        self.s.headers["User-Agent"] = UA
        self.delay, self.retries = delay, retries

    def get(self, url: str) -> str:
        for attempt in range(self.retries):
            try:
                r = self.s.get(url, timeout=20)
                if r.status_code == 200:
                    time.sleep(self.delay)
                    r.encoding = "utf-8"
                    return r.text
                if r.status_code in (429, 500, 502, 503, 504):
                    time.sleep(2 ** attempt)
                    continue
                r.raise_for_status()
            except requests.RequestException:
                if attempt == self.retries - 1:
                    raise
                time.sleep(2 ** attempt)
        raise RuntimeError(f"failed after retries: {url}")


def listing_urls(f: Fetcher) -> list[str]:
    """Walk the paginated catalogue and return every product URL."""
    urls, page = [], urljoin(BASE, "catalogue/page-1.html")
    while page:
        soup = BeautifulSoup(f.get(page), "html.parser")
        urls += [urljoin(page, a["href"]) for a in soup.select("article.product_pod h3 a")]
        nxt = soup.select_one("li.next a")
        page = urljoin(page, nxt["href"]) if nxt else None
    return urls


def parse_book(html: str, url: str) -> Book:
    soup = BeautifulSoup(html, "html.parser")
    table = {th.get_text(strip=True): td.get_text(strip=True) for th, td in zip(soup.select("table th"), soup.select("table td"))}
    price = float(re.sub(r"[^\d.]", "", table.get("Price (incl. tax)", "0")) or 0)
    m = re.search(r"(\d+)", table.get("Availability", ""))
    stock = int(m.group(1)) if m else 0
    rating_cls = soup.select_one("p.star-rating")["class"]
    rating = next((RATING[c] for c in rating_cls if c in RATING), 0)
    crumbs = soup.select("ul.breadcrumb li a")
    desc = soup.select_one("#product_description + p")
    return Book(
        upc=table.get("UPC", ""),
        title=soup.select_one("div.product_main h1").get_text(strip=True),
        category=crumbs[-1].get_text(strip=True) if crumbs else "",
        price_gbp=round(price, 2),
        rating=rating,
        in_stock=stock,
        url=url,
        description_words=len(desc.get_text(" ", strip=True).split()) if desc else 0,
    )


def validate(books: list[Book]) -> tuple[list[Book], list[dict]]:
    """Drop duplicates and records that fail basic rules; report every rejection."""
    seen, ok, rejected = set(), [], []
    for b in books:
        problems = []
        if not re.fullmatch(r"[0-9a-f]{16}", b.upc):
            problems.append("bad_upc")
        if b.upc in seen:
            problems.append("duplicate")
        if not (0 < b.price_gbp < 1000):
            problems.append("price_out_of_range")
        if not (1 <= b.rating <= 5):
            problems.append("missing_rating")
        if problems:
            rejected.append({"upc": b.upc, "title": b.title, "problems": problems})
        else:
            seen.add(b.upc)
            ok.append(b)
    return ok, rejected


def diff(previous: list[dict], current: list[Book]) -> dict:
    prev = {p["upc"]: p for p in previous}
    cur = {b.upc: b for b in current}
    changes = [
        {"upc": u, "title": b.title, "old": prev[u]["price_gbp"], "new": b.price_gbp, "delta_pct": round((b.price_gbp - prev[u]["price_gbp"]) / prev[u]["price_gbp"] * 100, 1)}
        for u, b in cur.items() if u in prev and abs(prev[u]["price_gbp"] - b.price_gbp) > 0.004
    ]
    return {
        "new": [asdict(cur[u]) for u in cur.keys() - prev.keys()],
        "removed": [prev[u] for u in prev.keys() - cur.keys()],
        "price_changes": sorted(changes, key=lambda c: abs(c["delta_pct"]), reverse=True),
    }


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--out", default="data")
    ap.add_argument("--workers", type=int, default=4)
    ap.add_argument("--limit", type=int, default=0, help="stop after N products (0 = all)")
    ap.add_argument("--previous", help="earlier books.json snapshot to diff against")
    a = ap.parse_args(argv)
    out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
    started = time.time(); f = Fetcher()
    urls = listing_urls(f)
    if a.limit:
        urls = urls[: a.limit]
    print(f"found {len(urls)} product pages", file=sys.stderr)
    books, errors = [], []
    with cf.ThreadPoolExecutor(max_workers=a.workers) as ex:
        futs = {ex.submit(f.get, u): u for u in urls}
        for i, fut in enumerate(cf.as_completed(futs), 1):
            u = futs[fut]
            try:
                books.append(parse_book(fut.result(), u))
            except Exception as e:  # keep going; report the page
                errors.append({"url": u, "error": str(e)})
            if i % 100 == 0:
                print(f"  {i}/{len(urls)}", file=sys.stderr)
    books.sort(key=lambda b: (b.category, b.title))
    ok, rejected = validate(books)
    meta = {"source": BASE, "collected_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "pages": len(urls), "records": len(ok),
            "rejected": len(rejected), "fetch_errors": len(errors), "seconds": round(time.time() - started, 1)}
    (out / "books.json").write_text(json.dumps({"meta": meta, "books": [asdict(b) for b in ok]}, ensure_ascii=False, indent=1))
    with open(out / "books.csv", "w", newline="", encoding="utf-8-sig") as fh:
        w = csv.DictWriter(fh, fieldnames=list(asdict(ok[0]).keys())); w.writeheader(); w.writerows(asdict(b) for b in ok)
    (out / "quality-report.json").write_text(json.dumps({"rejected": rejected, "fetch_errors": errors}, ensure_ascii=False, indent=1))
    if a.previous:
        d = diff(json.loads(Path(a.previous).read_text())["books"], ok)
        (out / "changes.json").write_text(json.dumps(d, ensure_ascii=False, indent=1))
        print(f"changes: {len(d['new'])} new, {len(d['removed'])} removed, {len(d['price_changes'])} price changes", file=sys.stderr)
    print(json.dumps(meta), file=sys.stderr)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())