from __future__ import annotations

import csv
import json
import os
import re
import time
from datetime import datetime, timezone, timedelta
from pathlib import Path
from urllib.parse import quote

import requests


ROOT = Path(__file__).resolve().parent
TZ_TAIPEI = timezone(timedelta(hours=8))
COLLECTED_AT = datetime.now(TZ_TAIPEI).isoformat(timespec="seconds")
HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; SHWRP-research/1.0; +https://bento24235111.com/)"}
MIN_REQUEST_INTERVAL_SECONDS = 1.0

PCHOME_QUERIES = ["白米", "台灣米", "糙米", "胚芽米", "香米", "越光米", "米 1公斤", "米 2公斤", "米 3公斤", "米 5公斤"]
RICE_INCLUDE = re.compile(r"(白米|糙米|胚芽米|香米|越光米|台[梗稉]米|池上米|關山米|壽司米|蓬萊米|有機米|履歷米|好米)")
RICE_EXCLUDE = re.compile(r"(米餅|米果|米粉|米麵|米糕|飯糰|即食|沖泡|稀飯|粥|便當|餐盒|米漢堡|爆米|米香|米乳|米漿|原漿|米酒|麻糬|年糕|蘿蔔糕|蛋糕|鍋巴|寵物|貓砂|犬|狗糧|貓糧|飼料|料理包|熟飯|冷凍飯|發芽米漿|杏仁餅|飲品|米麩)")


def request_json(url: str, attempts: int = 4):
    """Read only an unauthenticated public JSON endpoint.

    This collector does not log in, solve or bypass CAPTCHAs, rotate identities,
    evade access controls, or continue after an explicit authorization denial.
    """
    for attempt in range(attempts):
        try:
            response = requests.get(url, headers=HEADERS, timeout=30)
            if response.status_code in {401, 403}:
                raise PermissionError(f"Public access denied ({response.status_code}); collection stopped")
            if response.status_code == 429:
                retry_after = response.headers.get("Retry-After")
                wait_seconds = float(retry_after) if retry_after and retry_after.isdigit() else 5.0 * (attempt + 1)
                time.sleep(wait_seconds)
                continue
            response.raise_for_status()
            return response.json()
        except PermissionError:
            raise
        except Exception:
            if attempt == attempts - 1:
                raise
            time.sleep(1.5 * (attempt + 1))


def extract_weight(name: str, description: str) -> dict:
    text = f"{name} {description}".replace(",", "")
    normalized = text.lower().replace("公斤", "kg").replace("公克", "g").replace("克", "g")
    normalized = normalized.replace("＊", "*").replace("×", "*").replace("x", "*").replace("Ｘ", "*")
    matches = list(re.finditer(r"(?<!\d)(\d+(?:\.\d+)?)\s*(kg|g)", normalized))
    if not matches:
        return {"unit_weight_kg": None, "bundle_count": None, "total_weight_kg": None, "weight_confidence": "unparsed"}

    # Product titles are the most reliable source. Prefer the first title match.
    title_norm = name.lower().replace("公斤", "kg").replace("公克", "g").replace("克", "g")
    title_norm = title_norm.replace("＊", "*").replace("×", "*").replace("x", "*").replace("Ｘ", "*")
    title_matches = list(re.finditer(r"(?<!\d)(\d+(?:\.\d+)?)\s*(kg|g)", title_norm))
    match = title_matches[0] if title_matches else matches[0]
    value = float(match.group(1))
    unit_weight = value if match.group(2) == "kg" else value / 1000
    count = 1

    # Handle common forms such as "2公斤(6入)" where count follows parentheses.
    tail = (title_norm if title_matches else normalized)[match.end(): match.end() + 24]
    count_match = re.search(r"(?:^\s*\*\s*(\d+)|[\(（]?\s*(\d+)\s*(?:入|包|袋|組)[\)）]?)", tail)
    if count_match:
        count = int(count_match.group(1) or count_match.group(2))

    plausible = 0.2 <= unit_weight <= 30 and 1 <= count <= 30
    confidence = "title" if title_matches and plausible else "description" if plausible else "review"
    return {
        "unit_weight_kg": round(unit_weight, 4) if plausible else None,
        "bundle_count": count if plausible else None,
        "total_weight_kg": round(unit_weight * count, 4) if plausible else None,
        "weight_confidence": confidence if plausible else "review",
    }


def classify_product(name: str, description: str) -> dict:
    text = f"{name} {description}"
    return {
        "rice_type": "brown_or_germ" if re.search(r"糙米|胚芽米", text) else "white_rice",
        "organic": int(bool(re.search(r"有機", text))),
        "traceable": int(bool(re.search(r"產銷履歷|TAP|履歷", text, re.I))),
        "award": int(bool(re.search(r"得獎|冠軍|金賞|米王|十大經典|競賽", text))),
        "vacuum": int(bool(re.search(r"真空", text))),
        "gift": int(bool(re.search(r"禮盒|送禮", text))),
        "variety_named": int(bool(re.search(r"台[梗稉]\s*\d+號|高雄\s*\d+號|越光|秈米|香米", text))),
        "origin_named": int(bool(re.search(r"池上|關山|花蓮|富里|宜蘭|台東|臺東|雲林|彰化|嘉義|屏東|桃園|臺灣|台灣", text))),
        "text_small_household": int(bool(re.search(r"小家庭|單身|一人|小包裝|保鮮|少量|獨居", text))),
    }


def collect_products() -> list[dict]:
    if os.environ.get("PCHOME_COLLECTION_AUTHORIZED") != "YES":
        raise RuntimeError(
            "PChome collection is disabled by default. Recheck the platform's current terms "
            "and obtain any required permission before setting PCHOME_COLLECTION_AUTHORIZED=YES."
        )
    products: dict[str, dict] = {}
    for query in PCHOME_QUERIES:
        page = 1
        while page <= 40:
            url = f"https://ecshweb.pchome.com.tw/search/v4.3/all/results?q={quote(query)}&page={page}&sort=sale/dc"
            payload = request_json(url)
            rows = payload.get("Prods") or payload.get("prods") or []
            if not rows:
                break
            for item in rows:
                name = (item.get("Name") or item.get("name") or "").strip()
                description = (item.get("Describe") or item.get("describe") or "").strip()
                if not RICE_INCLUDE.search(name) or RICE_EXCLUDE.search(name):
                    continue
                price = item.get("Price") if item.get("Price") is not None else item.get("price")
                if not isinstance(price, (int, float)) or price <= 0:
                    continue
                parsed = extract_weight(name, description)
                product_id = item.get("Id") or item.get("id")
                if not product_id:
                    continue
                row = {
                    "product_id": product_id,
                    "source": "PChome 24h",
                    "query_first_seen": query,
                    "name": name,
                    "description": description.replace("\r", " ").replace("\n", " "),
                    "price_ntd": float(price),
                    "original_price_ntd": float(item.get("OriginPrice") or item.get("originPrice") or price),
                    "promoted": int(float(item.get("OriginPrice") or item.get("originPrice") or price) > float(price)),
                    "product_url": f"https://24h.pchome.com.tw/prod/{product_id}",
                    "collected_at": COLLECTED_AT,
                    **parsed,
                    **classify_product(name, description),
                }
                if row["total_weight_kg"]:
                    row["unit_price_ntd_per_kg"] = round(row["price_ntd"] / row["total_weight_kg"], 4)
                    row["package_size_group"] = (
                        "small_le_2kg" if row["unit_weight_kg"] <= 2
                        else "medium_2_to_3kg" if row["unit_weight_kg"] <= 3
                        else "large_3_to_5kg" if row["unit_weight_kg"] <= 5
                        else "bulk_over_5kg"
                    )
                else:
                    row["unit_price_ntd_per_kg"] = None
                    row["package_size_group"] = "unparsed"
                products.setdefault(product_id, row)
            page += 1
            if page > int(payload.get("TotalPage") or payload.get("totalPage") or page):
                break
            time.sleep(MIN_REQUEST_INTERVAL_SECONDS)
    rows = sorted(products.values(), key=lambda x: x["product_id"])
    fields = list(rows[0].keys()) if rows else []
    with (ROOT / "products-raw.csv").open("w", newline="", encoding="utf-8-sig") as handle:
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        writer.writerows(rows)
    return rows


def collect_households() -> list[dict]:
    output = []
    for roc_year in range(106, 115):
        page = 1
        while True:
            url = f"https://www.ris.gov.tw/rs-opendata/api/v1/datastore/ODRP025/{roc_year}?page={page}"
            payload = request_json(url)
            data = payload.get("responseData") or []
            output.extend(data)
            if page >= int(payload.get("totalPage") or 1):
                break
            page += 1
            time.sleep(MIN_REQUEST_INTERVAL_SECONDS)
    fields = sorted({key for row in output for key in row.keys()}) if output else []
    with (ROOT / "household-structure-2017-2025.csv").open("w", newline="", encoding="utf-8-sig") as handle:
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        writer.writerows(output)
    return output


if __name__ == "__main__":
    products = collect_products()
    households = collect_households()
    metadata = {
        "collected_at": COLLECTED_AT,
        "product_source": "PChome 24h public search endpoint",
        "product_queries": PCHOME_QUERIES,
        "product_rows_after_keyword_filter": len(products),
        "household_source": "Ministry of the Interior Department of Household Registration, ODRP025",
        "household_rows": len(households),
        "household_years": [2017, 2025],
    }
    (ROOT / "collection-metadata.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(metadata, ensure_ascii=False, indent=2))
