from __future__ import annotations

import csv
import io
import json
import time
import urllib.error
import urllib.parse
import urllib.request
import urllib.robotparser
from collections import defaultdict
from html.parser import HTMLParser
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
from matplotlib.patches import Patch, Rectangle

try:
    from pypdf import PdfReader
except ImportError:
    from PyPDF2 import PdfReader


ROOT = Path(__file__).resolve().parent
FIGURES = ROOT / "figures"
FIGURES.mkdir(exist_ok=True)
BASE = "https://bento24235111.com"
RETRIEVED_ON = "2026-07-15"
USER_AGENT = "SHWRP-technical-audit/1.0 (+mailto:handson0102@hotmail.com)"
REQUEST_DELAY_SECONDS = 0.5

GROUPS = {
    "technical": ["http_ok", "indexable_meta", "self_canonical", "in_sitemap", "linked_from_index", "pdf_http_ok"],
    "crawler": ["googlebot_allowed", "oai_searchbot_allowed", "perplexitybot_allowed"],
    "semantic": ["html_lang_present", "single_h1", "title_present", "meta_description_present", "substantial_main_text", "source_links_present"],
    "citation": [
        "scholarly_jsonld", "jsonld_author", "jsonld_dates", "citation_title", "citation_author",
        "citation_publication_date", "citation_pdf_url", "doi_present", "orcid_present", "pdf_searchable",
        "rights_correction_present",
    ],
}
FIELDS = [field for fields in GROUPS.values() for field in fields]


class PageParser(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.title_parts: list[str] = []
        self.in_title = False
        self.main_depth = 0
        self.main_text: list[str] = []
        self.html_lang = ""
        self.h1_count = 0
        self.metas: dict[str, list[str]] = defaultdict(list)
        self.canonical = ""
        self.links: list[str] = []
        self.in_jsonld = False
        self.jsonld_parts: list[str] = []
        self.jsonld_blocks: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        values = {key.lower(): (value or "") for key, value in attrs}
        if tag == "html":
            self.html_lang = values.get("lang", "")
        elif tag == "title":
            self.in_title = True
        elif tag == "main":
            self.main_depth += 1
        elif tag == "h1":
            self.h1_count += 1
        elif tag == "meta":
            key = values.get("name", "").lower() or values.get("property", "").lower()
            if key:
                self.metas[key].append(values.get("content", "").strip())
        elif tag == "link":
            rel = values.get("rel", "").lower().split()
            href = values.get("href", "").strip()
            if "canonical" in rel:
                self.canonical = href
            if href:
                self.links.append(href)
        elif tag == "a":
            href = values.get("href", "").strip()
            if href:
                self.links.append(href)
        elif tag == "script" and values.get("type", "").lower() == "application/ld+json":
            self.in_jsonld = True
            self.jsonld_parts = []

    def handle_endtag(self, tag: str) -> None:
        if tag == "title":
            self.in_title = False
        elif tag == "main" and self.main_depth:
            self.main_depth -= 1
        elif tag == "script" and self.in_jsonld:
            self.jsonld_blocks.append("".join(self.jsonld_parts).strip())
            self.in_jsonld = False

    def handle_data(self, data: str) -> None:
        value = " ".join(data.split())
        if not value:
            return
        if self.in_title:
            self.title_parts.append(value)
        if self.main_depth:
            self.main_text.append(value)
        if self.in_jsonld:
            self.jsonld_parts.append(data)


def request(url: str) -> tuple[int, bytes, str]:
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
    try:
        with urllib.request.urlopen(req, timeout=30) as response:
            body = response.read()
            return response.status, body, response.headers.get("Content-Type", "")
    except urllib.error.HTTPError as error:
        return error.code, error.read(), error.headers.get("Content-Type", "")


def normalize(url: str) -> str:
    return url.rstrip("/")


def flatten_jsonld(value: object) -> list[dict[str, object]]:
    if isinstance(value, list):
        return [item for value in value for item in flatten_jsonld(item)]
    if not isinstance(value, dict):
        return []
    items = [value]
    graph = value.get("@graph")
    if isinstance(graph, list):
        items.extend(item for node in graph for item in flatten_jsonld(node))
    return items


def searchable_pdf(url: str) -> tuple[int, int]:
    status, body, _ = request(url)
    if status != 200:
        return status, 0
    try:
        reader = PdfReader(io.BytesIO(body))
        chars = sum(len((page.extract_text() or "").strip()) for page in reader.pages)
        return status, chars
    except Exception:
        return status, 0


status, sitemap_bytes, _ = request(f"{BASE}/sitemap.xml")
if status != 200:
    raise RuntimeError("sitemap.xml could not be retrieved")
sitemap_text = sitemap_bytes.decode("utf-8", errors="replace")
sitemap_urls = {normalize(part.split("</loc>", 1)[0]) for part in sitemap_text.split("<loc>")[1:]}

status, index_bytes, _ = request(f"{BASE}/ai-upgrade-roadmap")
if status != 200:
    raise RuntimeError("research index could not be retrieved")
index_parser = PageParser()
index_parser.feed(index_bytes.decode("utf-8", errors="replace"))
index_links = {normalize(urllib.parse.urljoin(BASE, link)) for link in index_parser.links}

robots_url = f"{BASE}/robots.txt"
robots = urllib.robotparser.RobotFileParser()
robots.set_url(robots_url)
robots.read()

with (ROOT / "audit-input.csv").open(encoding="utf-8-sig", newline="") as handle:
    inputs = list(csv.DictReader(handle))

rows: list[dict[str, object]] = []
for index, source in enumerate(inputs):
    if index:
        time.sleep(REQUEST_DELAY_SECONDS)
    url = normalize(source["url"])
    status, body, content_type = request(url)
    html = body.decode("utf-8", errors="replace") if "html" in content_type else ""
    parser = PageParser()
    parser.feed(html)
    nodes: list[dict[str, object]] = []
    for block in parser.jsonld_blocks:
        try:
            nodes.extend(flatten_jsonld(json.loads(block)))
        except json.JSONDecodeError:
            continue
    scholarly = next((node for node in nodes if node.get("@type") == "ScholarlyArticle" or "ScholarlyArticle" in (node.get("@type") or [])), {})
    absolute_links = {urllib.parse.urljoin(url + "/", link) for link in parser.links}
    external_links = {
        link for link in absolute_links
        if urllib.parse.urlparse(link).scheme == "https" and urllib.parse.urlparse(link).netloc not in {"", "bento24235111.com"}
    }
    pdf_url = (parser.metas.get("citation_pdf_url") or [""])[0]
    pdf_status, pdf_text_chars = searchable_pdf(pdf_url) if pdf_url else (0, 0)
    main_text = " ".join(parser.main_text)
    robots_meta = " ".join(parser.metas.get("robots", [])).lower()
    lower_html = html.lower()
    row: dict[str, object] = {
        "paper_id": source["paper_id"],
        "url": url,
        "title": " ".join(parser.title_parts),
        "http_status": status,
        "main_text_chars": len(main_text),
        "external_source_link_count": len(external_links),
        "pdf_url": pdf_url,
        "pdf_status": pdf_status,
        "pdf_text_chars": pdf_text_chars,
        "http_ok": int(status == 200),
        "indexable_meta": int("noindex" not in robots_meta),
        "self_canonical": int(normalize(urllib.parse.urljoin(url, parser.canonical)) == url),
        "in_sitemap": int(url in sitemap_urls),
        "linked_from_index": int(url in index_links),
        "pdf_http_ok": int(pdf_status == 200),
        "googlebot_allowed": int(robots.can_fetch("Googlebot", url)),
        "oai_searchbot_allowed": int(robots.can_fetch("OAI-SearchBot", url)),
        "perplexitybot_allowed": int(robots.can_fetch("PerplexityBot", url)),
        "html_lang_present": int(bool(parser.html_lang)),
        "single_h1": int(parser.h1_count == 1),
        "title_present": int(bool(" ".join(parser.title_parts).strip())),
        "meta_description_present": int(bool((parser.metas.get("description") or [""])[0])),
        "substantial_main_text": int(len(main_text) >= 1000),
        "source_links_present": int(len(external_links) >= 3),
        "scholarly_jsonld": int(bool(scholarly)),
        "jsonld_author": int(bool(scholarly.get("author")) if scholarly else False),
        "jsonld_dates": int(bool(scholarly.get("datePublished") and scholarly.get("dateModified")) if scholarly else False),
        "citation_title": int(bool((parser.metas.get("citation_title") or [""])[0])),
        "citation_author": int(bool((parser.metas.get("citation_author") or [""])[0])),
        "citation_publication_date": int(bool((parser.metas.get("citation_publication_date") or [""])[0])),
        "citation_pdf_url": int(bool(pdf_url and urllib.parse.urlparse(pdf_url).netloc == "bento24235111.com")),
        "doi_present": int("10.5281/zenodo." in lower_html),
        "orcid_present": int("0009-0000-7018-5096" in html),
        "pdf_searchable": int(pdf_text_chars >= 1000),
        "rights_correction_present": int(
            ("第三方權利" in html or "cc by 4.0" in lower_html)
            and ("更正申請" in html or "更正政策" in html)
        ),
    }
    row["total_score"] = sum(int(row[field]) for field in FIELDS)
    for group, fields in GROUPS.items():
        row[f"{group}_score"] = sum(int(row[field]) for field in fields)
    rows.append(row)

output_fields = [
    "paper_id", "url", "title", "http_status", "main_text_chars", "external_source_link_count",
    "pdf_url", "pdf_status", "pdf_text_chars", *FIELDS,
    *[f"{group}_score" for group in GROUPS], "total_score",
]
with (ROOT / "audit-results.csv").open("w", encoding="utf-8-sig", newline="") as handle:
    writer = csv.DictWriter(handle, fieldnames=output_fields)
    writer.writeheader()
    writer.writerows(rows)

indicator_summary = {
    field: {
        "count": sum(int(row[field]) for row in rows),
        "total": len(rows),
        "percent": round(sum(int(row[field]) for row in rows) / len(rows) * 100, 1),
    }
    for field in FIELDS
}
group_summary = {
    group: {
        "passed": sum(int(row[field]) for row in rows for field in fields),
        "possible": len(rows) * len(fields),
        "percent": round(sum(int(row[field]) for row in rows for field in fields) / (len(rows) * len(fields)) * 100, 1),
    }
    for group, fields in GROUPS.items()
}
results = {
    "retrieved_on": RETRIEVED_ON,
    "sample_size": len(rows),
    "indicator_count": len(FIELDS),
    "group_definitions": GROUPS,
    "group_summary": group_summary,
    "indicator_summary": indicator_summary,
    "score_summary": {
        "minimum": min(int(row["total_score"]) for row in rows),
        "maximum": max(int(row["total_score"]) for row in rows),
        "mean": round(sum(int(row["total_score"]) for row in rows) / len(rows), 2),
    },
    "pages": [{key: row[key] for key in ["paper_id", "url", "total_score", *[f"{group}_score" for group in GROUPS]]} for row in rows],
    "guardrail": "本分數是研究者定義的技術準備度描述，不是Google、OpenAI、Microsoft或Perplexity的排名／引用分數，也未驗證對曝光、引用或流量的因果效果。",
}
(ROOT / "analysis-results.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

font_path = ROOT.parents[1] / "public" / "fonts" / "NotoSansTC-Regular.ttf"
if font_path.exists():
    from matplotlib.font_manager import FontProperties
    font = FontProperties(fname=str(font_path))
    plt.rcParams["axes.unicode_minus"] = False
else:
    font = None

group_labels = {"technical": "可發現性", "crawler": "AI爬蟲存取", "semantic": "文字與結構", "citation": "來源可引用性"}
colors = ["#6f3519", "#9a5a2c", "#315c4c", "#557c6f"]

fig, ax = plt.subplots(figsize=(9.5, 5.6))
groups = list(GROUPS)
values = [group_summary[group]["percent"] for group in groups]
bars = ax.bar([group_labels[group] for group in groups], values, color=colors)
for bar, value in zip(bars, values):
    ax.text(bar.get_x() + bar.get_width() / 2, value + 1.2, f"{value:.1f}%", ha="center", fontproperties=font)
ax.set_ylim(0, 108)
ax.set_ylabel("通過比例（%）", fontproperties=font)
ax.set_title("圖1　十篇研究頁面的AI搜尋技術準備度", fontproperties=font, fontsize=15)
ax.grid(axis="y", alpha=0.2)
for item in [ax.title, ax.xaxis.label, ax.yaxis.label, *ax.get_xticklabels(), *ax.get_yticklabels()]:
    item.set_fontproperties(font)
fig.tight_layout()
fig.savefig(FIGURES / "figure-1-readiness-groups.png", dpi=180)
plt.close(fig)

matrix = np.array([[int(row[field]) for row in rows] for field in FIELDS])
indicator_labels = {
    "http_ok": "頁面 HTTP 200",
    "indexable_meta": "未設定 noindex",
    "self_canonical": "自我指向 canonical",
    "in_sitemap": "收錄於 Sitemap",
    "linked_from_index": "研究列表可連入",
    "pdf_http_ok": "PDF 可正常開啟",
    "googlebot_allowed": "允許 Googlebot",
    "oai_searchbot_allowed": "允許 OAI-SearchBot",
    "perplexitybot_allowed": "允許 PerplexityBot",
    "html_lang_present": "標示 HTML 語言",
    "single_h1": "單一 H1 標題",
    "title_present": "頁面標題存在",
    "meta_description_present": "Meta description 存在",
    "substantial_main_text": "可見全文達門檻",
    "source_links_present": "原始來源連結存在",
    "scholarly_jsonld": "ScholarlyArticle JSON-LD",
    "jsonld_author": "JSON-LD 作者",
    "jsonld_dates": "JSON-LD 出版／修訂日期",
    "citation_title": "Scholar 論文標題",
    "citation_author": "Scholar 作者",
    "citation_publication_date": "Scholar 出版日期",
    "citation_pdf_url": "Scholar PDF 網址",
    "doi_present": "DOI 存在",
    "orcid_present": "ORCID 存在",
    "pdf_searchable": "PDF 文字可搜尋",
    "rights_correction_present": "授權與更正資訊",
}
group_surface_colors = {
    "technical": "#ead8c7",
    "crawler": "#f2df9d",
    "semantic": "#c8e2d9",
    "citation": "#d7e2f1",
}
group_text_colors = {
    "technical": "#5b2e18",
    "crawler": "#684b00",
    "semantic": "#194f3e",
    "citation": "#254d73",
}
group_for_field = {field: group for group, fields in GROUPS.items() for field in fields}
paper_labels = [str(row["paper_id"]).replace("SHWRP-2026-", "") for row in rows]
affected_papers = {"SHWRP-2026-003", "SHWRP-2026-006", "SHWRP-2026-007", "SHWRP-2026-008", "SHWRP-2026-009"}

fig, ax = plt.subplots(figsize=(10.8, 12.4))
for y, field in enumerate(FIELDS):
    group = group_for_field[field]
    for x, row in enumerate(rows):
        passed = int(row[field]) == 1
        facecolor = group_surface_colors[group] if passed else "#f4d6d2"
        ax.add_patch(Rectangle((x - 0.5, y - 0.5), 1, 1, facecolor=facecolor, edgecolor="white", linewidth=1.2))
        ax.text(
            x,
            y,
            "✓" if passed else "—",
            ha="center",
            va="center",
            fontsize=10,
            color=group_text_colors[group] if passed else "#8a2f26",
            fontproperties=font,
        )
        if field == "jsonld_dates" and str(row["paper_id"]) in affected_papers:
            ax.add_patch(Rectangle((x - 0.45, y - 0.45), 0.9, 0.9, fill=False, edgecolor="#d46a1f", linewidth=2.4))

group_ends = np.cumsum([len(fields) for fields in GROUPS.values()])
for boundary in group_ends[:-1]:
    ax.axhline(boundary - 0.5, color="#6e6258", linewidth=1.6)

ax.set_xlim(-0.5, len(rows) - 0.5)
ax.set_ylim(len(FIELDS) - 0.5, -0.5)
ax.set_xticks(range(len(rows)), paper_labels, fontsize=8)
ax.xaxis.tick_top()
ax.tick_params(axis="x", length=0, pad=8)
ax.set_xlabel("工作論文編號（SHWRP-2026-）", fontproperties=font, labelpad=12)
ax.xaxis.set_label_position("top")
ax.set_yticks(range(len(FIELDS)), [indicator_labels[field] for field in FIELDS], fontsize=8)
ax.tick_params(axis="y", length=0, pad=7)
ax.set_title(
    "圖2　逐項技術準備度驗證矩陣（目前狀態：260/260 全部通過）",
    fontproperties=font,
    fontsize=14,
    pad=48,
)
legend_handles = [
    Patch(facecolor=group_surface_colors[group], edgecolor="none", label=group_labels[group])
    for group in GROUPS
] + [
    Patch(facecolor="none", edgecolor="#d46a1f", linewidth=2, label="7/15 修正後通過")
]
legend = ax.legend(
    handles=legend_handles,
    loc="lower center",
    bbox_to_anchor=(0.5, -0.09),
    ncol=5,
    frameon=False,
    prop=font,
)
for text in legend.get_texts():
    text.set_fontproperties(font)
for item in [ax.title, ax.xaxis.label, *ax.get_xticklabels(), *ax.get_yticklabels()]:
    item.set_fontproperties(font)
for spine in ax.spines.values():
    spine.set_visible(False)
fig.text(
    0.5,
    0.025,
    "✓＝目前觀察到；橘框＝初次稽核未通過、補齊 datePublished 後已通過。",
    ha="center",
    fontsize=8.5,
    color="#5f554c",
    fontproperties=font,
)
fig.subplots_adjust(left=0.29, right=0.98, top=0.84, bottom=0.14)
fig.savefig(FIGURES / "figure-2-indicator-matrix.png", dpi=180, facecolor="white")
plt.close(fig)

print(json.dumps(results, ensure_ascii=False, indent=2))