from __future__ import annotations

import csv
import hashlib
import json
import re
import sys
from collections import defaultdict
from datetime import date, datetime, timedelta
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path
from zoneinfo import ZoneInfo

import jsonschema
import yaml
from pypdf import PdfReader


ROOT = Path(__file__).resolve().parent
REPORT = ROOT / "validation-report.json"
TAIPEI = ZoneInfo("Asia/Taipei")

FONT_SHA256 = "40452f54ea36b5c1f1ec8bd92dc18d5cd19fafb73b8abf9c48500a00d09ec762"
FONT_LICENSE_SHA256 = "1c05c68c34f9708415aada51f17e1b0092d2cea709bf4a94cd38114f9e73d7d9"
CFF_SCHEMA_SHA256 = "0b8d22140da702d766df318dcff3a91af2f39521298dcf36d76315fd99cc169b"

REQUIRED = [
    "README.md", "paper.md", "paper.pdf", "paper-content.json", "paper-metadata.json",
    "CITATION.cff", "LICENSE.txt", "RIGHTS-NOTICE.txt", "source-registry.csv",
    "source-log.csv", "source-evidence.json", "codebook.csv", "cleaning-rules.md",
    "requirements.txt", "fetch_sources.py", "prepare_data.py", "analysis.py",
    "build_artifacts.py", "validate.py", "verify_reproducibility.py", "package_release.py",
    "validate_package.py", "pdf-visual-qa.json", "reproducibility-check.json",
    "assets/fonts/NotoSansTC-Regular.ttf", "assets/fonts/OFL.txt",
    "assets/schemas/cff-1.2.0-schema.json",
    "data/energy-monthly-generation-2016-2025.csv",
    "data/taipower-net-generation-purchases-2016-2025.csv", "data/factor-observations.csv",
    "outputs/analysis-results.json", "outputs/annual-generation-mix.csv",
    "outputs/monthly-generation-shares-2025.csv", "outputs/taipower-scope-cross-check.csv",
    "outputs/electricity-factor-comparison.csv",
    "outputs/figures/figure-1-generation-mix-2016-2025.png",
    "outputs/figures/figure-2-mix-change-2024-2025.png",
    "outputs/figures/figure-3-monthly-generation-shares-2025.png",
    "outputs/figures/figure-4-factor-scope-comparison-2025.png",
]

FORBIDDEN_FIELD_NAMES = {
    "full_html", "raw_html", "page_text", "full_description", "review_text", "screenshot",
    "image_binary", "customer_name", "phone", "email", "address", "precise_latitude",
    "precise_longitude", "password", "secret", "token", "cookie", "api_key",
}

PRIVATE_KEY = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")
SECRET_ASSIGNMENT = re.compile(r"(?im)^\s*(?:password|secret|token|cookie|api[_-]?key)\s*[:=]\s*[^\s#][^\r\n]{5,}")
SENSITIVE_PART = re.compile(r"(?:^|[._-])(?:\.env|password|secret|token|cookie|credential|private[._-]?key)(?:$|[._-])", re.I)

MONTHLY_FIELDS = {
    "total": "total_million_kwh",
    "pumped_storage": "pumped_storage_million_kwh",
    "thermal_total": "thermal_total_million_kwh",
    "coal": "coal_million_kwh",
    "oil": "oil_million_kwh",
    "gas": "gas_million_kwh",
    "nuclear": "nuclear_million_kwh",
    "renewable_total": "renewable_total_million_kwh",
    "conventional_hydro": "conventional_hydro_million_kwh",
    "geothermal": "geothermal_million_kwh",
    "solar": "solar_million_kwh",
    "wind": "wind_million_kwh",
    "biomass": "biomass_million_kwh",
    "waste": "waste_million_kwh",
}

RENEWABLE_TAIPOWER_TYPES = {
    "geothermal", "conventional_hydro", "contracted_hydro", "hydro", "wind",
    "solar", "biomass", "waste", "waste_biogas",
}


def now_taipei() -> datetime:
    return datetime.now(TAIPEI).replace(microsecond=0)


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def read_rows(path: Path) -> list[dict[str, str]]:
    with path.open("r", encoding="utf-8", newline="") as stream:
        return list(csv.DictReader(stream))


def rounded(value: Decimal, digits: int = 4) -> Decimal:
    return value.quantize(Decimal(1).scaleb(-digits), rounding=ROUND_HALF_UP)


def parse_event_time(value: object, label: str, errors: list[str]) -> datetime | None:
    if not isinstance(value, str):
        errors.append(f"{label} is missing or is not an ISO timestamp")
        return None
    try:
        parsed = datetime.fromisoformat(value)
    except ValueError:
        errors.append(f"{label} is not a valid ISO timestamp: {value!r}")
        return None
    if parsed.tzinfo is None:
        errors.append(f"{label} lacks a timezone offset")
        return None
    if parsed > now_taipei() + timedelta(minutes=5):
        errors.append(f"{label} is in the future: {value}")
    return parsed


def collect_json_keys(value: object, keys: set[str]) -> None:
    if isinstance(value, dict):
        for key, nested in value.items():
            keys.add(str(key).lower())
            collect_json_keys(nested, keys)
    elif isinstance(value, list):
        for item in value:
            collect_json_keys(item, keys)


def normalize_yaml_types(value: object) -> object:
    if isinstance(value, (date, datetime)):
        return value.isoformat()
    if isinstance(value, dict):
        return {key: normalize_yaml_types(nested) for key, nested in value.items()}
    if isinstance(value, list):
        return [normalize_yaml_types(item) for item in value]
    return value


def main() -> None:
    errors: list[str] = []
    warnings: list[str] = []
    checks: dict[str, object] = {}

    missing = [relative for relative in REQUIRED if not (ROOT / relative).is_file() or (ROOT / relative).stat().st_size == 0]
    if missing:
        errors.extend(f"Missing or empty required file: {relative}" for relative in missing)
    checks["requiredFiles"] = {"expected": len(REQUIRED), "missing": missing}

    try:
        meta = json.loads((ROOT / "paper-metadata.json").read_text(encoding="utf-8"))
        content = json.loads((ROOT / "paper-content.json").read_text(encoding="utf-8"))
        results = json.loads((ROOT / "outputs/analysis-results.json").read_text(encoding="utf-8"))
        paper = (ROOT / "paper.md").read_text(encoding="utf-8")
    except (OSError, json.JSONDecodeError) as error:
        errors.append(f"Core publication metadata are unreadable: {error}")
        meta, content, results, paper = {}, {}, {}, ""

    expected_identity = {
        "reportNumber": "SHWRP-2026-030",
        "version": "1.1",
        "datePublished": "2026-08-03",
        "dateModified": "2026-08-03",
        "slug": "electricity-carbon-accounting-2025",
    }
    for key, expected in expected_identity.items():
        if meta.get(key) != expected:
            errors.append(f"Metadata mismatch for {key}: {meta.get(key)!r}")
    if meta.get("url") != "https://bento24235111.com/research-journal/electricity-carbon-accounting-2025":
        errors.append("Canonical publication URL is inconsistent")
    if meta.get("doi") != "10.5281/zenodo.21769500" or meta.get("zenodoUrl") != "https://zenodo.org/records/21769500":
        errors.append("DOI or Zenodo URL does not match the reserved version record")
    if meta.get("publicContactConsent") is not True:
        errors.append("The public author email requires explicit publicContactConsent=true")
    for key in ("reportNumber", "version"):
        if content.get(key) != meta.get(key):
            errors.append(f"paper-content mismatch for {key}")
    if content.get("date") != meta.get("datePublished") or content.get("revisionDate") != meta.get("dateModified"):
        errors.append("paper-content publication dates do not match metadata")
    if content.get("publicContactConsent") is not True:
        errors.append("paper-content did not preserve public contact consent")
    if meta.get("title") not in paper or meta.get("titleEnglish") not in paper:
        errors.append("Markdown title does not match metadata")
    for marker in (
        "周秉漢", "CHOU, BING-HAN", "0009-0000-7018-5096", "10.5281/zenodo.21769500",
        "未經外部同儕審查", "5.3574", "0.0620", "燃油減少2.9933",
    ):
        if marker not in paper:
            errors.append(f"Paper is missing required marker: {marker}")
    checks["metadataConsistency"] = not any("Metadata" in item or "paper-content" in item or "Markdown" in item for item in errors)

    # Validate CFF against the locally fixed official CFF 1.2.0 schema.
    try:
        schema_path = ROOT / "assets/schemas/cff-1.2.0-schema.json"
        if sha256(schema_path) != CFF_SCHEMA_SHA256:
            errors.append("Packaged CFF 1.2.0 schema hash mismatch")
        cff = normalize_yaml_types(yaml.safe_load((ROOT / "CITATION.cff").read_text(encoding="utf-8")))
        schema = json.loads(schema_path.read_text(encoding="utf-8"))
        jsonschema.validate(cff, schema)
        checks["cffSchema"] = "CFF 1.2.0 PASSED"
    except (OSError, yaml.YAMLError, json.JSONDecodeError, jsonschema.ValidationError, jsonschema.SchemaError) as error:
        errors.append(f"CFF 1.2.0 schema validation failed: {error}")
        checks["cffSchema"] = "FAILED"

    for relative, expected_hash in (
        ("assets/fonts/NotoSansTC-Regular.ttf", FONT_SHA256),
        ("assets/fonts/OFL.txt", FONT_LICENSE_SHA256),
    ):
        if (ROOT / relative).is_file() and sha256(ROOT / relative) != expected_hash:
            errors.append(f"Packaged asset hash mismatch: {relative}")

    # Independently recompute all important outputs from the fixed monthly source subset.
    monthly = read_rows(ROOT / "data/energy-monthly-generation-2016-2025.csv")
    expected_months = [f"{year}{month:02d}" for year in range(2016, 2026) for month in range(1, 13)]
    observed_months = [row.get("month", "") for row in monthly]
    if observed_months != expected_months or len(set(observed_months)) != 120:
        errors.append("Monthly data are not exactly 120 unique ordered observations from 2016-01 through 2025-12")

    grouped: dict[int, dict[str, Decimal]] = defaultdict(lambda: defaultdict(Decimal))
    maximum_gap = Decimal(0)
    for row in monthly:
        try:
            values = {name: Decimal(row[column]) for name, column in MONTHLY_FIELDS.items()}
            year = int(row["month"][:4])
        except (KeyError, ValueError, ArithmeticError) as error:
            errors.append(f"Invalid monthly numeric row: {error}")
            continue
        primary_gap = abs(values["total"] - (values["pumped_storage"] + values["thermal_total"] + values["nuclear"] + values["renewable_total"]))
        thermal_gap = abs(values["thermal_total"] - (values["coal"] + values["oil"] + values["gas"]))
        renewable_gap = abs(values["renewable_total"] - sum((values[name] for name in ("conventional_hydro", "geothermal", "solar", "wind", "biomass", "waste")), Decimal(0)))
        maximum_gap = max(maximum_gap, primary_gap, thermal_gap, renewable_gap)
        for name, value in values.items():
            grouped[year][name] += value
    if maximum_gap != 0:
        errors.append(f"Monthly component identity is not exact; maximum gap={maximum_gap}")

    def share(year: int, name: str) -> Decimal:
        return grouped[year][name] / grouped[year]["total"] * Decimal(100)

    annual_rows = read_rows(ROOT / "outputs/annual-generation-mix.csv")
    if [row.get("year") for row in annual_rows] != [str(year) for year in range(2016, 2026)]:
        errors.append("Annual output does not contain exactly 10 ordered years")
    else:
        for row in annual_rows:
            year = int(row["year"])
            if abs(Decimal(row["total_generation_million_kwh"]) - grouped[year]["total"]) > Decimal("0.0000005"):
                errors.append(f"Annual total mismatch for {year}")
            for name in ("coal", "gas", "oil", "nuclear", "renewable_total", "pumped_storage", "conventional_hydro", "solar", "wind"):
                if abs(Decimal(row[f"{name}_share_pct"]) - share(year, name)) > Decimal("0.00005"):
                    errors.append(f"Annual share mismatch for {year}/{name}")

    expected_change_2024 = {name: rounded(share(2025, name) - share(2024, name)) for name in ("coal", "gas", "oil", "nuclear", "renewable_total", "pumped_storage")}
    expected_change_2016 = {name: rounded(share(2025, name) - share(2016, name)) for name in ("coal", "gas", "oil", "nuclear", "renewable_total", "pumped_storage")}
    try:
        result_changes_2024 = results["year2025"]["changeFrom2024PercentagePoints"]
        result_changes_2016 = results["year2025"]["changeFrom2016PercentagePoints"]
        for name, expected in expected_change_2024.items():
            if Decimal(str(result_changes_2024.get(name))) != expected:
                errors.append(f"2024-to-2025 percentage-point mismatch for {name}: expected {expected}")
        for name, expected in expected_change_2016.items():
            if Decimal(str(result_changes_2016.get(name))) != expected:
                errors.append(f"2016-to-2025 percentage-point mismatch for {name}: expected {expected}")
        if Decimal(str(results["year2025"]["totalGenerationMillionKwh"])) != rounded(grouped[2025]["total"], 6):
            errors.append("2025 total generation mismatch in analysis-results.json")
        if results["sampleCounts"].get("monthlyGenerationRows") != 120:
            errors.append("Analysis monthly sample count is inconsistent")
        if results["sampleCounts"].get("taipowerInputRows") != 212 or results["sampleCounts"].get("taipowerCrossCheckAnnualRows") != 10:
            errors.append("Taipower input/output row counts are mislabeled or inconsistent")
    except (KeyError, TypeError, ArithmeticError) as error:
        errors.append(f"Analysis results structure is invalid: {error}")

    taipower_rows = read_rows(ROOT / "data/taipower-net-generation-purchases-2016-2025.csv")
    tp_grouped: dict[int, dict[str, Decimal]] = defaultdict(lambda: defaultdict(Decimal))
    for row in taipower_rows:
        raw_value = row.get("net_generation_or_purchase_kwh", "")
        if not raw_value:
            continue
        year = int(row["year"])
        value = Decimal(raw_value)
        energy = row["energy_type"]
        tp_grouped[year]["total"] += value
        tp_grouped[year][energy] += value
        if energy in RENEWABLE_TAIPOWER_TYPES:
            tp_grouped[year]["renewable_group"] += value

    direction_agreement: dict[str, bool] = {}
    for name in ("coal", "gas", "nuclear", "renewable_total"):
        tp_name = "renewable_group" if name == "renewable_total" else name
        tp_change = tp_grouped[2025][tp_name] / tp_grouped[2025]["total"] - tp_grouped[2024][tp_name] / tp_grouped[2024]["total"]
        nationwide_change = share(2025, name) - share(2024, name)
        direction_agreement[name] = tp_change == nationwide_change == 0 or tp_change * nationwide_change > 0
    if not all(direction_agreement.values()):
        errors.append(f"Independent cross-scope direction check failed: {direction_agreement}")
    if results.get("dataQuality", {}).get("directionAgreement2024to2025") != direction_agreement:
        errors.append("Recorded direction agreement differs from independent recomputation")

    factors = read_rows(ROOT / "data/factor-observations.csv")
    expected_factors = {
        ("2016", "public_electricity_sales"): ("0.530", "S04"),
        ("2024", "public_electricity_sales"): ("0.474", "S04"),
        ("2025", "national_electricity"): ("0.456", "S04"),
        ("2025", "public_electricity_sales"): ("0.467", "S03"),
        ("2025", "industrial_inventory"): ("0.466", "S03"),
        ("2025", "residential_inventory"): ("0.471", "S03"),
    }
    observed_factors = {(row.get("year"), row.get("factor_scope")): (row.get("value_kg_co2e_per_kwh"), row.get("source_id")) for row in factors}
    if observed_factors != expected_factors:
        errors.append("Factor observations or their source IDs differ from the fixed evidence table")
    checks["independentResearchRecomputation"] = {
        "months": len(monthly), "years": len(grouped), "maximumComponentGap": str(maximum_gap),
        "change2024to2025": {key: str(value) for key, value in expected_change_2024.items()},
        "directionAgreement": direction_agreement,
    }

    sources = read_rows(ROOT / "source-registry.csv")
    source_log = read_rows(ROOT / "source-log.csv")
    source_ids = {row.get("source_id") for row in sources}
    if source_ids != {"S01", "S02", "S03", "S04", "S05", "S06"}:
        errors.append(f"Source registry IDs are incomplete: {sorted(source_ids)}")
    if any(row.get("first_party_or_official") != "是" for row in sources):
        errors.append("Every cited evidence source must be marked first-party or official")
    log_by_id = {row.get("source_id"): row for row in source_log}
    if log_by_id.get("S04", {}).get("http_status") != "403" or log_by_id.get("S04", {}).get("sha256"):
        errors.append("S04 must transparently record the automated 403 without a content hash")
    if log_by_id.get("S05", {}).get("sha256") != "ea45ae45f32c521ed11f92da4296811c82cb061eda4097907e088d0c716a07ad":
        errors.append("S05 retrieval integrity record is missing or changed")
    try:
        evidence = json.loads((ROOT / "source-evidence.json").read_text(encoding="utf-8"))
        evidence_sources = evidence.get("sources", [])
        evidence_ids = {item.get("sourceId") for item in evidence_sources if isinstance(item, dict)}
        if evidence.get("fullPageContentStored") is not False or evidence_ids != {"S04", "S05"}:
            errors.append("source-evidence.json is incomplete")
    except (OSError, json.JSONDecodeError) as error:
        errors.append(f"source-evidence.json is invalid: {error}")

    # PDF semantic checks plus a PDF-hash-bound visual review record.
    try:
        pdf_path = ROOT / "paper.pdf"
        reader = PdfReader(str(pdf_path))
        if reader.is_encrypted:
            errors.append("PDF must not be encrypted")
        page_texts = [(page.extract_text() or "").strip() for page in reader.pages]
        compact_text = re.sub(r"\s+", "", "\n".join(page_texts))
        if len(reader.pages) < 8:
            errors.append(f"PDF has unexpectedly few pages: {len(reader.pages)}")
        blank_pages = [index + 1 for index, text in enumerate(page_texts) if len(text) < 30]
        if blank_pages:
            errors.append(f"PDF has blank or nearly blank text pages: {blank_pages}")
        for marker in (meta.get("title", ""), "SHWRP-2026-030", "5.3574", "0.0620", "燃油減少2.9933", "未經外部同儕審查"):
            if re.sub(r"\s+", "", marker) not in compact_text:
                errors.append(f"Searchable PDF text is missing: {marker}")
        pdf_meta = reader.metadata or {}
        if str(pdf_meta.get("/Title", "")) != meta.get("title") or "周秉漢" not in str(pdf_meta.get("/Author", "")):
            errors.append("PDF document metadata are inconsistent")
        pdf_hash = sha256(pdf_path)
        qa = json.loads((ROOT / "pdf-visual-qa.json").read_text(encoding="utf-8"))
        parse_event_time(qa.get("reviewedAt"), "PDF visual QA reviewedAt", errors)
        if qa.get("status") != "PASSED" or qa.get("pagesReviewed") != len(reader.pages) or qa.get("pdfSha256") != pdf_hash:
            errors.append("PDF visual QA is missing, incomplete, or bound to a different PDF hash")
        checks["pdf"] = {"pages": len(reader.pages), "bytes": pdf_path.stat().st_size, "sha256": pdf_hash, "visualQaBound": True}
    except (OSError, json.JSONDecodeError, Exception) as error:
        if not isinstance(error, (SystemExit, KeyboardInterrupt)):
            errors.append(f"PDF validation failed: {error}")

    # Reproducibility evidence must be generated by the executable rerun and predate validation.
    try:
        reproducibility = json.loads((ROOT / "reproducibility-check.json").read_text(encoding="utf-8"))
        parse_event_time(reproducibility.get("reproducedAt"), "Reproducibility reproducedAt", errors)
        files = reproducibility.get("files", [])
        if reproducibility.get("status") != "PASSED" or len(files) != 11 or not all(item.get("bitwiseMatch") is True for item in files):
            errors.append("Executable controlled-environment reproducibility check did not pass every derived file")
        checks["reproducibility"] = {"status": reproducibility.get("status"), "filesCompared": len(files)}
    except (OSError, json.JSONDecodeError) as error:
        errors.append(f"Reproducibility record is invalid: {error}")

    # Scan only public-release candidates; archive/attestation files are handled after packaging.
    ignored_names = {
        "SHWRP-2026-030-v1.0.zip", "SHWRP-2026-030-v1.0.zip.sha256",
        "SHWRP-2026-030-v1.1.zip", "SHWRP-2026-030-v1.1.zip.sha256",
        "release-manifest.json", "package-validation-report.json",
    }
    public_files = [
        path for path in ROOT.rglob("*")
        if path.is_file() and path.name not in ignored_names and "__pycache__" not in path.parts and "private-source-cache" not in path.parts
    ]
    forbidden_paths = [
        path.relative_to(ROOT).as_posix() for path in public_files
        if any(SENSITIVE_PART.search(part) for part in path.relative_to(ROOT).parts)
    ]
    if forbidden_paths:
        errors.append(f"Sensitive-looking public path names: {forbidden_paths}")
    field_hits: list[str] = []
    credential_hits: list[str] = []
    consented_email_files = {"paper-metadata.json", "paper-content.json"} if meta.get("publicContactConsent") is True else set()
    for path in public_files:
        relative = path.relative_to(ROOT).as_posix()
        suffix = path.suffix.lower()
        if suffix == ".csv":
            with path.open("r", encoding="utf-8", newline="") as stream:
                headers = {item.lower() for item in next(csv.reader(stream), [])}
            hits = sorted(FORBIDDEN_FIELD_NAMES.intersection(headers))
            if hits:
                field_hits.append(f"{relative}: {hits}")
        elif suffix == ".json":
            try:
                value = json.loads(path.read_text(encoding="utf-8"))
            except json.JSONDecodeError:
                errors.append(f"Invalid JSON in public release candidate: {relative}")
                continue
            keys: set[str] = set()
            collect_json_keys(value, keys)
            # The vendored official schema necessarily names fields such as
            # `email` and `address`; those vocabulary definitions are not data.
            hits = [] if relative == "assets/schemas/cff-1.2.0-schema.json" else sorted(FORBIDDEN_FIELD_NAMES.intersection(keys))
            if path.name in consented_email_files and "email" in hits:
                hits.remove("email")
            if hits:
                field_hits.append(f"{relative}: {hits}")
        if suffix in {".md", ".txt", ".py", ".json", ".csv", ".cff", ".tsx"}:
            text = path.read_text(encoding="utf-8", errors="ignore")
            if PRIVATE_KEY.search(text) or SECRET_ASSIGNMENT.search(text):
                credential_hits.append(relative)
    if field_hits:
        errors.append(f"Forbidden structured fields found: {field_hits}")
    if credential_hits:
        errors.append(f"Credential-like values found: {credential_hits}")
    checks["sensitiveDataScan"] = {
        "filesScanned": len(public_files), "publicContactConsent": meta.get("publicContactConsent") is True,
        "forbiddenPaths": forbidden_paths, "structuredFieldHits": field_hits, "credentialPatternHits": credential_hits,
    }

    result = {
        "schemaVersion": 2,
        "reportNumber": "SHWRP-2026-030",
        "version": "1.1",
        "validatedAt": now_taipei().isoformat(),
        "status": "PASSED" if not errors else "FAILED",
        "scope": "Content, data, metadata, rights, PDF and controlled-environment reproduction checks before packaging; ZIP integrity is validated separately after packaging.",
        "checks": checks,
        "errors": errors,
        "warnings": warnings,
    }
    REPORT.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps({"status": result["status"], "errors": len(errors), "warnings": len(warnings), "report": str(REPORT)}, ensure_ascii=False))
    if errors:
        sys.exit(1)


if __name__ == "__main__":
    main()
