#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
statistics.py  –  Genealogy statistics
Reads four Gramps HTML reports and produces statistics.json

Usage:
    python statistics.py

Expects these files in the same folder (or adjust PATHS below):
    birthdays.html          ← birthday and anniversary report
    complete.html           ← complete individual report
    records.html            ← records report
    tag.html                ← tag report (used for precise birth/death dates)

Output:
    statistics.json
"""

import json
import re
import sys
from collections import defaultdict
from datetime import date
from pathlib import Path

from bs4 import BeautifulSoup

# ── Paths ───────────────────────────────────────────────────────────────────
BASE           = Path(__file__).parent
FILE_BIRTHDAYS = BASE / "birthdays.html"
FILE_COMPLETE  = BASE / "complete.html"
FILE_RECORDS   = BASE / "records.html"
FILE_TAG       = BASE / "tag.html"
FILE_JSON      = BASE / "statistics.json"

# ── Helper functions ────────────────────────────────────────────────────────

def load(path: Path) -> BeautifulSoup:
    return BeautifulSoup(path.read_text(encoding="utf-8"), "html.parser")

def strip_id(name: str) -> str:
    """Remove Gramps ID such as [I0007] from a name."""
    return re.sub(r"\s*\[I\d+\]", "", name).strip()

# NOTE: Month names must remain in English to match Gramps output
MONTH_NAMES = {
    "january": 1, "february": 2, "march": 3, "april": 4,
    "may": 5, "june": 6, "july": 7, "august": 8,
    "september": 9, "october": 10, "november": 11, "december": 12,
}

def parse_date(text: str):
    """
    Parse a date string into a date object.
    Understands: "1 May 1921", "May 1921", "1921", "1921-05-01", "about 1870".
    Returns date or None.
    """
    if not text:
        return None
    text = text.strip().lower()
    text = re.sub(r"^(about|ca\.?|before|after)\s*", "", text)

    # ISO format: YYYY-MM-DD
    m = re.match(r"(\d{4})-(\d{2})-(\d{2})$", text)
    if m:
        try:
            return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
        except ValueError:
            pass

    # DD Month YYYY
    m = re.match(r"(\d{1,2})\s+(\w+)\s+(\d{4})", text)
    if m:
        day, mon, year = int(m.group(1)), m.group(2), int(m.group(3))
        mn = MONTH_NAMES.get(mon)
        if mn:
            try:
                return date(year, mn, day)
            except ValueError:
                pass

    # Month YYYY
    m = re.match(r"(\w+)\s+(\d{4})", text)
    if m:
        mon, year = m.group(1), int(m.group(2))
        mn = MONTH_NAMES.get(mon)
        if mn:
            return date(year, mn, 15)

    # YYYY only
    m = re.match(r"(\d{4})$", text)
    if m:
        return date(int(m.group(1)), 7, 1)

    return None

def age_in_years(born: date, at: date) -> float:
    return (at - born).days / 365.25

# ══════════════════════════════════════════════════════════════════════════
# 1. TODAY IN HISTORY  –  birthdays.html
# ══════════════════════════════════════════════════════════════════════════

# NOTE: Month names must remain in English to match Gramps output
MONTHS = {
    "January": 1, "February": 2, "March": 3, "April": 4,
    "May": 5, "June": 6, "July": 7, "August": 8,
    "September": 9, "October": 10, "November": 11, "December": 12,
}

def parse_birthdays(soup: BeautifulSoup) -> list:
    entries = []
    year_now = date.today().year
    month = day = None

    for p in soup.find_all("p"):
        cls  = p.get("class", [""])[0]
        text = p.get_text(strip=True)
        if not text:
            continue

        if cls == "BIR-Monthstyle":
            month = MONTHS.get(text)
        elif cls == "BIR-Daystyle":
            try:
                day = int(text)
            except ValueError:
                pass
        elif cls == "BIR-Datastyle" and month and day:
            if text.startswith("*"):
                etype = "birth"
            elif text.startswith("⚭"):
                etype = "marriage"
            elif text.startswith("✝"):
                etype = "death"
            else:
                continue

            m = re.search(r"\((\d{4})\)", text)
            if not m:
                continue
            year = int(m.group(1))

            clean = re.sub(r"^[*⚭✝]\s*", "", text)
            clean = re.sub(r"\(\d{4}\)\s*", "", clean).strip()
            clean = re.sub(r",\s*\d+\s*$", "", clean).strip()

            entries.append({
                "month":      month,
                "day":        day,
                "type":       etype,
                "year":       year,
                "text":       clean,
                "years_ago":  year_now - year,
            })

    return entries

# ══════════════════════════════════════════════════════════════════════════
# 2. TAG  –  tag.html
#    Used for precise birth/death dates for age calculations
# ══════════════════════════════════════════════════════════════════════════

def parse_tag(soup: BeautifulSoup) -> tuple:
    """
    Returns (persons, filter_name).
    persons:     dict: { gramps_id: { "name": str, "born": date|None, "died": date|None } }
    filter_name: str extracted from title e.g. "ProfileTag" from
                 "Tag Report for ProfileTag Items"
    Parses the simple table: Id | Name | Birth | Death
    """
    # Extract filter name from title: "Tag Report for X Items" → X
    filter_name = ""
    title_tag = soup.find("title")
    if title_tag:
        m = re.search(r"Tag Report for (.+?) Items", title_tag.get_text())
        if m:
            filter_name = m.group(1).strip()

    persons = {}
    rows = soup.find_all("tr")

    for row in rows:
        cells = row.find_all("td")
        if len(cells) != 4:
            continue
        texts = [c.get_text(strip=True) for c in cells]
        pid, name, born_txt, died_txt = texts

        # Skip header row
        if pid in ("Id", "") or name in ("Navn", "Name", ""):
            continue
        #   becomes "\xa0" or empty
        born_txt = born_txt.replace("\xa0", "").strip()
        died_txt = died_txt.replace("\xa0", "").strip()

        persons[pid] = {
            "name": name,
            "born": parse_date(born_txt) if born_txt else None,
            "died": parse_date(died_txt) if died_txt else None,
        }

    return persons, filter_name

# ══════════════════════════════════════════════════════════════════════════
# 3. COMPLETE  –  complete.html
#    Used for: gender, number of families, children count, names
# ══════════════════════════════════════════════════════════════════════════

def parse_complete(soup: BeautifulSoup):
    """
    Returns (persons, families).
    persons:  list of { name, gramps_id, gender }
    families: list of { person1, person2, children: [names, foster children excluded] }
    """
    persons  = []
    families = []

    all_p = soup.find_all("p")
    i = 0
    n = len(all_p)

    while i < n:
        p   = all_p[i]
        cls = p.get("class", [""])[0]

        if cls == "IDS-Title" and "Complete Individual Report" in p.get_text():
            # Next IDS-Title is the name
            if i + 1 < n and all_p[i + 1].get("class", [""])[0] == "IDS-Title":
                name_raw  = all_p[i + 1].get_text(strip=True)
                name      = strip_id(name_raw)
                mid       = re.search(r"\[(I\d+)\]", name_raw)
                gramps_id = mid.group(1) if mid else None
            else:
                i += 1
                continue

            person = {"name": name, "gramps_id": gramps_id, "gender": None}

            # Scan forward for gender
            j = i + 2
            while j < n:
                pj   = all_p[j]
                txt  = pj.get_text(strip=True)
                cls2 = pj.get("class", [""])[0]
                if cls2 == "IDS-Title" and "Complete Individual Report" in txt:
                    break
                if txt in ("Male", "Female", "Unknown"):
                    person["gender"] = txt
                    break
                j += 1

            # Scan families: only children under "Familier" section, not foster parents
            j            = i + 2
            cur_family   = None
            in_families  = False   # are we inside a "Familier" section?
            in_listcell  = False   # are we inside a children list cell?

            while j < n:
                pj   = all_p[j]
                txt  = pj.get_text(strip=True)
                cls2 = pj.get("class", [""])[0]

                if cls2 == "IDS-Title" and "Complete Individual Report" in txt:
                    break

                # Section change
                if cls2 == "IDS-SectionTitle":
                    in_families = txt in ("Families", "Family")
                    in_listcell = False

                # New spouse (only relevant inside Familier section)
                if cls2 == "IDS-Spouse" and txt and in_families:
                    spouse      = strip_id(txt)
                    cur_family  = {
                        "person1":  name,
                        "person2":  spouse,
                        "children": [],
                    }
                    families.append(cur_family)
                    in_listcell = False

                # Children list cell starts
                if pj.parent and pj.parent.name == "td":
                    td = pj.parent
                    if "IDS-ListCell" in td.get("class", []):
                        in_listcell = True

                # Children in list cell – only under Familier, not foster parents
                if (in_listcell and in_families and cur_family
                        and cls2 == "IDS-Normal"
                        and re.search(r"\[I\d+\]", txt)):
                    child = strip_id(txt)
                    if child and child not in (name, cur_family["person2"]):
                        cur_family["children"].append(child)

                j += 1

            persons.append(person)
            i = j
        else:
            i += 1

    return persons, families


def calculate_counts(persons: list, soup_complete: BeautifulSoup) -> dict:
    men     = sum(1 for p in persons if p["gender"] == "Male")
    women   = sum(1 for p in persons if p["gender"] == "Female")
    unknown = sum(1 for p in persons if p["gender"] not in ("Male", "Female"))
    # born is enriched from tag before this call — count those without date
    no_birth = sum(1 for p in persons if not p.get("born"))

    # Number of families = number of "Familier" section headings (exact match)
    num_families = len(soup_complete.find_all(
        "p", class_="IDS-SectionTitle",
        string=re.compile(r"^Families$")
    ))

    return {
        "total":           len(persons),
        "men":             men,
        "women":           women,
        "unknown_gender":  unknown,
        "no_birth_date":   no_birth,
        "families":        num_families,
    }


def calculate_names(persons: list) -> dict:
    first_names: dict = defaultdict(int)
    last_names:  dict = defaultdict(int)

    for p in persons:
        parts = p["name"].split()
        if not parts:
            continue
        fn = parts[0]
        if fn.lower() not in ("pige", "dreng", "ukendt", "male", "female", "unknown"):
            first_names[fn] += 1
        if len(parts) > 1:
            ln = parts[-1]
            if ln.lower() not in ("pige", "dreng", "ukendt", "male", "female", "unknown"):
                last_names[ln] += 1

    total = len(persons)

    def top10(d: dict) -> list:
        sorted_d = sorted(d.items(), key=lambda x: -x[1])[:10]
        return [{"name": k, "count": v, "pct": round(v / total * 100, 1)}
                for k, v in sorted_d]

    return {"first_names": top10(first_names), "last_names": top10(last_names)}


def calculate_age_distribution(tag: dict) -> dict:
    """Calculate lifespan statistics from tag.html (precise dates)."""
    lifespans = []
    for pid, p in tag.items():
        if p["born"] and p["died"]:
            a = age_in_years(p["born"], p["died"])
            if 0 <= a <= 120:
                lifespans.append(round(a, 1))

    def stats(lst):
        if not lst:
            return None
        return {
            "minimum":  round(min(lst), 1),
            "average":  round(sum(lst) / len(lst), 1),
            "maximum":  round(max(lst), 1),
            "count":    len(lst),
        }

    return {"lifespan": stats(lifespans)}

# ══════════════════════════════════════════════════════════════════════════
# 4. RECORDS  –  records.html
# ══════════════════════════════════════════════════════════════════════════

def parse_records(soup: BeautifulSoup) -> dict:
    sections: dict = {}
    current = None
    for p in soup.find_all("p"):
        cls = p.get("class", [""])[0]
        txt = p.get_text(strip=True)
        if not txt:
            continue
        if cls == "REC-Heading":
            current = txt
            sections[current] = []
        elif cls == "REC-Normal" and current:
            sections[current].append(txt)
    return sections

def map_records(sec: dict) -> dict:
    def get(name: str, max_n: int = 5) -> list:
        return sec.get(name, [])[:max_n]

    return {
        "fader_flest_born":         get("Father with most children", 5),
        "moder_flest_born":         get("Mother with most children", 5),
        "fader_flest_borneborn":    get("Father with most grandchildren", 4),
        "moder_flest_borneborn":    get("Mother with most grandchildren", 4),
        "par_flest_born":           get("Couple with most children", 5),
        "yngste_nulevende":         get("Youngest living person", 3),
        "aeldste_nulevende":        get("Oldest living person", 3),
        "doed_laveste_alder":       get("Person died at youngest age", 3),
        "doed_hoejeste_alder":      get("Person died at oldest age", 3),
        "gift_laveste_alder":       get("Person married at youngest age", 3),
        "gift_hoejeste_alder":      get("Person married at oldest age", 3),
        "par_mindst_aldersforskel": get("Couple with smallest age difference", 3),
        "par_stoerst_aldersforskel":get("Couple with biggest age difference", 3),
    }

# ══════════════════════════════════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════════════════════════════════

def main():
    for f in (FILE_BIRTHDAYS, FILE_COMPLETE, FILE_RECORDS, FILE_TAG):
        if not f.exists():
            print(f"ERROR: File '{f}' not found.", file=sys.stderr)
            sys.exit(1)

    print("Reading birthdays.html …")
    today_data = parse_birthdays(load(FILE_BIRTHDAYS))

    print("Reading tag.html …")
    tag, filter_name = parse_tag(load(FILE_TAG))
    print(f"  → {len(tag)} persons with date information (filter: {filter_name})")

    print("Reading complete.html …")
    soup_complete = load(FILE_COMPLETE)
    persons, families = parse_complete(soup_complete)
    print(f"  → {len(persons)} persons found")

    # Enrich persons with dates from tag – match by name
    tag_by_name = {v["name"]: v for v in tag.values()}
    for p in persons:
        match = tag_by_name.get(p["name"])
        if match:
            p["born"] = match["born"]
            p["died"] = match["died"]
        else:
            p["born"] = p["died"] = None

    counts           = calculate_counts(persons, soup_complete)
    names            = calculate_names(persons)
    age_distribution = calculate_age_distribution(tag)

    print("Reading records.html …")
    rec = map_records(parse_records(load(FILE_RECORDS)))

    # Replace Danish placeholder name "Dødfødt" with English "Stillborn"
    for entry in rec.get("died_youngest", []):
        pass  # handled below via string replacement in map_records output
    for key in rec:
        rec[key] = [s.replace("Dødfødt", "Stillborn") for s in rec[key]]

    output = {
        "generated":   date.today().isoformat(),
        "filter_name": filter_name,
        "today":       today_data,
        "counts": {
            **counts,
            **{k: rec[k] for k in (
                "fader_flest_born", "moder_flest_born",
                "fader_flest_borneborn", "moder_flest_borneborn",
                "par_flest_born",
            )},
        },
        "age": {
            **age_distribution,
            **{k: rec[k] for k in (
                "yngste_nulevende", "aeldste_nulevende",
                "doed_laveste_alder", "doed_hoejeste_alder",
                "gift_laveste_alder", "gift_hoejeste_alder",
                "par_mindst_aldersforskel", "par_stoerst_aldersforskel",
            )},
        },
        "names": names,
    }

    FILE_JSON.write_text(
        json.dumps(output, ensure_ascii=False, indent=2, default=str),
        encoding="utf-8",
    )
    lev = age_distribution["lifespan"]
    print(f"\nDone! Result saved to: {FILE_JSON}")
    print(f"  today       : {len(today_data)} entries")
    print(f"  counts      : {counts['total']} persons, {counts['families']} families")
    print(f"  age         : lifespan for {lev['count']} persons (avg {lev['average']} years)" if lev else "  age: no lifespan data")
    print(f"  names       : top-{len(names['first_names'])} first names, top-{len(names['last_names'])} last names")

if __name__ == "__main__":
    main()