"""All numbers for the video: writes analysis/results.json, results.md and draft charts (charts/draft_*.png).

Every number that ends up in the script has to come from results.md.
"""
import gzip
import json
import random
import re
from collections import Counter
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

ROOT = Path(__file__).resolve().parent.parent
CLEAN = ROOT / "data" / "clean"
CHARTS = ROOT / "charts"
OUT = ROOT / "analysis"
FIRST, LAST_FULL = 2010, 2025  # 2026 is still running, shown but not used for fits

STOP = set("""a an the of in to and or is as at by for from on with be my i me i'm it its this that so but
was are who what how why no not just even though if when will can his her their your you we our one all
into than then out up has have had do does did get got""".split())


def gz_ratio(titles):
    text = "\n".join(titles).encode("utf-8")
    return len(gzip.compress(text, 9)) / len(text)


def template_score(titles, budget=1500, reps=300, seed=1, **_):
    """Mean gzip ratio (compressed / raw, lower = more repeated phrases) of random titles drawn until `budget`
    characters are reached. The same text length for every group, because gzip compresses longer text better anyway."""
    rng = random.Random(seed)
    titles = list(titles)
    if sum(map(len, titles)) < budget * 1.5:
        return None
    ratios = []
    for _ in range(reps):
        pool, picked, size = titles[:], [], 0
        rng.shuffle(pool)
        for t in pool:
            if size >= budget:
                break
            picked.append(t)
            size += len(t) + 1
        ratios.append(gz_ratio(picked))
    return round(float(np.mean(ratios)), 4)


def per_year(df, col):
    g = df.groupby("year")[col]
    return pd.DataFrame({"n": g.size(), "mean": g.mean().round(2), "median": g.median()})


def fit(years, values, threshold):
    years, values = np.asarray(years, float), np.asarray(values, float)
    lin = np.polyfit(years, values, 1)
    exp = np.polyfit(years, np.log(values), 1)
    out = {"linear_slope_per_year": round(lin[0], 3), "exp_growth_per_year_pct": round((np.exp(exp[0]) - 1) * 100, 2)}
    out["linear_cross_year"] = round((threshold - lin[1]) / lin[0], 1) if lin[0] > 0 else None
    out["exp_cross_year"] = round((np.log(threshold) - exp[1]) / exp[0], 1) if exp[0] > 0 else None
    r = np.corrcoef(years, values)[0, 1]
    out["r"] = round(r, 3)
    return out, lin, exp


def main():
    a = pd.read_csv(CLEAN / "anime.csv")
    main_ = a[a["main"] & a.year.between(2000, 2026)].copy()
    en = main_[main_.has_english]
    R = {"definitions": {"isekai": "AniList tag Isekai, rank >= 60", "dataset": "Japanese TV + ONA anime, not adult, first seasons only",
                         "title": "official English title on AniList (title.english); titles without one are counted separately"}}

    # --- sizes
    win = main_[main_.year.between(FIRST, 2026)]
    R["counts"] = {
        "all_main_2010_2026": int(len(win)), "isekai_2010_2026": int(win.isekai.sum()),
        "isekai_with_english": int((win.isekai & win.has_english).sum()),
        "control_with_english": int((~win.isekai & win.has_english).sum()),
        "fetched_total": int(len(a)),
    }
    share = win.groupby("year").isekai.mean().mul(100).round(1)
    R["isekai_share_pct_by_year"] = share.to_dict()

    # --- the curve
    isek = en[en.isekai]
    ctrl = en[~en.isekai]
    curve = per_year(isek, "words_en").join(per_year(ctrl, "words_en"), lsuffix="_isekai", rsuffix="_control")
    curve["chars_isekai"] = isek.groupby("year").chars_en.mean().round(1)
    curve["chars_control"] = ctrl.groupby("year").chars_en.mean().round(1)
    curve["native_isekai"] = main_[main_.isekai].groupby("year").chars_native.mean().round(1)
    curve["native_control"] = main_[~main_.isekai].groupby("year").chars_native.mean().round(1)
    curve["sentence_pct_isekai"] = isek.groupby("year").sentence_title.mean().mul(100).round(1)
    curve["sentence_pct_control"] = ctrl.groupby("year").sentence_title.mean().mul(100).round(1)
    R["curve_by_year"] = json.loads(curve.reset_index().to_json(orient="records"))

    def era(df, col, y0, y1):
        d = df[df.year.between(y0, y1)][col]
        return {"n": int(len(d)), "mean": round(float(d.mean()), 2) if len(d) else None, "median": float(d.median()) if len(d) else None}

    eras = [(2000, 2009), (2010, 2013), (2014, 2017), (2018, 2021), (2022, 2025), (2026, 2026)]
    R["eras_words"] = {f"{y0}-{y1}": {"isekai": era(isek, "words_en", y0, y1), "control": era(ctrl, "words_en", y0, y1)} for y0, y1 in eras}
    R["eras_native_chars"] = {f"{y0}-{y1}": {"isekai": era(main_[main_.isekai], "chars_native", y0, y1),
                                             "control": era(main_[~main_.isekai], "chars_native", y0, y1)} for y0, y1 in eras}

    # --- by source material (is it isekai, or is it light novels?)
    src = en[en.year.between(2018, 2025)].groupby(["source", "isekai"]).words_en.agg(["size", "mean"]).round(2)
    R["by_source_2018_2025"] = {f"{s}|{'isekai' if i else 'other'}": {"n": int(r["size"]), "mean_words": r["mean"]}
                                for (s, i), r in src.iterrows() if r["size"] >= 5}

    # --- record holders
    cols = ["year", "words_en", "chars_en", "title_used", "title_native", "format", "isekai_rank"]
    allisek = a[a.isekai & ~a.adult & a.has_english]
    R["longest_isekai_words"] = json.loads(allisek.sort_values(["words_en", "chars_en"], ascending=False)[cols].head(15).to_json(orient="records", force_ascii=False))
    R["longest_isekai_native"] = json.loads(a[a.isekai & ~a.adult].sort_values("chars_native", ascending=False)[cols].head(10).to_json(orient="records", force_ascii=False))
    R["longest_any_words"] = json.loads(a[~a.adult & a.has_english].sort_values(["words_en", "chars_en"], ascending=False)[cols + ["source"]].head(15).to_json(orient="records", force_ascii=False))
    R["shortest_isekai_recent"] = json.loads(isek[isek.year >= 2020].sort_values("words_en")[cols].head(10).to_json(orient="records", force_ascii=False))

    # --- compression and words
    comp = {}
    for y0, y1 in eras[1:5]:
        ti = isek[isek.year.between(y0, y1)].title_used.str.lower().tolist()
        tc = ctrl[ctrl.year.between(y0, y1)].title_used.str.lower().tolist()
        comp[f"{y0}-{y1}"] = {"isekai": template_score(ti), "control": template_score(tc), "n_isekai": len(ti), "n_control": len(tc)}
    R["gzip_ratio_30_titles"] = comp
    recent = en[en.year.between(2018, 2025)]
    R["gzip_ratio_2018_2025_same_length_text"] = {
        "isekai": template_score(recent[recent.isekai].title_used.str.lower(), budget=3000),
        "control_all": template_score(recent[~recent.isekai].title_used.str.lower(), budget=3000),
        "control_light_novel_not_isekai": template_score(recent[~recent.isekai & (recent.source == "LIGHT_NOVEL")].title_used.str.lower(), budget=3000),
        "control_manga_not_isekai": template_score(recent[~recent.isekai & (recent.source == "MANGA")].title_used.str.lower(), budget=3000),
        "control_original": template_score(recent[recent.source == "ORIGINAL"].title_used.str.lower(), budget=3000)}

    def top_words(titles, n=25):
        c = Counter()
        for t in titles:
            c.update({w for w in re.findall(r"[a-z']+", t.lower()) if w not in STOP and len(w) > 1})
        return c.most_common(n)

    def top_bigrams(titles, n=20):
        c = Counter()
        for t in titles:
            w = re.findall(r"[a-z']+", t.lower())
            c.update({" ".join(p) for p in zip(w, w[1:])})
        return c.most_common(n)

    R["top_words_isekai_2018_2026"] = top_words(isek[isek.year >= 2018].title_used)
    R["top_bigrams_isekai_2018_2026"] = top_bigrams(isek[isek.year >= 2018].title_used)
    R["top_words_control_2018_2026"] = top_words(ctrl[ctrl.year >= 2018].title_used)
    phrases = ["another world", "reincarnat", "villainess", "slow life", "skill", "demon lord", "hero", "saint", "level",
               "cheat", "exiled", "banished", "kicked out", "party", "strongest", "overpowered", "princess", "fianc", "noble"]
    R["phrase_share_pct_isekai_by_era"] = {
        p: {f"{y0}-{y1}": round(float(isek[isek.year.between(y0, y1)].title_used.str.lower().str.contains(p).mean() * 100), 1)
            for y0, y1 in eras[1:]} for p in phrases}

    # --- score vs length (isekai, English titles, at least some popularity so the score means something)
    s = isek[isek.score.notna() & (isek.popularity >= 5000) & (isek.year >= 2010)]
    R["score_vs_words_isekai"] = {"n": int(len(s)), "pearson": round(float(s.words_en.corr(s.score)), 3),
                                  "spearman": round(float(s.words_en.corr(s.score, method="spearman")), 3),
                                  "mean_score_short_le5": round(float(s[s.words_en <= 5].score.mean()), 1),
                                  "mean_score_long_ge10": round(float(s[s.words_en >= 10].score.mean()), 1),
                                  "n_short": int((s.words_en <= 5).sum()), "n_long": int((s.words_en >= 10).sum())}
    s2 = isek[isek.popularity.notna() & (isek.year >= 2010)]
    R["popularity_vs_words_isekai_spearman"] = round(float(s2.words_en.corr(s2.popularity, method="spearman")), 3)

    # --- forecast on mean English characters (a phone shows characters, not words)
    yrs = curve.loc[FIRST:LAST_FULL]
    R["forecast"] = {}
    # thresholds: one phone line (measured on AniList, 390 px, 16px font: measure_truncation.py), Google's result title
    # (~60 chars, SEO rule of thumb), PEP 8 line length (79), YouTube's title limit (100), a tweet (280)
    for name, thr in [("phone_line", 40), ("google", 60), ("pep8", 79), ("youtube", 100), ("tweet", 280)]:
        res, lin, exp = fit(yrs.index, yrs["chars_isekai"], thr)
        res["threshold_chars"] = thr
        R["forecast"][name] = res
    R["forecast"]["words_12"] = fit(yrs.index, yrs["mean_isekai"], 12)[0]
    R["phone_line_crossed"] = {int(y): float(v) for y, v in curve.chars_isekai.items() if y >= 2018}
    tf = CLEAN / "truncation.csv"
    if tf.exists():
        t = en.merge(pd.read_csv(tf), on="id")
        t = t[t.year.between(2010, 2026)]
        t["era"] = pd.cut(t.year, [2009, 2013, 2017, 2021, 2025, 2026], labels=["2010-2013", "2014-2017", "2018-2021", "2022-2025", "2026"])
        R["anilist_phone_card_cut_pct"] = {
            "isekai": t[t.isekai].groupby("era", observed=True).cut_card.mean().mul(100).round(1).to_dict(),
            "control": t[~t.isekai].groupby("era", observed=True).cut_card.mean().mul(100).round(1).to_dict()}
        R["anilist_phone_line_over_40_pct"] = {
            "isekai": t[t.isekai].assign(o=t.chars_en > 40).groupby("era", observed=True).o.mean().mul(100).round(1).to_dict(),
            "control": t[~t.isekai].assign(o=t.chars_en > 40).groupby("era", observed=True).o.mean().mul(100).round(1).to_dict()}

    # --- light novels vs their anime
    novf = CLEAN / "novels.csv"
    if novf.exists():
        nov = pd.read_csv(novf)
        nid = nov.set_index("id")
        pairs = []
        for _, r in main_[main_.source_novel_ids.notna()].iterrows():
            for sid in str(r.source_novel_ids).split("|"):
                sid = int(float(sid))
                if sid in nid.index and isinstance(nid.loc[sid, "title_english"], str):
                    pairs.append({"anime": r.title_used, "novel": nid.loc[sid, "title_english"], "year": r.year, "isekai": r.isekai,
                                  "native_identical": re.sub(r"\s", "", str(r.title_native)) == re.sub(r"\s", "", str(nid.loc[sid, "title_native"])),
                                  "anime_words": r.words_en, "novel_words": int(nid.loc[sid, "words_en"]),
                                  "anime_native": r.chars_native, "novel_native": int(nid.loc[sid, "chars_native"])})
        p = pd.DataFrame(pairs)
        if len(p):
            p.to_csv(CLEAN / "novel_anime_pairs.csv", index=False, encoding="utf-8")
            p["cut"] = p.novel_words - p.anime_words
            R["novel_vs_anime"] = {"pairs": int(len(p)), "isekai_pairs": int(p.isekai.sum()),
                                   "mean_words_novel": round(float(p.novel_words.mean()), 2), "mean_words_anime": round(float(p.anime_words.mean()), 2),
                                   "pct_anime_shorter": round(float((p.cut > 0).mean() * 100), 1), "pct_same": round(float((p.cut == 0).mean() * 100), 1),
                                   "native_pct_identical_title": round(float(p.native_identical.mean() * 100), 1),
                                   "native_pct_identical_length": round(float((p.anime_native == p.novel_native).mean() * 100), 1),
                                   "native_pct_anime_shorter": round(float((p.anime_native < p.novel_native).mean() * 100), 1),
                                   "biggest_cuts": json.loads(p.sort_values("cut", ascending=False).head(10).to_json(orient="records", force_ascii=False))}
        ln = nov[~nov.adult & nov.has_english & nov.first_season]
        R["novels_isekai_words_by_year"] = per_year(ln[ln.isekai], "words_en").reset_index().to_dict(orient="records")
        R["novels_control_words_by_year"] = per_year(ln[~ln.isekai], "words_en").reset_index().to_dict(orient="records")
        R["novels_native_isekai_by_year"] = per_year(nov[nov.isekai & ~nov.adult], "chars_native").reset_index().to_dict(orient="records")

    (OUT / "results.json").write_text(json.dumps(R, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
    charts(curve, isek, s)
    print(json.dumps({k: R[k] for k in ["counts", "eras_words", "eras_native_chars", "forecast", "score_vs_words_isekai"]}, indent=1))


def charts(curve, isek, s):
    CHARTS.mkdir(exist_ok=True)
    c = curve.loc[2005:2026]
    fig, ax = plt.subplots(1, 2, figsize=(14, 5))
    ax[0].plot(c.index, c.mean_isekai, "o-", label="isekai (mean words, EN)")
    ax[0].plot(c.index, c.mean_control, "o-", label="all other anime")
    ax[0].plot(c.index, c.median_isekai, ":", label="isekai median")
    for y, n in c.n_isekai.items():
        ax[0].annotate(int(n) if n == n else "", (y, c.mean_isekai[y]), fontsize=7, xytext=(0, 6), textcoords="offset points")
    ax[0].set_title("Words per title, first seasons, TV+ONA"); ax[0].legend(); ax[0].grid(alpha=.3)
    ax[1].plot(c.index, c.native_isekai, "o-", label="isekai (Japanese chars)")
    ax[1].plot(c.index, c.native_control, "o-", label="all other anime")
    ax[1].set_title("Japanese title length (characters)"); ax[1].legend(); ax[1].grid(alpha=.3)
    fig.tight_layout(); fig.savefig(CHARTS / "draft_curve.png", dpi=110); plt.close(fig)

    fig, ax = plt.subplots(figsize=(7, 5))
    ax.scatter(s.words_en, s.score, alpha=.5)
    ax.set_xlabel("words in English title"); ax.set_ylabel("AniList score"); ax.set_title("Isekai: score vs title length")
    fig.savefig(CHARTS / "draft_score.png", dpi=110); plt.close(fig)


if __name__ == "__main__":
    main()