"""
Nafs Results Analytics - Aseer Region (official data)
=====================================================
Sources (data/raw):
  * nafs_school_report_1444_1446.xlsx  school-level Nafs report 1444-1446 AH with tested /
    proficient counts, performance levels and official office / department / national averages
  * nafs_1447_full.xlsx  consolidated 1447 AH results: every school x grade x domain and
    sub-domain, proficiency (0-1) and score (0-100) - no student counts
  * 1447/nafs_1447_<grade>_<subject>.xlsx  earlier per-grade exports, kept to reconcile the
    consolidated file (DQ-14)

1447 tested counts are reconstructed exactly: within a school and grade every result shares
the same tested count (verified on 1444-1446), so the count is the least common multiple of
the reduced denominators of its 7-12 proficiency fractions. Back-test on 1445/1446: the exact
count is recovered for 99.8% of school-grades; regional rates within 0.01 pts.

Pipeline: load -> data-quality rules (incl. reconciliation to official figures) -> conformed
model -> analytics (trend, gaps, segments, statistics, projection) -> dashboard.json + charts.

Author : Adel Asiri (CDMP, PMP)
Stack  : Python 3 · pandas · NumPy · SciPy · Matplotlib

    python nafs_pipeline.py
Public outputs (./output) contain aggregates only - no school names or ministerial numbers.
The named priority list is written to data/private/ (blocked from the web).
"""
from __future__ import annotations

import json
import math
import re
from datetime import datetime, timezone
from fractions import Fraction
from pathlib import Path

import numpy as np
import pandas as pd
from scipy import stats

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402

ROOT = Path(__file__).resolve().parent
RAW = ROOT / "data" / "raw"
REF = ROOT / "data" / "reference"
PRIVATE = ROOT / "data" / "private"
OUT = ROOT / "output"
CHARTS = OUT / "charts"
for p in (CHARTS, PRIVATE):
    p.mkdir(parents=True, exist_ok=True)

YEARS = [1444, 1445, 1446, 1447]
EST_YEAR = 1447                # counts not published -> reconstructed from proficiency fractions
MIN_N = 15                     # school-level analytics: minimum tested students (prior year)
PRIORITY_N = 25                # minimum cohort to be listed for intervention
MATERIAL = 5.0                 # material gap / decline (percentage points)
SUPPRESS_SCHOOLS = 3           # dashboard hides any slice with fewer schools

SUBJ = {"الرياضيات": "MATH", "العلوم": "SCI", "القراءة": "READ"}
GEN = {"بنين": "B", "بنات": "G"}
AUTH = {"حكومي": "PUB", "أهلي": "PRV", "عالمي وأجنبي": "INT"}
SUB_EN = {
    "الأعداد والعمليات عليها": "Numbers & operations", "الجبر": "Algebra", "الهندسة والقياس": "Geometry & measurement",
    "البيانات والاحتمالات": "Data & probability", "الجبر والدوال الحقيقية والتحليل الرياضي": "Algebra, functions & analysis",
    "العلوم الفيزيائية والكيميائية": "Physical & chemical sciences", "علم الأرض والفلك": "Earth & space science",
    "علوم الحياة": "Life sciences", "استيعاب المقروء": "Reading comprehension", "المفردات": "Vocabulary",
    "دلالات الألفاظ": "Word meaning (semantics)",
}
OFFICE_STD = {  # non-standard office labels -> standard reference names
    "بارق": "مكتب التعليم ببارق", "البشائر": "مكتب التعليم بالبشائر", "بحر أبو سكينه": "مكتب التعليم ببحر أبو سكينة",
    "خيبر": "مكتب التعليم بخيبر", "الشعف": "مكتب التعليم بالشعف", "بللحمر": "مكتب التعليم ببللحمر", "بللسمر": "مكتب التعليم ببللسمر",
}
COLS = {
    "العام الدراسي": "year", "الرقم الوزاري": "school_id", "اسم المدرسة": "school_name", "مكتب التعليم": "office",
    "السلطة": "authority", "جنس المدرسة": "gender", "الصف": "grade", "مجموع طلاب الصف": "enrolled",
    "مجموع المختبرين": "tested", "المجال": "domain", "المجال الفرعي": "subdomain", "المتوسط": "score_500",
    "مجموع المتقنين": "proficient", "نسبة المتقنين": "prof_rate",
    "نسبة الطلاب في مستوى الأداء المنخفض جدا": "lvl_vlow", "نسبة الطلاب في مستوى الأداء المنخفض": "lvl_low",
    "نسبة الطلاب في مستوى الأداء المتوسط": "lvl_mid", "نسبة الطلاب في مستوى الأداء المرتفع": "lvl_high",
    "المتوسط على مستوى مكتب التعليم": "ref_office", "المتوسط على مستوى إدارة التعليم": "ref_dept",
    "المتوسط على مستوى المملكة": "ref_kingdom",
}


# ================================================================ 1. LOAD
def load():
    cache = PRIVATE / "_cache.pkl"
    src = RAW / "nafs_school_report_1444_1446.xlsx"
    full = RAW / "nafs_1447_full.xlsx"
    files = sorted((RAW / "1447").glob("*.xlsx"))
    stamp = [src.stat().st_mtime, full.stat().st_mtime] + [f.stat().st_mtime for f in files]
    if cache.exists():
        c = pd.read_pickle(cache)
        if c.get("stamp") == stamp:
            return c["hist"], c["cur"], c["govs"], c["exports"]

    hist = pd.read_excel(src, dtype=str).rename(columns=COLS)[list(COLS.values())]
    for c in ["enrolled", "tested", "score_500", "proficient", "prof_rate", "lvl_vlow", "lvl_low", "lvl_mid",
              "lvl_high", "ref_office", "ref_dept", "ref_kingdom"]:
        hist[c] = pd.to_numeric(hist[c], errors="coerce")
    hist["year"] = hist["year"].astype(int)
    hist["grade"] = hist["grade"].astype(int)

    # consolidated 1447 file (primary source)
    grade_map = {"الثالث ابتدائي": 3, "السادس ابتدائي": 6, "الثالث متوسط": 9}
    cur = pd.read_excel(full, dtype={"الرقم الوزاري للمدرسة": str}).rename(columns={
        "نسبة الإتقان": "prof_frac", "متوسط الدرجة": "score_100", "المجال": "domain", "المجال الفرعي": "subdomain",
        "الصف": "grade_text", "جنس_المدرسة": "gender", "نوع المدرسة": "authority", "المدينة/المحافظة": "gov",
        "اسم المدرسة": "school_name", "الرقم الوزاري للمدرسة": "school_id"})
    for c in ["domain", "subdomain", "grade_text", "gender", "authority", "gov", "school_name", "school_id"]:
        cur[c] = cur[c].astype(str).str.strip()
    cur["grade"] = cur["grade_text"].map(grade_map)
    cur["year"] = EST_YEAR

    # earlier per-grade exports (reconciliation only)
    frames = []
    for f in files:
        head = pd.read_excel(f, header=None, nrows=1).iloc[0, 0]
        grade_txt = re.search(r"grade_text هو (.+)", head).group(1).strip()
        dom = re.findall(r"\nDomain_Name هو (.+)", head)[0].strip()
        d = pd.read_excel(f, header=2, dtype={"الرقم الوزاري": str, "الصف": str})
        d = d.rename(columns={"نسبة الإتقان": "prof_frac", "متوسط الدرجة": "score_100", "المجال/المجال الفرعي": "subdomain",
                              "الصف": "grade", "جنس_المدرسة": "gender", "نوع المدرسة": "authority",
                              "المدينة/المحافظة": "gov", "اسم المدرسة": "school_name", "الرقم الوزاري": "school_id"})
        d["domain"] = dom
        d["source_file"] = f.name
        d["filter_grade"] = grade_txt
        frames.append(d)
    exports = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
    if len(exports):
        exports["grade"] = exports["grade"].astype(int)
        exports["school_id"] = exports["school_id"].astype(str).str.strip()
    govs = pd.read_csv(REF / "governorates.csv", encoding="utf-8-sig")
    pd.to_pickle(dict(stamp=stamp, hist=hist, cur=cur, govs=govs, exports=exports), cache)
    return hist, cur, govs, exports


# ================================================================ 2. COUNT RECONSTRUCTION (1447)
def reduced_den(p, max_den=5000):
    """Denominator of the reduced fraction behind a published proficiency share (1 for 0 or 1)."""
    if not 0 < p < 1:
        return 1
    f = Fraction(float(p)).limit_denominator(max_den)
    return f.denominator if abs(float(f) - p) < 1e-9 else 0


def reconstruct(df, share="prof_frac"):
    """Tested count per school-grade = LCM of the reduced denominators of all its results.
    Valid because every result of a school-grade shares one tested count (DQ-15)."""
    d = df[["school_id", "grade", share]].copy()
    d["q"] = d[share].map(reduced_den)
    bad = int((d["q"] == 0).sum())
    g = d.groupby(["school_id", "grade"]).agg(
        n=("q", lambda q: math.lcm(*[int(v) for v in q if v > 0])),
        informative=(share, lambda p: int(((p > 0) & (p < 1)).sum())), results=(share, "size"))
    return g, bad


def backtest(hist):
    """Apply the reconstruction to 1445 and 1446, whose true counts are known."""
    out, reg = {}, []
    for y in (1445, 1446):
        x = hist[hist["year"] == y].copy()
        x["share"] = x["proficient"] / x["tested"]
        g, _ = reconstruct(x, "share")
        true = x.groupby(["school_id", "grade"])["tested"].first()
        out[y] = dict(exact=round(100 * float((g["n"] == true.reindex(g.index)).mean()), 2),
                      students_true=int(true.sum()), students_rec=int(g["n"].sum()))
        m = x[x["domain"] == x["subdomain"]].join(g["n"], on=["school_id", "grade"])
        for (gr, dom), z in m.groupby(["grade", "domain"]):
            reg.append(100 * (z["share"] * z["n"]).sum() / z["n"].sum() - 100 * z["proficient"].sum() / z["tested"].sum())
        overall = 100 * (m["share"] * m["n"]).sum() / m["n"].sum() - 100 * m["proficient"].sum() / m["tested"].sum()
        out[y]["region_err"] = round(float(overall), 4)
    return dict(exact_1445=out[1445]["exact"], exact_1446=out[1446]["exact"],
                exact=round(min(out[1445]["exact"], out[1446]["exact"]), 1),
                max_err=round(float(np.abs(reg).max()), 3),
                region_err=round(max(abs(out[1445]["region_err"]), abs(out[1446]["region_err"])), 3),
                students_1446=[out[1446]["students_true"], out[1446]["students_rec"]])


# ================================================================ 3. DATA QUALITY
def data_quality(hist, cur, govs, exports):
    rules = []

    def rule(rid, dim_ar, dim_en, ar, en, checked, failed, status, act_ar, act_en):
        rules.append(dict(id=rid, dim_ar=dim_ar, dim_en=dim_en, rule_ar=ar, rule_en=en, checked=int(checked),
                          failed=int(failed), pass_rate=round(100 * (1 - failed / checked), 2) if checked else 100.0,
                          status=status, action_ar=act_ar, action_en=act_en))

    nh, nc = len(hist), len(cur)
    hist = hist.copy(); cur = cur.copy()

    # DQ-01 completeness: ministerial number present
    miss = hist["school_id"].isna().sum() + cur["school_id"].fillna("").str.strip().eq("").sum()
    rule("DQ-01", "الاكتمال", "Completeness", "الرقم الوزاري موجود في كل سجل", "Ministerial school number present",
         nh + nc, miss, "pass" if miss == 0 else "fixed", "عزل السجل وإعادته للمصدر", "Quarantine & return to source")

    # DQ-02 uniqueness
    dup = hist.duplicated(["year", "school_id", "grade", "domain", "subdomain"]).sum() + \
        cur.duplicated(["school_id", "grade", "domain", "subdomain"]).sum()
    rule("DQ-02", "التفرد", "Uniqueness", "سجل واحد لكل عام ومدرسة وصف ومجال فرعي", "One row per year/school/grade/sub-domain",
         nh + nc, dup, "pass" if dup == 0 else "fixed", "حذف التكرار بعد المراجعة", "Remove duplicates after review")

    # DQ-03 validity: counts
    bad = ((hist["tested"] > hist["enrolled"]) | (hist["proficient"] > hist["tested"]) | (hist["tested"] <= 0)).sum()
    rule("DQ-03", "الصحة", "Validity", "المختبرون ≤ طلاب الصف، والمتقنون ≤ المختبرين", "Tested ≤ enrolled and proficient ≤ tested",
         nh, bad, "pass" if bad == 0 else "fixed", "التحقق من ملف المصدر", "Verify against source file")

    # DQ-04 accuracy: reported rate = recomputed from counts
    rec = 100 * hist["proficient"] / hist["tested"]
    mism = ((rec - hist["prof_rate"]).abs() > 0.1).sum()
    rule("DQ-04", "الدقة", "Accuracy", "نسبة الإتقان المنشورة = المتقنون ÷ المختبرون", "Published rate equals proficient ÷ tested",
         nh, mism, "pass" if mism == 0 else "fixed", "إعادة الاحتساب من الأعداد", "Recompute from counts")

    # DQ-05 consistency: performance levels sum to 100
    lv = hist[["lvl_vlow", "lvl_low", "lvl_mid", "lvl_high"]].sum(axis=1)
    off = ((lv - 100).abs() > 0.5).sum()
    rule("DQ-05", "الاتساق", "Consistency", "مستويات الأداء الأربعة مجموعها 100%", "Four performance levels sum to 100%",
         nh, off, "pass" if off == 0 else "fixed", "مراجعة توزيع المستويات", "Review level distribution")

    # DQ-06 reconciliation to the official department average (per gender)
    main = hist[hist["domain"] == hist["subdomain"]]
    rec_rows = []
    for (y, g, d, s), x in main.groupby(["year", "grade", "domain", "gender"]):
        calc = 100 * x["proficient"].sum() / x["tested"].sum()
        offi = x["ref_dept"].mode().iloc[0]
        rec_rows.append(dict(year=int(y), grade=int(g), subject=SUBJ[d], gender=GEN[s], calc=round(calc, 2), official=float(offi), diff=round(calc - offi, 2)))
    recon = pd.DataFrame(rec_rows)
    fails = int((recon["diff"].abs() > 0.05).sum())
    rule("DQ-06", "المطابقة", "Reconciliation", "الإتقان المحسوب يطابق متوسط إدارة التعليم الرسمي (لكل صف ومجال وجنس)",
         "Computed proficiency matches the official department average (per grade, subject, gender)",
         len(recon), fails, "pass" if fails == 0 else "open", "التحقيق في الفرق مع مالك البيانات", "Investigate with data owner")

    # DQ-07 conformity: 1447 proficiency delivered as a 0-1 fraction
    frac = cur["prof_frac"].between(0, 1).sum()
    rule("DQ-07", "التوافق", "Conformity", "نسبة الإتقان بمقياس مئوي موحد (0–100)", "Proficiency on a common 0–100 scale",
         nc, frac, "fixed", "تحويل تصدير 1447 من كسر (0–1) إلى نسبة مئوية", "Convert 1447 export from 0–1 fraction to %")

    # DQ-08 comparability: score scale changed
    rule("DQ-08", "قابلية المقارنة", "Comparability", "مقياس متوسط الدرجة ثابت عبر السنوات", "Average-score scale stable across years",
         4, 1, "fixed", "عدم مقارنة متوسط الدرجة عبر السنوات (≈500 في 1444–1446 مقابل 0–100 في 1447) واعتماد نسبة الإتقان",
         "Do not compare scores across years (≈500 in 1444–46 vs 0–100 in 1447); use proficiency")

    # DQ-09 completeness: office missing in 1446
    miss_off = hist["office"].isna().sum()
    rule("DQ-09", "الاكتمال", "Completeness", "مكتب التعليم موجود لكل سجل", "Education office present on every row",
         nh, miss_off, "open", "الاعتماد على المحافظة من البيانات المرجعية ورفع الملاحظة لمالك البيانات",
         "Use governorate reference data; raise with data owner")

    # DQ-10 consistency: non-standard office labels
    nonstd = hist["office"].isin(OFFICE_STD).sum()
    hist["office"] = hist["office"].replace(OFFICE_STD)
    rule("DQ-10", "الاتساق", "Consistency", "اسم المكتب مطابق للقائمة المرجعية", "Office label matches reference list",
         int(hist["office"].notna().sum()), nonstd, "fixed", "توحيد 7 تسميات مختصرة (مثل «بارق» ← «مكتب التعليم ببارق»)",
         "Standardise 7 short labels (e.g. 'Bariq' → 'Bariq Education Office')")

    # DQ-11 master data: one name per ministerial number
    names = pd.concat([hist[["school_id", "school_name", "year"]], cur[["school_id", "school_name", "year"]]])
    multi = int((names.groupby("school_id")["school_name"].nunique() > 1).sum())
    rule("DQ-11", "البيانات الرئيسية", "Master data", "اسم واحد لكل رقم وزاري عبر السنوات", "One name per ministerial number across years",
         names["school_id"].nunique(), multi, "fixed", "اعتماد اسم أحدث سنة كاسم مرجعي", "Adopt latest-year name as golden record")

    # DQ-12 reference data: governorate available for every school
    gmap = cur.dropna(subset=["gov"]).assign(gov=lambda d: d["gov"].str.strip()).query("gov != ''").groupby("school_id")["gov"].agg(lambda s: s.mode().iloc[0])
    ids = pd.Index(pd.concat([hist["school_id"], cur["school_id"]]).unique())
    unmapped = int((~ids.isin(gmap.index)).sum())
    rule("DQ-12", "البيانات المرجعية", "Reference data", "المحافظة معروفة لكل مدرسة", "Governorate known for every school",
         len(ids), unmapped, "open", "ربط المدارس بالمحافظة عبر الرقم الوزاري؛ غير المربوطة تظهر «غير محددة»",
         "Join on ministerial number; unmatched shown as 'Unmapped'")

    # DQ-13 completeness: 1447 has no student counts -> reconstructed
    rec_n, bad_q = reconstruct(cur)
    weak = int((rec_n["informative"] == 0).sum())
    rule("DQ-13", "الاكتمال", "Completeness", "أعداد المختبرين والمتقنين متوفرة", "Tested / proficient counts available",
         nc, nc, "fixed", f"استنتاج عدد المختبرين لكل مدرسة وصف من مقامات نسب الإتقان (المضاعف المشترك الأصغر) — {len(rec_n):,} مدرسة×صف، {weak} بلا نسبة كاشفة",
         f"Tested count rebuilt per school-grade from the denominators of its proficiency shares (LCM) — {len(rec_n):,} school-grades, {weak} without an informative share")

    # DQ-14 reconciliation: consolidated 1447 file vs the earlier per-grade exports
    if len(exports):
        e = exports.rename(columns={"prof_frac": "p_e", "score_100": "s_e"})[["school_id", "grade", "subdomain", "domain", "p_e", "s_e"]]
        m = e.merge(cur[["school_id", "grade", "domain", "subdomain", "prof_frac", "score_100"]], on=["school_id", "grade", "domain", "subdomain"], how="left")
        diff = int((m["prof_frac"].isna() | ((m["prof_frac"] - m["p_e"]).abs() > 1e-9) | ((m["score_100"] - m["s_e"]).abs() > 1e-6)).sum())
        rule("DQ-14", "المطابقة", "Reconciliation", "ملف 1447 الموحد يطابق ملفات التصدير السابقة سجلًا بسجل", "Consolidated 1447 file matches the earlier exports row by row",
             len(m), diff, "pass" if diff == 0 else "open", "اعتماد الملف الموحد مصدرًا رئيسيًا", "Adopt the consolidated file as the primary source")

    # DQ-15 consistency: one tested count per school-grade across subjects and sub-domains (basis of DQ-13)
    k = hist.groupby(["year", "school_id", "grade"])["tested"].nunique()
    rule("DQ-15", "الاتساق", "Consistency", "عدد مختبرين واحد لكل مدرسة وصف في كل المجالات والمجالات الفرعية (1444–1446)",
         "One tested count per school-grade across all subjects and sub-domains (1444–1446)",
         len(k), int((k > 1).sum()), "pass" if (k > 1).sum() == 0 else "open", "أساس استنتاج أعداد 1447", "Basis for the 1447 count reconstruction")

    dq = pd.DataFrame(rules)
    summary = dict(rows=int(nh + nc), rows_hist=int(nh), rows_cur=int(nc), n_rules=len(rules),
                   passed=int((dq["status"] == "pass").sum()), fixed=int((dq["status"] == "fixed").sum()),
                   open=int((dq["status"] == "open").sum()), recon_checked=len(recon), recon_matched=len(recon) - fails,
                   schools=int(len(ids)), unmapped=unmapped)
    summary["rec_school_grades"] = int(len(rec_n)); summary["rec_students"] = int(rec_n["n"].sum()); summary["rec_bad"] = bad_q
    return hist, cur, gmap, dq, recon, summary, rec_n


# ================================================================ 4. MODEL
def build_model(hist, cur, gmap, govs, rec_n):
    gov_idx = {g: i for i, g in enumerate(govs["gov_ar"])}
    UNM = len(gov_idx)  # "unmapped" bucket index

    h = hist.copy()
    h["subject"] = h["domain"].map(SUBJ)
    h["est"] = 0

    c = cur.copy()
    c["subject"] = c["domain"].map(SUBJ)
    c["prof_rate"] = 100 * c["prof_frac"]
    # reconstructed counts: one tested count per school-grade (see reconstruct / DQ-13, DQ-15)
    c = c.join(rec_n["n"].rename("tested"), on=["school_id", "grade"])
    c["proficient"] = (c["prof_frac"] * c["tested"]).round()
    c["est"] = 1

    keep = ["year", "school_id", "grade", "subject", "domain", "subdomain", "gender", "authority", "tested", "proficient",
            "prof_rate", "est"]
    allr = pd.concat([h[keep + ["enrolled", "lvl_vlow", "lvl_low", "lvl_mid", "lvl_high", "ref_kingdom"]], c[keep]], ignore_index=True)
    allr["g"] = allr["school_id"].map(gmap).map(gov_idx).fillna(UNM).astype(int)
    allr["s"] = allr["gender"].map(GEN)
    allr["a"] = allr["authority"].map(AUTH)
    main = allr[allr["domain"] == allr["subdomain"]].copy()
    subs = allr[allr["domain"] != allr["subdomain"]].copy()
    for lv in ["lvl_high", "lvl_mid", "lvl_low", "lvl_vlow"]:
        main[lv + "_n"] = main[lv] / 100 * main["tested"]
    main["nat_x_n"] = main["ref_kingdom"] * main["tested"]
    return main, subs, UNM


def rate(x):
    return 100 * x["proficient"].sum() / x["tested"].sum()


# ================================================================ 5. ANALYSIS
def analyse(main, subs, govs, UNM):
    gov_list = list(govs["gov_ar"])
    en = dict(zip(govs["gov_ar"], govs["gov_en"]))
    y1, y0 = EST_YEAR, EST_YEAR - 1

    # cube for the interactive dashboard
    agg = dict(tested=("tested", "sum"), prof=("proficient", "sum"), expected=("enrolled", "sum"),
               hi=("lvl_high_n", "sum"), mid=("lvl_mid_n", "sum"), low=("lvl_low_n", "sum"), vlow=("lvl_vlow_n", "sum"),
               nat=("nat_x_n", "sum"))
    cube = main.groupby(["year", "grade", "subject", "s", "a", "g"]).agg(**agg).reset_index()
    est = cube["year"] == EST_YEAR
    cube.loc[est, ["expected", "hi", "mid", "low", "vlow", "nat"]] = np.nan
    for c in ["tested", "prof", "expected", "hi", "mid", "low", "vlow", "nat"]:
        cube[c] = cube[c].round(1)

    # school register for distinct counts & suppression: [gov, s, a, mask1444, mask1445, mask1446, mask1447]
    bits = {3: 1, 6: 2, 9: 4}
    reg = main.groupby(["school_id", "year"])["grade"].agg(lambda g: sum(bits[x] for x in set(g))).unstack(fill_value=0)
    attrs = main.sort_values("year").groupby("school_id")[["g", "s", "a"]].last()
    reg = attrs.join(reg)
    register = [[int(r["g"]), r["s"], r["a"]] + [int(r.get(y, 0)) for y in YEARS] for _, r in reg.iterrows()]

    # regional trend + national (official kingdom reference, weighted by regional tested mix)
    trend = []
    for y in YEARS:
        d = main[main["year"] == y]
        nat = float(d["nat_x_n"].sum() / d["tested"].sum()) if y != EST_YEAR else None
        part = float(100 * d["tested"].sum() / d["enrolled"].sum()) if y != EST_YEAR else None
        students = int(d.groupby(["school_id", "grade"])["tested"].max().sum())
        trend.append(dict(year=int(y), region=round(rate(d), 2), national=None if nat is None else round(nat, 2),
                          participation=None if part is None else round(part, 2), estimated=False, derived=bool(y == EST_YEAR),
                          schools=int(d["school_id"].nunique()), students=students, results=int(d["tested"].sum())))

    # region per grade & subject (both genders), all years
    grid = main.groupby(["year", "grade", "subject"]).apply(rate).round(2).unstack("subject")

    # school-level indicators: 1447 vs 1446
    two = main[main["year"].isin([y0, y1])]
    sch = two.groupby(["school_id", "year"]).agg(prof=("proficient", "sum"), wsum=("tested", "sum")).reset_index()
    sch["rate"] = 100 * sch["prof"] / sch["wsum"]
    # cohort size = students tested (max over subjects within a grade, summed over grades) - not student-subject rows
    coh = two.groupby(["school_id", "year", "grade"])["tested"].max().groupby(["school_id", "year"]).sum().rename("tested")
    sch = sch.merge(coh.reset_index(), on=["school_id", "year"])
    wide = sch.pivot(index="school_id", columns="year", values=["rate", "tested"])
    wide.columns = [f"{a}_{b}" for a, b in wide.columns]
    wide = wide.dropna()
    wide = wide[wide[f"tested_{y0}"] >= MIN_N]
    attrs = main.sort_values("year").groupby("school_id")[["g", "s", "a", "grade"]].last()
    names = main.sort_values("year").groupby("school_id")["grade"].agg(lambda g: sorted(set(g)))
    wide = wide.join(attrs)
    wide["grades"] = names.reindex(wide.index)
    wide["abs_change"] = wide[f"rate_{y1}"] - wide[f"rate_{y0}"]
    region_now = next(t["region"] for t in trend if t["year"] == y1)
    region_prev = next(t["region"] for t in trend if t["year"] == y0)
    region_change = region_now - region_prev
    wide["change"] = wide["abs_change"] - region_change      # change relative to the region's own change

    def seg(r):
        up, hi = r["change"] >= 0, r[f"rate_{y1}"] >= region_now
        return "lead_up" if hi and up else "lead_down" if hi else "low_up" if up else "low_down"
    wide["segment"] = wide.apply(seg, axis=1)
    wide["priority"] = (wide[f"rate_{y1}"] <= region_now - MATERIAL) & (wide["change"] <= -MATERIAL) & (wide[f"tested_{y0}"] >= PRIORITY_N)
    wide["priority_score"] = (region_now - wide[f"rate_{y1}"]) + (-wide["change"])
    pri = wide[wide["priority"]].sort_values("priority_score", ascending=False)

    def stage(gr):
        s = set(gr)
        return "مجمع" if len(s) > 1 and 9 in s and (3 in s or 6 in s) else "متوسطة" if s == {9} else "ابتدائية"
    gov_name = lambda i: gov_list[i] if i < len(gov_list) else "غير محددة"
    gov_en = lambda i: en[gov_list[i]] if i < len(gov_list) else "Unmapped"
    priority = [dict(id=f"P-{k+1:02d}", gov=gov_name(int(r["g"])), gov_en=gov_en(int(r["g"])), stage=stage(r["grades"]),
                     gender="بنات" if r["s"] == "G" else "بنين", rate=round(r[f"rate_{y1}"], 1), prev=round(r[f"rate_{y0}"], 1),
                     change=round(r["abs_change"], 1), rel=round(r["change"], 1), tested=int(r[f"tested_{y0}"]))
                for k, (_, r) in enumerate(pri.head(12).iterrows())]
    # private, named list for internal follow-up (never published)
    return_named = pri.reset_index()[["school_id", f"rate_{y0}", f"rate_{y1}", "abs_change", "change", f"tested_{y0}"]].rename(columns={"change": "change_vs_region"})

    # statistics on 1447 school rates
    girls = wide.loc[wide["s"] == "G", f"rate_{y1}"]; boys = wide.loc[wide["s"] == "B", f"rate_{y1}"]
    t_g = stats.ttest_ind(girls, boys, equal_var=False)
    pub = wide.loc[wide["a"] == "PUB", f"rate_{y1}"]; prv = wide.loc[wide["a"] != "PUB", f"rate_{y1}"]
    t_a = stats.ttest_ind(prv, pub, equal_var=False)
    rho = stats.spearmanr(wide[f"tested_{y0}"], wide[f"rate_{y1}"])
    tr = {t["year"]: t["region"] for t in trend}
    gstud = main[main["year"] == y1].groupby("s").apply(rate)
    statistics = dict(
        gender=dict(girls=round(girls.mean(), 2), boys=round(boys.mean(), 2), diff=round(girls.mean() - boys.mean(), 2),
                    t=round(float(t_g.statistic), 2), p=float(t_g.pvalue), n_girls=int(len(girls)), n_boys=int(len(boys))),
        authority=dict(private=round(prv.mean(), 2), public=round(pub.mean(), 2), diff=round(prv.mean() - pub.mean(), 2),
                       t=round(float(t_a.statistic), 2), p=float(t_a.pvalue), n_private=int(len(prv)), n_public=int(len(pub))),
        size_corr=dict(rho=round(float(rho.statistic), 3), p=float(rho.pvalue)),
        change=dict(from_year=int(y0), to_year=int(y1), value=round(tr[y1] - tr[y0], 2), base=round(tr[y0], 2),
                    since_first=round(tr[y0] - tr[YEARS[0]], 2)),
        gender_students=dict(B=round(float(gstud.get("B", np.nan)), 2), G=round(float(gstud.get("G", np.nan)), 2)),
        schools_classified=int(len(wide)), min_n=MIN_N, region_change=round(region_change, 2),
    )

    # sub-domains
    sd = subs.groupby(["year", "grade", "subject", "subdomain"]).agg(tested=("tested", "sum"), prof=("proficient", "sum")).reset_index()
    sd["sub_en"] = sd["subdomain"].map(SUB_EN)

    # governorates
    def gov_rates(y):
        d = main[(main["year"] == y) & (main["g"] < UNM)]
        return d.groupby("g").apply(rate)
    gv1, gv0 = gov_rates(y1), gov_rates(y0)
    gchg = (gv1 - gv0).dropna().sort_values()
    subj1 = main[main["year"] == y1].groupby("subject").apply(rate)
    grade_subj1 = main[main["year"] == y1].groupby(["grade", "subject"]).apply(rate)

    t = {r["year"]: r for r in trend}
    nat46 = t[1446]["national"]; gap46 = t[1446]["region"] - nat46; gap44 = t[1444]["region"] - t[1444]["national"]
    weakest = grade_subj1.idxmin(); strongest = grade_subj1.idxmax()
    sd47 = sd[(sd["year"] == y1)].assign(r=lambda d: 100 * d["prof"] / d["tested"]).sort_values("r").iloc[0]
    gname = {3: ("الثالث الابتدائي", "Grade 3"), 6: ("السادس الابتدائي", "Grade 6"), 9: ("الثالث المتوسط", "Grade 9")}
    sname = {"MATH": ("الرياضيات", "Mathematics"), "SCI": ("العلوم", "Science"), "READ": ("القراءة", "Reading")}
    seg_counts = wide["segment"].value_counts().to_dict()
    dsub = (main[main["year"] == y1].groupby(["grade", "subject"]).apply(rate) - main[main["year"] == y0].groupby(["grade", "subject"]).apply(rate)).sort_values()
    drop_k, rise_k = dsub.index[0], dsub.index[-1]
    # change decomposition: performance within grade x subject vs change in the tested mix
    cell = main[main["year"].isin([y0, y1])].groupby(["year", "grade", "subject"]).agg(t=("tested", "sum"), p=("proficient", "sum"))
    cell["r"] = 100 * cell["p"] / cell["t"]
    cell["w"] = cell["t"] / cell.groupby(level="year")["t"].transform("sum")
    c0, c1 = cell.loc[y0], cell.loc[y1]
    idx = c1.index.intersection(c0.index)
    rate_eff = float((c0.loc[idx, "w"] * (c1.loc[idx, "r"] - c0.loc[idx, "r"])).sum())
    mix_eff = float(((c1.loc[idx, "w"] - c0.loc[idx, "w"]) * c1.loc[idx, "r"]).sum())
    std = float((c0.loc[idx, "w"] * c1.loc[idx, "r"]).sum())
    same = main[main["year"].isin([y0, y1])].groupby(["school_id", "grade", "year"])["tested"].max().unstack("year").dropna()
    coh_chg = (same[y1].groupby(level="grade").sum() / same[y0].groupby(level="grade").sum() - 1) * 100
    statistics["decomp"] = dict(rate_effect=round(rate_eff, 2), mix_effect=round(mix_eff, 2), standardised=round(std, 2),
                                cohort_change={int(k): round(float(v), 1) for k, v in coh_chg.items()})
    gs = statistics["gender_students"]; gsch = statistics["gender"]
    sig = "دال إحصائيًا" if gsch["p"] < 0.05 else "غير دال إحصائيًا"
    sig_en = "statistically significant" if gsch["p"] < 0.05 else "not statistically significant"
    insights = [
        dict(kind="up",
             ar=f"ارتفعت نسبة الإتقان في عسير من {t[1444]['region']:.1f}% (1444هـ) إلى {t[1446]['region']:.1f}% (1446هـ) بزيادة {t[1446]['region']-t[1444]['region']:.1f} نقطة — أرقام مطابقة للمتوسط الرسمي لإدارة التعليم.",
             en=f"Aseer proficiency rose from {t[1444]['region']:.1f}% (1444 AH) to {t[1446]['region']:.1f}% (1446 AH), +{t[1446]['region']-t[1444]['region']:.1f} pts — reconciled to the official department averages."),
        dict(kind="gap",
             ar=f"في 1447هـ بلغت النسبة {t[1447]['region']:.2f}% ({t[1447]['region']-t[1446]['region']:+.2f} نقطة عن 1446هـ). أكبر تراجع في {sname[drop_k[1]][0]} {gname[drop_k[0]][0]} ({dsub[drop_k]:+.1f} نقطة)، وأكبر تحسن في {sname[rise_k[1]][0]} {gname[rise_k[0]][0]} ({dsub[rise_k]:+.1f}).",
             en=f"1447 AH proficiency is {t[1447]['region']:.2f}% ({t[1447]['region']-t[1446]['region']:+.2f} pts vs 1446). The largest drop is {gname[drop_k[0]][1]} {sname[drop_k[1]][1]} ({dsub[drop_k]:+.1f} pts) and the largest gain {gname[rise_k[0]][1]} {sname[rise_k[1]][1]} ({dsub[rise_k]:+.1f})."),
        dict(kind="subject",
             ar=f"من تغير {t[1447]['region']-t[1446]['region']:+.2f} نقطة: {rate_eff:+.2f} تغير فعلي في الأداء داخل الصفوف والمجالات، و{mix_eff:+.2f} أثر تغير مزيج المختبرين (مختبرو الثالث الابتدائي {coh_chg[3]:+.1f}% والسادس {coh_chg[6]:+.1f}% في المدارس نفسها). بمزيج 1446هـ تكون النسبة {std:.2f}%.",
             en=f"Of the {t[1447]['region']-t[1446]['region']:+.2f} pt change, {rate_eff:+.2f} is real performance change within grade and subject and {mix_eff:+.2f} is the shift in who was tested (grade 3 testers {coh_chg[3]:+.1f}%, grade 6 {coh_chg[6]:+.1f}% in the same schools). At the 1446 mix, 1447 would be {std:.2f}%."),
        dict(kind="gap",
             ar=f"الفجوة مع متوسط المملكة {gap46:+.1f} نقطة في 1446هـ مقابل {gap44:+.1f} في 1444هـ: تحسنت المنطقة بوتيرة المملكة نفسها دون أن تضيق الفجوة.",
             en=f"The gap to the national average is {gap46:+.1f} pts in 1446 AH vs {gap44:+.1f} in 1444 AH: the region improved at the national pace without closing the gap."),
        dict(kind="subject",
             ar=f"في 1447هـ أعلى إتقان في {sname[strongest[1]][0]} {gname[strongest[0]][0]} ({grade_subj1[strongest]:.1f}%) وأدناه في {sname[weakest[1]][0]} {gname[weakest[0]][0]} ({grade_subj1[weakest]:.1f}%)؛ وأضعف مجال فرعي «{sd47['subdomain']}» في {sname[sd47['subject']][0]} {gname[int(sd47['grade'])][0]} ({sd47['r']:.1f}%).",
             en=f"In 1447 AH the highest proficiency is {gname[strongest[0]][1]} {sname[strongest[1]][1]} ({grade_subj1[strongest]:.1f}%) and the lowest {gname[weakest[0]][1]} {sname[weakest[1]][1]} ({grade_subj1[weakest]:.1f}%); the weakest sub-domain is {sd47['sub_en']} in {gname[int(sd47['grade'])][1]} {sname[sd47['subject']][1]} ({sd47['r']:.1f}%)."),
        dict(kind="gov",
             ar=f"أعلى المحافظات في 1447هـ {gov_list[gv1.idxmax()]} ({gv1.max():.1f}%) وأدناها {gov_list[gv1.idxmin()]} ({gv1.min():.1f}%)، والأكثر تحسنًا عن 1446هـ {gov_list[gchg.index[-1]]} ({gchg.iloc[-1]:+.1f}).",
             en=f"In 1447 AH the top governorate is {en[gov_list[gv1.idxmax()]]} ({gv1.max():.1f}%) and the lowest {en[gov_list[gv1.idxmin()]]} ({gv1.min():.1f}%); most improved since 1446 is {en[gov_list[gchg.index[-1]]]} ({gchg.iloc[-1]:+.1f})."),
        dict(kind="gender",
             ar=f"تقارب البنين والبنات على مستوى الطلاب في 1447هـ ({gs['B']:.1f}% مقابل {gs['G']:.1f}%) بعد أن كانت البنات متقدمات في 1444هـ؛ وعلى مستوى المدرسة الفرق {gsch['diff']:+.1f} نقطة ({sig}).",
             en=f"Boys and girls converge at student level in 1447 AH ({gs['B']:.1f}% vs {gs['G']:.1f}%) after girls led in 1444; at school level the difference is {gsch['diff']:+.1f} pts ({sig_en})."),
        dict(kind="priority",
             ar=f"{len(pri)} مدرسة مرشحة لأولوية التدخل: دون متوسط المنطقة بـ{MATERIAL:.0f} نقاط فأكثر، وتراجعها يزيد على تراجع المنطقة بـ{MATERIAL:.0f} نقاط فأكثر (من {len(wide):,} مدرسة مصنفة).",
             en=f"{len(pri)} schools flagged for priority intervention: ≥{MATERIAL:.0f} pts below the regional average and declining ≥{MATERIAL:.0f} pts more than the region (of {len(wide):,} classified)."),
    ]
    segments = dict(threshold=region_now, year=int(y1), prev=int(y0), region_change=round(region_change, 2), counts={k: int(v) for k, v in seg_counts.items()},
                    priority=int(len(pri)), material=MATERIAL, min_n=MIN_N,
                    points=[[round(r[f"rate_{y1}"], 1), round(r["change"], 1), "priority" if r["priority"] else r["segment"]] for _, r in wide.iterrows()])
    return dict(cube=cube, register=register, trend=trend, grid=grid, segments=segments, priority=priority, named=return_named,
                statistics=statistics, subdomains=sd, insights=insights, gv1=gv1, gv0=gv0, wide=wide)


# ================================================================ 6. CHARTS (static, report-ready)
INK, MUTED, GRID = "#0B2E3A", "#5B6B73", "#E3E8EA"
C_REG, C_NAT, C_ACC = "#0E7C7B", "#C8A24A", "#C2410C"


def style(ax, title):
    ax.set_title(title, loc="left", fontsize=12, color=INK, fontweight="bold", pad=10)
    for s in ["top", "right"]:
        ax.spines[s].set_visible(False)
    for s in ["left", "bottom"]:
        ax.spines[s].set_color(GRID)
    ax.tick_params(colors=MUTED, labelsize=9)
    ax.grid(axis="y", color=GRID, linewidth=0.8)
    ax.set_axisbelow(True)


def charts(res, dq, govs):
    en = list(govs["gov_en"])
    tr = pd.DataFrame(res["trend"])
    fig, ax = plt.subplots(figsize=(7, 3.8), dpi=150)
    act = tr[tr["national"].notna()]
    ax.plot(tr["year"], tr["region"], marker="o", color=C_REG, lw=2.4, label="Aseer region")
    ax.plot(act["year"], act["national"], marker="o", color=C_NAT, lw=2, ls="--", label="National (published to 1446)")
    for _, r in tr.iterrows():
        ax.annotate(f"{r['region']:.2f}%", (r["year"], r["region"]), textcoords="offset points", xytext=(0, 9), ha="center", fontsize=8, color=C_REG)
    ax.set_xticks(tr["year"]); ax.set_xticklabels([f"{y} AH" for y in tr["year"]])
    ax.set_ylabel("Proficiency %", color=MUTED); ax.legend(frameon=False, fontsize=8)
    style(ax, "Proficiency trend vs national benchmark"); fig.tight_layout(); fig.savefig(CHARTS / "01_trend.png"); plt.close(fig)

    c = res["cube"]
    h = c[c["year"] == EST_YEAR].groupby(["subject", "grade"]).apply(lambda d: 100 * d["prof"].sum() / d["tested"].sum()).unstack().reindex(["MATH", "SCI", "READ"])
    fig, ax = plt.subplots(figsize=(6.2, 3.4), dpi=150)
    im = ax.imshow(h.values, cmap="YlGnBu", vmin=15, vmax=60)
    ax.set_xticks(range(h.shape[1])); ax.set_xticklabels([f"Grade {g}" for g in h.columns])
    ax.set_yticks(range(h.shape[0])); ax.set_yticklabels(["Mathematics", "Science", "Reading"])
    for i in range(h.shape[0]):
        for j in range(h.shape[1]):
            v = h.values[i, j]
            ax.text(j, i, "—" if np.isnan(v) else f"{v:.1f}%", ha="center", va="center", fontsize=10, color="white" if (not np.isnan(v) and v > 42) else INK)
    ax.set_title(f"Subject x grade · {EST_YEAR} AH", loc="left", fontsize=12, color=INK, fontweight="bold")
    for s in ax.spines.values():
        s.set_visible(False)
    fig.colorbar(im, ax=ax, fraction=0.04); fig.tight_layout(); fig.savefig(CHARTS / "02_subject_grade.png"); plt.close(fig)

    gv = res["gv1"].sort_values()
    fig, ax = plt.subplots(figsize=(7, 5), dpi=150)
    thr = res["segments"]["threshold"]
    ax.barh([en[i] for i in gv.index], gv.values, color=[C_REG if v >= thr else "#9FB8BF" for v in gv.values])
    ax.axvline(thr, color=C_ACC, ls="--", lw=1.2)
    for i, v in enumerate(gv.values):
        ax.text(v + 0.3, i, f"{v:.1f}%", va="center", fontsize=8, color=MUTED)
    style(ax, f"Proficiency by governorate · {EST_YEAR} AH"); ax.grid(axis="x", color=GRID); ax.grid(axis="y", visible=False)
    fig.tight_layout(); fig.savefig(CHARTS / "03_governorates.png"); plt.close(fig)

    w = res["wide"].assign(segment=lambda d: np.where(d["priority"], "priority", d["segment"]))
    y1, y0 = EST_YEAR, EST_YEAR - 1
    pal = {"lead_up": C_REG, "lead_down": C_NAT, "low_up": "#6B8E9B", "low_down": "#B7C4C9", "priority": C_ACC}
    lab = {"lead_up": "Leading & improving", "lead_down": "Leading, declining", "low_up": "Below avg, improving",
           "low_down": "Below avg, declining (watch)", "priority": "Priority for intervention"}
    fig, ax = plt.subplots(figsize=(7, 4.6), dpi=150)
    for k in ["low_down", "low_up", "lead_down", "lead_up", "priority"]:
        d = w[w["segment"] == k]
        ax.scatter(d[f"rate_{y1}"], d["change"], s=9, alpha=0.6, color=pal[k], label=f"{lab[k]} ({len(d)})", edgecolors="none")
    ax.axvline(thr, color=MUTED, lw=0.8); ax.axhline(0, color=MUTED, lw=0.8)
    ax.set_xlabel(f"School proficiency {y1} AH (%)", color=MUTED); ax.set_ylabel(f"Change vs {y0}, relative to region (pts)", color=MUTED)
    ax.legend(frameon=False, fontsize=7.5, loc="upper left")
    style(ax, "School segmentation"); fig.tight_layout(); fig.savefig(CHARTS / "04_segmentation.png"); plt.close(fig)

    fig, ax = plt.subplots(figsize=(7, 3.6), dpi=150)
    st = dq["status"].map({"pass": C_REG, "fixed": C_NAT, "open": C_ACC})
    ax.barh(dq["id"][::-1], dq["pass_rate"][::-1], color=st[::-1])
    ax.set_xlim(0, 105)
    for i, (v, s) in enumerate(zip(dq["pass_rate"][::-1], dq["status"][::-1])):
        ax.text(min(v, 100) + 1, i, {"pass": "pass", "fixed": "remediated", "open": "open"}[s], va="center", fontsize=7.5, color=MUTED)
    style(ax, "Data-quality rules · pass rate on raw data"); ax.grid(axis="x", color=GRID); ax.grid(axis="y", visible=False)
    fig.tight_layout(); fig.savefig(CHARTS / "05_dq_scorecard.png"); plt.close(fig)


# ================================================================ 7. EXPORT
def export(res, dq, recon, summary, govs, back):
    c = res["cube"]
    fields = ["year", "grade", "subject", "s", "a", "g", "tested", "expected", "prof", "hi", "mid", "low", "vlow", "nat"]
    rows = [[None if (isinstance(v, float) and np.isnan(v)) else v for v in r] for r in c[fields].values.tolist()]
    sd = res["subdomains"]
    feed = dict(
        meta=dict(title_ar="تحليل نتائج اختبارات نافس — منطقة عسير", title_en="Nafs Results Analytics — Aseer Region",
                  generated=datetime.now(timezone.utc).strftime("%Y-%m-%d"), years=YEARS, est_year=EST_YEAR,
                  pipeline="Python · pandas · NumPy · SciPy · Matplotlib", suppress=SUPPRESS_SCHOOLS,
                  derived_year=EST_YEAR, backtest=back,
                  source_ar="تقرير نافس على مستوى المدرسة لإدارة تعليم عسير 1444–1446هـ، وملف نتائج نافس الموحد لعام 1447هـ (كل المدارس والمجالات الفرعية)",
                  source_en="Aseer school-level Nafs report 1444–1446 AH and the consolidated 1447 AH Nafs results file (all schools and sub-domains)"),
        labels=dict(
            gov=[dict(ar=a, en=e) for a, e in zip(govs["gov_ar"], govs["gov_en"])] + [dict(ar="غير محددة", en="Unmapped")],
            subject={"MATH": dict(ar="الرياضيات", en="Mathematics"), "SCI": dict(ar="العلوم", en="Science"), "READ": dict(ar="القراءة", en="Reading")},
            grade={"3": dict(ar="الثالث الابتدائي", en="Grade 3"), "6": dict(ar="السادس الابتدائي", en="Grade 6"), "9": dict(ar="الثالث المتوسط", en="Grade 9")},
        ),
        dq=dict(rules=dq.to_dict("records"), recon=recon.to_dict("records"), **summary),
        cube=dict(fields=fields, rows=rows), trend=res["trend"], segments=res["segments"], priority=res["priority"],
        statistics=res["statistics"], insights=res["insights"], register=res["register"],
        subdomains=[[int(r.year), int(r.grade), r.subject, r.subdomain, r.sub_en, round(float(r.tested), 1), round(float(r.prof), 1)] for r in sd.itertuples()],
    )
    (OUT / "dashboard.json").write_text(json.dumps(feed, ensure_ascii=False, separators=(",", ":"), default=float), encoding="utf-8")
    dq.drop(columns=[]).to_csv(OUT / "dq_report.csv", index=False, encoding="utf-8-sig")
    res["named"].to_csv(PRIVATE / "priority_schools_named.csv", index=False, encoding="utf-8-sig")
    return feed


def main():
    hist, cur, govs, exports = load()
    hist, cur, gmap, dq, recon, summary, rec_n = data_quality(hist, cur, govs, exports)
    main_df, subs, UNM = build_model(hist, cur, gmap, govs, rec_n)
    back = backtest(hist)
    res = analyse(main_df, subs, govs, UNM)
    charts(res, dq, govs)
    feed = export(res, dq, recon, summary, govs, back)
    print(f"rows={summary['rows']:,} rules={summary['n_rules']} pass={summary['passed']} fixed={summary['fixed']} open={summary['open']} "
          f"recon={summary['recon_matched']}/{summary['recon_checked']} cube={len(feed['cube']['rows']):,} rec={summary['rec_school_grades']}/{summary['rec_students']} backtest={back}")
    for t in feed["trend"]:
        print(" ", t)
    for i in feed["insights"]:
        print(" •", i["en"])
    print(feed["statistics"])


if __name__ == "__main__":
    main()