hardware-counters

#!/usr/bin/env python3
"""
Do the CrayPat hardware counters actually earn their keep?

This is the central question for the project. The premise is that hardware
counters let you predict application performance. But some of the features are
free configuration facts (core count, rank count, thread count) that require no
profiling at all, and an independent review found that a config-only linear
model already scores well. If the counters add nothing beyond what the job
submission script already knows, the premise does not hold.

Feature groups, in increasing order of measurement cost:

  config     log2(ncore), log2(nrank), log2(nthread)
             Free. Known before the job runs.

  analytic   + log10(cycles / peak_clock)
             One counter (PAPI_TOT_CYC). Cheap: a single profiled run with one
             counter, no rotation needed.

  cheap      + instructions-derived ratios (ipc, log_instr_per_rank,
             flops_per_instr, flops_per_cycle)
             Two counters, one counter set.

  full       + cache, TLB, stall, prefetch, energy and Roofline features
             All five counter sets, i.e. five profiled runs per configuration.

If "full" does not beat "config" by a useful margin, the profiling campaign is
not justified by the prediction task.

Protocol is identical to retrain2.py: merged configurations, leave-one-
application-out, error = max(pred/act, act/pred), median-constant reference.
"""
import os, warnings, sys
from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import Ridge
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from scipy.stats import wilcoxon

warnings.filterwarnings("ignore")
# Data and output roots. Overridable so the pipeline can be relocated
# (a different account, a scratch copy, another site) without editing code;
# the default keeps existing invocations working unchanged.
D      = os.environ.get("DISS_ROOT", "/work/project/project/user")
OUT    = Path(f"{D}/analysis/out"); OUT.mkdir(parents=True, exist_ok=True)
F_PEAK = 2.25e9
sys.path.insert(0, f"{D}/analysis")
from retrain2 import load_merged, build_features, fac

GROUPS = {
    "config":   ["log_ncore", "log_nrank", "log_nthread"],
    "analytic": ["log_t_analytic"],
    "cheap":    ["ipc", "log_instr_per_rank", "flops_per_instr", "flops_per_cycle"],
    "full":     ["l2_hit_rate", "l2_miss_per_instr", "l1_access_per_instr",
                 "l3_miss_per_instr", "l3_lat_per_miss", "tlb_miss_per_instr",
                 "stall_load_frac", "stall_store_frac", "stall_fp_frac",
                 "prefetch_l2_frac", "energy_per_instr", "core_energy_frac",
                 "arith_intensity"],
}
CUMULATIVE = [
    ("config only (free)",              ["config"]),
    ("+ analytic cycles (1 counter)",   ["config", "analytic"]),
    ("+ instruction ratios (1 set)",    ["config", "analytic", "cheap"]),
    ("+ all counters (5 sets)",         ["config", "analytic", "cheap", "full"]),
]


def pipe(est):
    return Pipeline([("i", SimpleImputer(strategy="median")),
                     ("s", StandardScaler()), ("e", est)])


def loao(X, y, t_an, eta, apps_col, est_fn):
    r = np.full(len(y), np.nan)
    for a in sorted(apps_col.unique()):
        te = (apps_col == a).values
        m = est_fn()
        m.fit(X[~te], np.log10(eta[~te]))
        r[te] = fac(t_an[te] / (10 ** m.predict(X[te])), y[te])
    return r


def main():
    df = load_merged()
    Xall = build_features(df)
    y    = df.runtime_s.values
    t_an = (df.PAPI_TOT_CYC / F_PEAK).values
    eta  = t_an / y

    L, res = [], {}
    say = lambda s="": (print(s, flush=True), L.append(s))
    say("Do the hardware counters earn their keep?")
    say("=" * 78)
    say(f"{len(df)} merged configurations, {len(df.app.unique())} applications")
    say("Leave-one-application-out; error = max(pred/act, act/pred)")
    say("")

    # reference: predict a constant efficiency factor, no features at all
    const = np.full(len(y), np.nan)
    for a in sorted(df.app.unique()):
        te = (df.app == a).values
        const[te] = fac(t_an[te] / (10 ** np.median(np.log10(eta[~te]))), y[te])
    res["constant (no features)"] = const
    say(f"  {'constant (no features)':32s} median {np.nanmedian(const):.4f}   "
        f"p90 {np.nanpercentile(const,90):.4f}")
    say("")

    for label, grps in CUMULATIVE:
        cols = [c for g in grps for c in GROUPS[g] if c in Xall.columns]
        X = Xall[cols].values
        for mname, fn in [("RF", lambda: pipe(RandomForestRegressor(
                                n_estimators=400, min_samples_leaf=2,
                                random_state=0, n_jobs=-1))),
                          ("Ridge", lambda: pipe(Ridge(alpha=10.0)))]:
            r = loao(X, y, t_an, eta, df.app, fn)
            key = f"{label} [{mname}]"
            res[key] = r
            say(f"  {key:44s} n_feat={len(cols):2d}  "
                f"median {np.nanmedian(r):.4f}  p90 {np.nanpercentile(r,90):.4f}")
        say("")

    # ---- the decisive comparison ----------------------------------------
    say("--- Does adding counters beat config-only? (paired Wilcoxon) ---")
    base = res["config only (free) [RF]"]
    for key in [k for k in res if k.startswith("+") and k.endswith("[RF]")]:
        v = res[key]
        m = ~np.isnan(v) & ~np.isnan(base)
        try:
            _, p = wilcoxon(v[m], base[m])
        except ValueError:
            p = float("nan")
        d = np.nanmedian(base) - np.nanmedian(v)
        say(f"  {key:44s} delta={d:+.4f}  wins {int((v[m]<base[m]).sum()):3d}/"
            f"{int(m.sum())}  p={p:.4g}")
    say("")

    say("--- vs the no-feature constant ---")
    c = res["constant (no features)"]
    for key in [k for k in res if k != "constant (no features)"]:
        v = res[key]
        m = ~np.isnan(v) & ~np.isnan(c)
        try:
            _, p = wilcoxon(v[m], c[m])
        except ValueError:
            p = float("nan")
        tag = "better" if np.nanmedian(v) < np.nanmedian(c) else "WORSE"
        say(f"  {key:44s} p={p:.4g}  ({tag})")

    say("")
    say("--- per-application median, RF ---")
    tb = pd.DataFrame({k: v for k, v in res.items() if "Ridge" not in k})
    tb["app"] = df.app.values
    say(tb.groupby("app").median().round(3).to_string())

    tb.to_csv(OUT / "counter_value_perrow.csv", index=False)
    (OUT / "counter_value_summary.txt").write_text("\n".join(L) + "\n")


if __name__ == "__main__":
    main()