hardware-counters

#!/usr/bin/env python3
"""
Final neural-network configuration, after ablating each proposed fix.

Starting point: the first MLP attempt was significantly worse than Random
Forest (1.176x vs 1.146x) and no better than a no-counter baseline.

Five candidate problems were identified and tested individually. The ablation
(ablation.py) showed that only some helped, and one actively hurt:

  H1  Dataset size -- KEPT, and by far the biggest win.
      Each configuration was collapsed into one row, discarding four of the
      five counter-set runs. Each set is an independent execution with its own
      Thread Time, so the usable dataset is 147 rows, not 40. No extra compute.

  H2  Feature skew (signed log1p) -- KEPT, small but consistent gain
      (1.072 -> 1.069).

  H3  Missingness indicators -- REJECTED. Intended to let the model tell
      "counter absent" from "counter average". In fact each application has a
      unique indicator signature (5 applications, 5 distinct signatures), so
      the columns act as a one-hot application label: harmless in training,
      useless for a held-out application, and they wrecked the STREAM fold
      (1.096 -> 1.552). Removed.

  H4  Bagging -- REJECTED as a default. Slightly worse overall
      (1.069 -> 1.077) and it did not rescue the STREAM fold. Kept available
      via --bag for reference.

  H5  lbfgs solver -- REJECTED on its own (1.157 -> 1.173 with alpha=10), but
      lbfgs plus the corrected alpha is the best combination, so the real
      finding is that the ORIGINAL alpha=10 was over-regularised. Lowering
      alpha to 1.0 was the single largest modelling gain (1.173 -> 1.072).

Remaining known weakness: STREAM. Its stall_load_frac and l3_miss_per_instr
lie entirely outside the range spanned by the other four applications (0% of
rows inside), so predicting it is extrapolation rather than interpolation.
Tree ensembles clamp to the training range and degrade gracefully; neural
networks extrapolate linearly and can diverge. This is a property of the
data, not of the tuning, and is reported as such.
"""
import argparse, warnings, sys
from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor, BaggingRegressor
from sklearn.dummy import DummyRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, FunctionTransformer
from sklearn.impute import SimpleImputer
from scipy.stats import wilcoxon

warnings.filterwarnings("ignore")
D      = "/work/project/project/user"
OUT    = Path(f"{D}/analysis/out"); OUT.mkdir(parents=True, exist_ok=True)
F_PEAK = 2.25e9
SEEDS  = (0, 1, 2, 3, 4)
sys.path.insert(0, f"{D}/analysis")
from train_nn2 import features, signed_log1p, factors, load


def nn(hidden=(32, 16), alpha=1.0, bag=1):
    def fn(seed):
        base = MLPRegressor(hidden_layer_sizes=hidden, alpha=alpha,
                            solver="lbfgs", max_iter=5000, random_state=seed)
        est = base if bag == 1 else BaggingRegressor(
            estimator=base, n_estimators=bag, max_samples=0.8,
            random_state=seed, n_jobs=-1)
        return Pipeline([("impute", SimpleImputer(strategy="median")),
                         ("log",    FunctionTransformer(signed_log1p, validate=False)),
                         ("scale",  StandardScaler()),
                         ("est",    est)])
    return fn


def plain(est_fn):
    return lambda s: Pipeline([("impute", SimpleImputer(strategy="median")),
                               ("scale",  StandardScaler()),
                               ("est",    est_fn(s))])


def loao(fn, X, y, t_an, eta, groups, seeds=SEEDS):
    acc = np.zeros(len(y))
    for s in seeds:
        for app in sorted(groups.unique()):
            te = (groups == app).values
            m = fn(s); m.fit(X[~te], np.log10(eta[~te]))
            p = t_an[te] / (10 ** m.predict(X[te]))
            acc[te] += factors(p, y[te])
    return acc / len(seeds)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--bag", type=int, default=1)
    a = ap.parse_args()

    df, Xall, y, t_an, eta = load(f"{D}/data/runs_persets.csv", per_set=True)
    # H3 rejected: drop the missingness indicators
    X = Xall[[c for c in Xall.columns if not c.endswith("__missing")]]
    g = df.app

    L = []
    say = lambda s="": (print(s), L.append(s))
    say("Final neural-network configuration")
    say("=" * 78)
    say(f"{len(df)} samples (per-counter-set), {X.shape[1]} features, "
        f"{g.nunique()} applications")
    say("Leave-one-application-out; median of 5 seeds; error = multiplicative factor")
    say("")

    models = {
        "MLP final (32,16) a=1.0":  nn((32, 16), 1.0, a.bag),
        "MLP original recipe":      lambda s: Pipeline([
            ("i", SimpleImputer(strategy="median")), ("sc", StandardScaler()),
            ("e", MLPRegressor(hidden_layer_sizes=(32, 16, 8), alpha=10.0,
                               max_iter=20000, random_state=s))]),
        "Random Forest":            plain(lambda s: RandomForestRegressor(
            n_estimators=500, min_samples_leaf=2, random_state=s, n_jobs=-1)),
        "Constant baseline":        plain(lambda s: DummyRegressor(strategy="mean")),
    }

    res = {}
    for name, fn in models.items():
        seeds = (0,) if "Forest" in name or "Constant" in name else SEEDS
        res[name] = loao(fn, X, y, t_an, eta, g, seeds)

    say("Overall (median error factor):")
    for k, v in sorted(res.items(), key=lambda kv: np.median(kv[1])):
        say(f"  {k:28s} {np.median(v):.3f}")
    say("")

    say("Per-application median error factor:")
    tb = pd.DataFrame({k: v for k, v in res.items()}).assign(app=g.values) \
           .groupby("app").median().round(3)
    say(tb.to_string())
    say("")

    say("Paired significance (Wilcoxon, n=%d):" % len(y))
    fin = res["MLP final (32,16) a=1.0"]
    for other in ["MLP original recipe", "Constant baseline", "Random Forest"]:
        o = res[other]
        try:
            _, p = wilcoxon(fin, o)
        except ValueError:
            p = float("nan")
        verdict = ("MLP better" if np.median(fin) < np.median(o) else "MLP worse")
        sig = "significant" if p < 0.05 else "not significant"
        say(f"  vs {other:24s} wins {int((fin<o).sum()):3d}/{len(fin)}  "
            f"p={p:.4f}  ({verdict}, {sig})")
    say("")

    # excluding STREAM, which is an extrapolation fold for every model
    m = (g != "stream").values
    say("Excluding STREAM (extrapolation fold; 0% of its rows lie inside the")
    say("training range for stall_load_frac and l3_miss_per_instr):")
    for k, v in sorted(res.items(), key=lambda kv: np.median(kv[1][m])):
        say(f"  {k:28s} {np.median(v[m]):.3f}")

    txt = "\n".join(L)
    (OUT / "nn_final_summary.txt").write_text(txt + "\n")
    pd.DataFrame({**{k: v for k, v in res.items()},
                  "app": g.values, "ncore": df.ncore.values,
                  "cset": df.cset.values}).to_csv(OUT / "nn_final_perrow.csv",
                                                  index=False)


if __name__ == "__main__":
    main()