hardware-counters

#!/usr/bin/env python3
"""
Ablation: which of the five fixes actually helped, and why does STREAM regress?

Reporting "the improved MLP is better" without isolating the contributions
would leave the dissertation unable to say *what* mattered. Each fix is added
cumulatively and separately so its individual effect is visible.
"""
import warnings, sys
import numpy as np, pandas as pd
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor, BaggingRegressor
from sklearn.dummy import DummyRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, FunctionTransformer
from sklearn.impute import SimpleImputer

warnings.filterwarnings("ignore")
D = "/work/project/project/user"
sys.path.insert(0, f"{D}/analysis")
from train_nn2 import features, signed_log1p, factors, load, F_PEAK
SEEDS = (0, 1, 2, 3, 4)


def run(fn, X, y, t_an, eta, groups, seeds=SEEDS):
    acc = np.zeros(len(y))
    for s in seeds:
        for app in sorted(groups.unique()):
            te = (groups == app).values
            m = fn(s); m.fit(X[~te], np.log10(eta[~te]))
            p = t_an[te] / (10 ** m.predict(X[te]))
            acc[te] += factors(p, y[te])
    return acc / len(seeds)


def build(hidden=(32, 16), alpha=1.0, log=True, bag=1, solver="lbfgs"):
    def fn(seed):
        base = MLPRegressor(hidden_layer_sizes=hidden, alpha=alpha,
                            solver=solver, max_iter=5000 if solver == "lbfgs"
                            else 20000, random_state=seed)
        est = base if bag == 1 else BaggingRegressor(
            estimator=base, n_estimators=bag, max_samples=0.8,
            random_state=seed, n_jobs=-1)
        steps = [("impute", SimpleImputer(strategy="median"))]
        if log:
            steps.append(("log", FunctionTransformer(signed_log1p, validate=False)))
        steps += [("scale", StandardScaler()), ("est", est)]
        return Pipeline(steps)
    return fn


df, X, y, t_an, eta = load(f"{D}/data/runs_persets.csv", per_set=True)
g = df.app
# feature matrix WITHOUT the missingness indicators, for the H3 ablation
Xn = X[[c for c in X.columns if not c.endswith("__missing")]]

print(f"per-set dataset: {len(df)} rows, {X.shape[1]} features "
      f"({Xn.shape[1]} without missingness indicators)\n")

configs = [
    ("original recipe (adam, no log, no bag, no indic.)",
     build(alpha=10.0, log=False, bag=1, solver="adam"), Xn),
    ("+ lbfgs solver (H5)",
     build(alpha=10.0, log=False, bag=1), Xn),
    ("+ alpha 10 -> 1",
     build(alpha=1.0, log=False, bag=1), Xn),
    ("+ log1p features (H2)",
     build(alpha=1.0, log=True, bag=1), Xn),
    ("+ missingness indicators (H3)",
     build(alpha=1.0, log=True, bag=1), X),
    ("+ bagging x10 (H4)",
     build(alpha=1.0, log=True, bag=10), X),
]

rows = []
for name, fn, Xu in configs:
    r = run(fn, Xu, y, t_an, eta, g)
    per_app = pd.Series(r).groupby(g.values).median()
    rows.append({"config": name, "overall": np.median(r), **per_app.round(3)})
    print(f"{name:52s} {np.median(r):.3f}")

ref = {}
for name, fn in [("Random Forest",
                  lambda s: Pipeline([("i", SimpleImputer(strategy="median")),
                                      ("sc", StandardScaler()),
                                      ("e", RandomForestRegressor(
                                          n_estimators=500, min_samples_leaf=2,
                                          random_state=s, n_jobs=-1))])),
                 ("constant baseline",
                  lambda s: Pipeline([("i", SimpleImputer(strategy="median")),
                                      ("sc", StandardScaler()),
                                      ("e", DummyRegressor(strategy="mean"))]))]:
    r = run(fn, X, y, t_an, eta, g, seeds=(0,))
    per_app = pd.Series(r).groupby(g.values).median()
    rows.append({"config": name, "overall": np.median(r), **per_app.round(3)})
    ref[name] = r
    print(f"{name:52s} {np.median(r):.3f}")

tb = pd.DataFrame(rows).set_index("config").round(3)
print("\n=== per-application breakdown ===")
print(tb.to_string())
tb.to_csv(f"{D}/analysis/out/nn_ablation.csv")

print("\n=== why does STREAM regress? ===")
print("STREAM has the most extreme feature values (lowest IPC, highest")
print("stall fractions). Check whether it sits outside the training range:")
feats = ["ipc", "stall_load_frac", "l3_miss_per_instr", "core_energy_frac"]
z = X[feats].assign(app=g.values).groupby("app").median()
print(z.round(4).to_string())