hardware-counters

"""
Is the best MLP actually different from Random Forest, or is the 1.176 vs
1.146 gap noise? Paired per-configuration comparison, same protocol.
"""
import warnings, sys
import numpy as np, pandas as pd
from scipy.stats import wilcoxon
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.dummy import DummyRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer

warnings.filterwarnings("ignore")
D = "/work/project/project/user"
sys.path.insert(0, f"{D}/analysis")
from train_model import build_features
F_PEAK = 2.25e9

df = pd.read_csv(f"{D}/data/runs.csv")
df = df[(df.app != "cp2k") & df.runtime_s.notna()].reset_index(drop=True)
wc = [c for c in ["wall_B","wall_C","wall_D","wall_E"] if c in df]
y = df[wc].median(axis=1).fillna(df.runtime_s).values
cyc = pd.to_numeric(df.PAPI_TOT_CYC, errors="coerce").values
t_an = cyc / F_PEAK
eta = t_an / y
X, groups = build_features(df), df.app

def pipe(e):
    return Pipeline([("i", SimpleImputer(strategy="median")),
                     ("s", StandardScaler()), ("e", e)])

def per_config(fn, seeds=(0,1,2,3,4)):
    """Return per-configuration error factor, averaged over seeds."""
    acc = np.zeros(len(df))
    for s in seeds:
        for app in sorted(groups.unique()):
            te = (groups == app).values
            m = fn(s); m.fit(X[~te], np.log10(eta[~te]))
            p = t_an[te] / (10 ** m.predict(X[te]))
            acc[te] += np.maximum(p/y[te], y[te]/p)
    return acc / len(seeds)

rf  = per_config(lambda s: pipe(RandomForestRegressor(
        n_estimators=500, min_samples_leaf=2, random_state=s, n_jobs=-1)))
nn  = per_config(lambda s: pipe(MLPRegressor(hidden_layer_sizes=(32,16,8),
        alpha=10.0, max_iter=20000, random_state=s)))
bl  = per_config(lambda s: pipe(DummyRegressor(strategy="mean")), seeds=(0,))

print("Paired comparison over 40 configurations (mean of 5 seeds each)")
print()
print(f"{'':28s} {'median':>8s} {'mean':>8s}")
for n, v in [("Random Forest", rf), ("MLP (32,16,8) alpha=10", nn),
             ("Constant efficiency", bl)]:
    print(f"{n:28s} {np.median(v):8.3f} {np.mean(v):8.3f}")

print()
st, p = wilcoxon(nn, rf)
print(f"MLP vs Random Forest:      MLP wins {int((nn<rf).sum())}/40,  p = {p:.4f}")
st, p = wilcoxon(nn, bl)
print(f"MLP vs constant baseline:  MLP wins {int((nn<bl).sum())}/40,  p = {p:.4f}")
st, p = wilcoxon(rf, bl)
print(f"RF  vs constant baseline:  RF  wins {int((rf<bl).sum())}/40,  p = {p:.4f}")

print()
print("Per-application median error factor:")
out = pd.DataFrame({"app": groups, "RandomForest": rf, "MLP": nn, "baseline": bl})
print(out.groupby("app").median().round(3).to_string())