hardware-counters

"""
Diagnose the specific weaknesses of the MLP setup, beyond "not enough data".

Candidate holes:
  H1  target scale     - is log10(eta) well-conditioned for a NN?
  H2  feature skew     - heavy-tailed ratios hurt gradient descent badly
  H3  imputation       - median-fill on structurally-missing counters injects
                         values that are wrong in a systematic, per-app way
  H4  fold imbalance   - each LOAO fold has only 32 train samples
  H5  no early stopping / no validation split inside the fold
"""
import sys, warnings
import numpy as np, pandas as pd
warnings.filterwarnings("ignore")

D = "/work/project/project/user"
sys.path.insert(0, f"{D}/analysis")
from train_model import build_features
F_PEAK = 2.25e9

df = pd.read_csv(f"{D}/data/runs.csv")
df = df[(df.app != "cp2k") & df.runtime_s.notna()].reset_index(drop=True)
wc = [c for c in ["wall_B","wall_C","wall_D","wall_E"] if c in df]
y = df[wc].median(axis=1).fillna(df.runtime_s).values
cyc = pd.to_numeric(df.PAPI_TOT_CYC, errors="coerce").values
eta = (cyc / F_PEAK) / y
X = build_features(df)

print("H1  TARGET  log10(eta)")
t = np.log10(eta)
print(f"    range {t.min():+.3f} to {t.max():+.3f}, sd {t.std():.3f}")
print(f"    -> spread is only {t.max()-t.min():.2f} decades; a NN initialised")
print("       near zero output already sits close to the mean, so gradients")
print("       are small and it takes heavy training to beat a constant.\n")

print("H2  FEATURE SKEW (|skew| > 2 is problematic for gradient descent)")
sk = X.skew(numeric_only=True).abs().sort_values(ascending=False)
for k, v in sk.head(8).items():
    rng = X[k].max() / max(X[k].min(), 1e-12)
    print(f"    {k:22s} skew {v:6.2f}   max/min = {rng:.3g}")
print(f"    {(sk>2).sum()} of {len(sk)} features are heavily skewed.")
print("    -> StandardScaler centres these but does NOT fix the tail; a few")
print("       configs dominate the loss. log1p first, then scale.\n")

print("H3  MISSINGNESS is structural, not random")
m = X.isna().sum()
print(f"    {int(m.sum())} missing cells over {X.shape[0]}x{X.shape[1]}")
per_app = X.assign(app=df.app).groupby("app").apply(lambda g: g.isna().sum().sum())
print("    missing cells per application:")
for k, v in per_app.items():
    print(f"      {k:10s} {v}")
print("    -> median imputation fills with the OTHER apps' median, which is")
print("       systematically wrong for the held-out app. Add missingness")
print("       indicator columns so the model can tell 'absent' from 'typical'.\n")

print("H4  FOLD SIZE")
for a in sorted(df.app.unique()):
    print(f"    hold out {a:10s}: train n={int((df.app!=a).sum())}, test n={int((df.app==a).sum())}")
print("    -> 32 training rows. Any net with >150 params is over-parameterised.\n")

print("H5  TRAINING PROTOCOL")
print("    current: solver=adam, max_iter=20000, early_stopping=False")
print("    -> no validation split, so training runs to convergence on 32")
print("       samples every time. early_stopping needs a val split that is")
print("       itself tiny; better to use lbfgs (deterministic, suits small n)")
print("       and rely on L2 + ensembling over seeds.")