hardware-counters

"""
Why do missingness indicators wreck the STREAM fold (1.096 -> 1.552)?

Hypothesis: the indicators are perfectly correlated with application identity.
OpenFOAM and GROMACS have missing counters; STREAM and HPL have none. So an
indicator column is effectively a label saying "this row is OpenFOAM", which
the network can exploit on the training applications but which carries no
usable information about a held-out application -- and for STREAM (zero
missing) the indicator is constant 0 in test while varying in train.
"""
import sys, warnings
import numpy as np, pandas as pd
warnings.filterwarnings("ignore")

D = "/work/project/project/user"
sys.path.insert(0, f"{D}/analysis")
from train_nn2 import features, load

df, X, y, t_an, eta = load(f"{D}/data/runs_persets.csv", per_set=True)
ind = [c for c in X.columns if c.endswith("__missing")]
print(f"{len(ind)} missingness indicator columns\n")

print("=== mean of each indicator per application ===")
print("(1.00 = always missing for that app, 0.00 = never)")
t = X[ind].assign(app=df.app.values).groupby("app").mean().round(2)
print(t.to_string())

print("\n=== can application identity be read off the indicators? ===")
sig = X[ind].assign(app=df.app.values).groupby("app").mean().round(1)
uniq = sig.drop_duplicates()
print(f"{len(sig)} applications produce {len(uniq)} distinct indicator "
      f"signatures.")
if len(uniq) == len(sig):
    print("-> every application has a UNIQUE signature. The indicators are")
    print("   effectively a one-hot application label, which cannot generalise")
    print("   to a held-out application and invites the network to memorise.")

print("\n=== STREAM feature ranges vs the other four ===")
feats = ["ipc", "stall_load_frac", "l3_miss_per_instr", "flops_per_cycle"]
for f in feats:
    s = X.loc[df.app == "stream", f].dropna()
    o = X.loc[df.app != "stream", f].dropna()
    if len(s) and len(o):
        inside = ((s >= o.min()) & (s <= o.max())).mean() * 100
        print(f"  {f:20s} stream median {s.median():9.4f}   "
              f"others {o.min():.4f}-{o.max():.4f}   "
              f"{inside:5.1f}% of stream rows inside training range")
print("\n-> where that percentage is low, predicting STREAM is extrapolation,")
print("   not interpolation. Tree models clamp to the training range;")
print("   neural networks extrapolate linearly and can go badly wrong.")