hardware-counters

"""Figure: neural-network sweep — regularisation and the overfitting gap."""
import numpy as np, pandas as pd
import matplotlib; matplotlib.use("Agg")
import matplotlib.pyplot as plt

D = "/work/project/project/user"
OUT = f"{D}/analysis/out"
r = pd.read_csv(f"{OUT}/nn_sweep.csv")
nn = r[r.model.str.startswith("MLP")].copy()
nn["alpha"] = nn.model.str.extract(r"alpha=([\d.]+)").astype(float)
nn["arch"]  = nn.model.str.extract(r"MLP (\([^)]*\))")

rf = r[r.model.str.startswith("Random Forest")].iloc[0]
bl = r[r.model.str.startswith("Constant")].iloc[0]

fig, ax = plt.subplots(1, 2, figsize=(11.5, 4.3))

# --- left: test error vs regularisation, one line per architecture --------
for arch, g in nn.groupby("arch"):
    g = g.sort_values("alpha")
    ax[0].plot(g.alpha, g.test_factor, "o-", label=arch, lw=1.2, ms=4)
ax[0].axhline(rf.test_factor, c="tab:green", ls="--", lw=1.6,
              label=f"Random Forest ({rf.test_factor:.2f}x)")
ax[0].axhline(bl.test_factor, c="k", ls=":", lw=1.6,
              label=f"constant $\\eta$ ({bl.test_factor:.2f}x)")
ax[0].set(xscale="log", xlabel=r"L2 regularisation $\alpha$",
          ylabel="test error factor (leave-one-app-out)",
          title="MLPs only reach baseline-level accuracy,\nand only when heavily regularised")
ax[0].grid(alpha=.3); ax[0].legend(fontsize=7, ncol=2)

# --- right: train vs test (the overfitting gap) ---------------------------
sc = ax[1].scatter(nn.train_factor, nn.test_factor, c=np.log10(nn.alpha),
                   cmap="viridis", s=45, zorder=3)
ax[1].scatter([rf.train_factor], [rf.test_factor], marker="*", s=260,
              c="tab:green", edgecolor="k", zorder=4, label="Random Forest")
lim = [1.0, 3.0]
ax[1].plot(lim, lim, "k-", lw=1, label="no overfitting")
ax[1].set(xlim=[1.0, 1.25], ylim=[1.0, 3.0],
          xlabel="training error factor", ylabel="test error factor",
          title="Every MLP fits training data well\nbut generalises far worse")
plt.colorbar(sc, ax=ax[1], label=r"$\log_{10}\alpha$")
ax[1].grid(alpha=.3); ax[1].legend(fontsize=8, loc="upper left")

fig.tight_layout(); fig.savefig(f"{OUT}/fig_nn.png", dpi=150)
print("wrote", f"{OUT}/fig_nn.png")