hardware-counters

#!/usr/bin/env python3
"""
Parse the Cirrus sweep into the same schema as the ARCHER2 dataset.

One row per (app, size, ncore, nthread, cset). Each counter set is an
independent timed execution with its own Thread Time and PAPI_TOT_CYC, so it
contributes a row; they are merged into per-configuration rows later.

Cirrus differences handled here:
  * pat_report REQUIRES CRAYPAT_ROOT to be set, unlike ARCHER2.
  * No cray_rapl / cray_zenl3, so PACKAGE_ENERGY, PP0_ENERGY and UNC_L3_* are
    simply absent rather than missing-at-random.
"""
import re, subprocess, csv, sys, os
from pathlib import Path

CPROOT = "/opt/cray/pe/perftools/25.03.0"
PR     = f"{CPROOT}/bin/pat_report"
# Data and output roots. Overridable so the pipeline can be relocated
# (a different account, a scratch copy, another site) without editing code;
# the default keeps existing invocations working unchanged.
D      = os.environ.get("DISS_ROOT", "/work/project/project/user")
BASE   = Path(f"{D}/runs/sweep")
OUT    = Path(f"{D}/data")

VAL  = re.compile(r"^  (?P<n>[A-Z][A-Za-z0-9_:]*)\s+"
                  r"(?:(?:[\d.]+)\s*(?:[GMK]/sec|W)\s+)?"
                  r"(?P<v>\d[\d,]*(?:\.\d+)?)\s*"
                  r"(?:ops|instr|cycles|J|refs|misses|hits)?\s*$")
TIME = re.compile(r"^  Thread Time\s+([\d.]+)\s+secs")

env = dict(os.environ, CRAYPAT_ROOT=CPROOT)


def parse(exp):
    try:
        out = subprocess.run([PR, "-O", "hwpc", str(exp)], env=env,
                             capture_output=True, text=True, timeout=600).stdout
    except Exception:
        return None, {}
    vals, t = {}, None
    for line in out.splitlines():
        if "PAT_RT_PERFCTR" in line:
            break
        mt = TIME.match(line)
        if mt:
            t = float(mt.group(1)); continue
        m = VAL.match(line)
        if m and m.group("n") not in ("Thread", "Total", "Average", "CrayPat"):
            try:
                vals[m.group("n")] = float(m.group("v").replace(",", ""))
            except ValueError:
                pass
    return t, vals


rows = []
for d in sorted(BASE.glob("*_c*_s*_t*")):
    m = re.match(r"([a-z]+)_c(\d+)_s(\d+)_t(\d+)$", d.name)
    if not m or not d.is_dir():
        continue
    app, nc, size, nt = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4))
    for s in "ABCDE":
        exp = d / f"exp_{s}"
        if not exp.exists():
            continue
        t, vals = parse(exp)
        if t is None or not vals:
            continue
        rows.append({"platform": "cirrus", "app": app, "ncore": nc, "size": size,
                     "nthread": nt, "nrank": nc // max(nt, 1), "cset": s,
                     "runtime_s": t, **vals})
    print(f"{d.name}", file=sys.stderr)

lead = ["platform", "app", "ncore", "nthread", "nrank", "size", "cset", "runtime_s"]
keys = lead + sorted({k for r in rows for k in r} - set(lead))
OUT.mkdir(parents=True, exist_ok=True)
with open(OUT / "cirrus_persets.csv", "w", newline="") as fh:
    w = csv.DictWriter(fh, fieldnames=keys); w.writeheader(); w.writerows(rows)

print(f"\nwrote cirrus_persets.csv: {len(rows)} rows x {len(keys)} cols")
from collections import Counter
print("by app:", dict(Counter(r["app"] for r in rows)))
print("counters seen:", sorted(k for k in keys if k[0].isupper()))