hardware-counters

#!/usr/bin/env python3
"""
Parse the Cirrus WEAK-SCALING sweep.

Directory names encode the per-rank work unit (base) rather than the global
size, so both are recorded: `base` is the independent variable that was varied,
`size` is what weak_sizes.py expanded it to for that rank count.
"""
import re, subprocess, csv, sys, os
from pathlib import Path

CPROOT = "/opt/cray/pe/perftools/25.03.0"
PR     = f"{CPROOT}/bin/pat_report"
# Data and output roots. Overridable so the pipeline can be relocated
# (a different account, a scratch copy, another site) without editing code;
# the default keeps existing invocations working unchanged.
D      = os.environ.get("DISS_ROOT", "/work/project/project/user")
BASE   = Path(f"{D}/runs/weak")
OUT    = Path(f"{D}/data")

VAL  = re.compile(r"^  (?P<n>[A-Z][A-Za-z0-9_:]*)\s+"
                  r"(?:(?:[\d.]+)\s*(?:[GMK]/sec|W)\s+)?"
                  r"(?P<v>\d[\d,]*(?:\.\d+)?)\s*"
                  r"(?:ops|instr|cycles|J|refs|misses|hits)?\s*$")
TIME = re.compile(r"^  Thread Time\s+([\d.]+)\s+secs")
env  = dict(os.environ, CRAYPAT_ROOT=CPROOT)


def parse(exp):
    try:
        out = subprocess.run([PR, "-O", "hwpc", str(exp)], env=env,
                             capture_output=True, text=True, timeout=600).stdout
    except Exception:
        return None, {}
    vals, t = {}, None
    for line in out.splitlines():
        if "PAT_RT_PERFCTR" in line:
            break
        mt = TIME.match(line)
        if mt:
            t = float(mt.group(1)); continue
        m = VAL.match(line)
        if m and m.group("n") not in ("Thread", "Total", "Average", "CrayPat"):
            try:
                vals[m.group("n")] = float(m.group("v").replace(",", ""))
            except ValueError:
                pass
    return t, vals


# recover the expanded global size from the job's META line
def meta_size(app, nc, base, nt):
    for f in BASE.glob(f"w_{app}_c{nc}_b{base}_t{nt}-*.out"):
        for line in f.read_text(errors="replace").splitlines():
            if line.startswith("META "):
                m = re.search(r"size=(\d+)", line)
                if m:
                    return int(m.group(1))
    return 0


rows = []
for d in sorted(BASE.glob("*_c*_b*_t*")):
    m = re.match(r"([a-z]+)_c(\d+)_b(\d+)_t(\d+)$", d.name)
    if not m or not d.is_dir():
        continue
    app, nc, base, nt = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4))
    size = meta_size(app, nc, base, nt)
    for s in "ABCDE":
        exp = d / f"exp_{s}"
        if not exp.exists():
            continue
        t, vals = parse(exp)
        if t is None or not vals:
            continue
        rows.append({"platform": "cirrus", "scaling": "weak", "app": app,
                     "ncore": nc, "base": base, "size": size, "nthread": nt,
                     "nrank": nc // max(nt, 1), "cset": s,
                     "runtime_s": t, **vals})
    print(d.name, file=sys.stderr)

lead = ["platform", "scaling", "app", "ncore", "nthread", "nrank", "base",
        "size", "cset", "runtime_s"]
keys = lead + sorted({k for r in rows for k in r} - set(lead))
OUT.mkdir(parents=True, exist_ok=True)
with open(OUT / "cirrus_weak.csv", "w", newline="") as fh:
    w = csv.DictWriter(fh, fieldnames=keys); w.writeheader(); w.writerows(rows)

print(f"\nwrote cirrus_weak.csv: {len(rows)} rows x {len(keys)} cols")
from collections import Counter
print("by app:", dict(Counter(r["app"] for r in rows)))
rt = [r["runtime_s"] for r in rows]
short = sum(1 for x in rt if x < 1.0)
print(f"runtime: min {min(rt):.2f}s  median {sorted(rt)[len(rt)//2]:.2f}s  "
      f"max {max(rt):.2f}s   under 1s: {100*short/len(rt):.1f}%")