hardware-counters

#!/bin/bash
#SBATCH --job-name=sweep2
#SBATCH --nodes=1
#SBATCH --exclusive
#SBATCH --account=project
#SBATCH --partition=standard
#SBATCH --qos=standard
#SBATCH --output=%x-%j.out

# Extended single-node core sweep: hpl, cp2k. Reuses the 5-set counter rotation.
set -u
B=/work/project/project/user
module load perftools-base perftools >/dev/null 2>&1

APP=${APP:?}
NCORE=${NCORE:?}
RUNDIR=$B/runs/sweep/${APP}_c${NCORE}
mkdir -p "$RUNDIR" && cd "$RUNDIR"
echo "META app=$APP ncore=$NCORE node=$(hostname) jobid=$SLURM_JOB_ID"

CP2KROOT=/work/y07/shared/apps/core/cp2k/cp2k-2023.1

# ---- one-time per-app setup ------------------------------------------------
if [ "$APP" = "hpl" ]; then
    set -- $(python3 $B/scripts/pq.py $NCORE)
    P=$1; Q=$2
    # keep memory per rank modest: N ~ sqrt(ncore * 1GB / 8) rounded to NB
    N=$(python3 -c "import math;print(max(5000,int(math.sqrt($NCORE*1.25e8/8)/192)*192))")
    echo "HPL grid P=$P Q=$Q N=$N"
    {
      echo "HPLinpack benchmark input file"
      echo "ICL, University of Tennessee"
      echo "HPL.out      output file name"
      echo "6            device out"
      echo "1            # of problems sizes (N)"
      echo "$N           Ns"
      echo "1            # of NBs"
      echo "192          NBs"
      echo "0            PMAP process mapping"
      echo "1            # of process grids (P x Q)"
      echo "$P           Ps"
      echo "$Q           Qs"
      echo "16.0         threshold"
      echo "1            # of panel fact"
      echo "2            PFACTs"
      echo "1            # of recursive stopping criterium"
      echo "4            NBMINs"
      echo "1            # of panels in recursion"
      echo "2            NDIVs"
      echo "1            # of recursive panel fact."
      echo "1            RFACTs"
      echo "1            # of broadcast"
      echo "1            BCASTs"
      echo "1            # of lookahead depth"
      echo "1            DEPTHs"
      echo "2            SWAP"
      echo "64           swapping threshold"
      echo "0            L1 in transposed form"
      echo "0            U  in transposed form"
      echo "1            Equilibration"
      echo "8            memory alignment in double"
    } > HPL.dat
    cp $B/builds/hpl/xhpl+pat .
elif [ "$APP" = "cp2k" ]; then
    cp $CP2KROOT/benchmarks/QS/H2O-64.inp .
    export CP2K_DATA_DIR=$CP2KROOT/data
fi

# ---- counter rotation ------------------------------------------------------
for SET in A B C D E; do
    PAT_RT_PERFCTR=$($B/scripts/counter_sets.sh $SET)
    export PAT_RT_PERFCTR
    export PAT_RT_EXPDIR_BASE=$RUNDIR
    export PAT_RT_EXPDIR_NAME=exp_${SET}
    export PAT_RT_EXPDIR_REPLACE=1
    export OMP_NUM_THREADS=1
    echo "=====BEGIN_SET $SET====="
    T0=$(date +%s.%N)

    if [ "$APP" = "hpl" ]; then
        srun --ntasks=$NCORE --cpus-per-task=1 --hint=nomultithread \
             --distribution=block:block --cpu-bind=cores ./xhpl+pat
        RC=$?
    elif [ "$APP" = "cp2k" ]; then
        export CP2K_DATA_DIR=$CP2KROOT/data
        srun --ntasks=$NCORE --cpus-per-task=1 --hint=nomultithread \
             --distribution=block:block --cpu-bind=cores \
             pat_run -w -- $CP2KROOT/exe/ARCHER2/cp2k.psmp -i H2O-64.inp -o cp2k_${SET}.out
        RC=$?
    else
        echo "unknown APP=$APP"; RC=1
    fi

    T1=$(date +%s.%N)
    echo "=====END_SET $SET rc=$RC wall=$(echo "$T1 - $T0" | bc)====="
done
echo "SWEEP_DONE"