hardware-counters

#!/bin/bash
#SBATCH --job-name=sweep
#SBATCH --time=01:00:00
#SBATCH --nodes=1
#SBATCH --exclusive
#SBATCH --account=project
#SBATCH --partition=standard
#SBATCH --qos=standard
#SBATCH --output=%x-%j.out

# Single-node core-count sweep with rotating CrayPat counter sets.
# Required env: APP (stream|hpcg|gromacs), NCORE
set -u
B=/work/project/project/user
module load perftools-base perftools cray-fftw >/dev/null 2>&1

APP=${APP:?}; NCORE=${NCORE:?}
RUNDIR=$B/runs/sweep/${APP}_c${NCORE}
mkdir -p $RUNDIR && cd $RUNDIR

echo "META app=$APP ncore=$NCORE node=$(hostname) jobid=$SLURM_JOB_ID"

for SET in A B C D E; do
  export PAT_RT_PERFCTR=$($B/scripts/counter_sets.sh $SET)
  export PAT_RT_EXPDIR_BASE=$RUNDIR
  export PAT_RT_EXPDIR_NAME=exp_${SET}
  export PAT_RT_EXPDIR_REPLACE=1
  echo "=====BEGIN_SET $SET====="
  T0=$(date +%s.%N)

  case $APP in
    stream)
      export OMP_NUM_THREADS=$NCORE
      export OMP_PROC_BIND=close
      export OMP_PLACES=cores
      export OMP_DYNAMIC=false
      srun --ntasks=1 --cpus-per-task=$NCORE --hint=nomultithread \
           $B/builds/stream/stream_c.exe+pat
      ;;
    hpcg)
      export OMP_NUM_THREADS=1
      cp $B/builds/hpcg/xhpcg+pat .
      printf "HPCG benchmark input file\nsweep\n32 32 32\n20\n" > hpcg.dat
      srun --ntasks=$NCORE --cpus-per-task=1 --hint=nomultithread \
           --distribution=block:block --cpu-bind=cores ./xhpcg+pat
      ;;
    gromacs)
      export OMP_NUM_THREADS=1
      NSTEPS=$(( 8 * NCORE )); [ $NSTEPS -lt 20 ] && NSTEPS=20
      echo "NSTEPS=$NSTEPS"
      srun --ntasks=$NCORE --cpus-per-task=1 --hint=nomultithread \
           --distribution=block:block --cpu-bind=cores \
           $B/builds/gromacs/gmx_mpi+pat mdrun -notunepme -dlb yes \
           -resethway -noconfout -nsteps $NSTEPS \
           -s $B/builds/gromacs/benchRIB.tpr -g md_${SET}.log
      ;;
  esac

  RC=$?
  T1=$(date +%s.%N)
  echo "=====END_SET $SET rc=$RC wall=$(echo "$T1 - $T0" | bc)====="
done
echo "SWEEP_DONE"