hardware-counters

#!/bin/bash
#SBATCH --job-name=csweep
#SBATCH --nodes=1
#SBATCH --exclusive
#SBATCH --account=project
#SBATCH --partition=standard
#SBATCH --qos=standard
#SBATCH --output=%x-%j.out

# Cirrus single-node sweep, mirroring the ARCHER2 run_expand.slurm so the two
# platforms produce directly comparable data.
#
# Env: APP, NCORE, SIZE, NTHREAD (default 1)
set -u
B=/work/project/project/user
source "$B/scripts/cirrus_env.sh"          # module init - see cirrus_env.sh

APP=${APP:?}; NCORE=${NCORE:?}; SIZE=${SIZE:?}; NTHREAD=${NTHREAD:-1}
NRANK=$(( NCORE / NTHREAD ))
[ "$NRANK" -lt 1 ] && { echo "bad decomposition"; exit 1; }

RUNDIR=$B/runs/sweep/${APP}_c${NCORE}_s${SIZE}_t${NTHREAD}
rm -rf "$RUNDIR"; mkdir -p "$RUNDIR"; cd "$RUNDIR" || exit 1
echo "META platform=cirrus app=$APP ncore=$NCORE size=$SIZE nthread=$NTHREAD nrank=$NRANK node=$(hostname) jobid=$SLURM_JOB_ID"

export OMP_NUM_THREADS=$NTHREAD
export OMP_PROC_BIND=close
export OMP_PLACES=cores

case $APP in
  hpcg)
    printf "HPCG\nsweep\n%d %d %d\n30\n" "$SIZE" "$SIZE" "$SIZE" > hpcg.dat
    cp -f "$B/builds/hpcg/xhpcg+pat" . ;;
  hpl)
    read -r P Q < <(python3 "$B/scripts/pq.py" "$NRANK")
    echo "HPL grid P=$P Q=$Q N=$SIZE"
    { echo "HPLinpack"; echo "ICL"; echo "HPL.out"; echo "6"
      echo "1"; echo "$SIZE"; echo "1"; echo "192"; echo "0"
      echo "1"; echo "$P"; echo "$Q"; echo "16.0"
      echo "1"; echo "2"; echo "1"; echo "4"; echo "1"; echo "2"
      echo "1"; echo "1"; echo "1"; echo "1"; echo "1"; echo "1"
      echo "2"; echo "64"; echo "0"; echo "0"; echo "1"; echo "8"; } > HPL.dat
    cp -f "$B/builds/hpl/xhpl+pat" . ;;
esac

for SET in A B C D E; do
    PAT_RT_PERFCTR=$(bash "$B/scripts/counter_sets.sh" "$SET"); export PAT_RT_PERFCTR
    export PAT_RT_EXPDIR_BASE=$RUNDIR
    export PAT_RT_EXPDIR_NAME=exp_${SET}
    export PAT_RT_EXPDIR_REPLACE=1
    echo "=====BEGIN_SET $SET====="
    T0=$(date +%s.%N)
    SR="srun --ntasks=$NRANK --cpus-per-task=$NTHREAD --hint=nomultithread --distribution=block:block --cpu-bind=cores"

    case $APP in
      stream)  $SR --ntasks=1 --cpus-per-task="$NCORE" \
                  "$B/builds/stream/stream_${SIZE}+pat" > "run_$SET.log" 2>&1 ;;
      hpcg)    $SR ./xhpcg+pat                          > "run_$SET.log" 2>&1 ;;
      hpl)     $SR ./xhpl+pat                           > "run_$SET.log" 2>&1 ;;
      minife)  $SR "$B/builds/minife/minife.x+pat" nx=$SIZE ny=$SIZE nz=$SIZE \
                                                        > "run_$SET.log" 2>&1 ;;
      lulesh)  $SR "$B/builds/lulesh/lulesh2.0+pat" -s "$SIZE" -i 100 \
                                                        > "run_$SET.log" 2>&1 ;;
      comd)    read -r I J K < <(python3 "$B/scripts/pq3.py" "$NRANK")
               $SR "$B/builds/comd/CoMD-mpi+pat" -x "$SIZE" -y "$SIZE" -z "$SIZE" \
                   -N 100 -i "$I" -j "$J" -k "$K"        > "run_$SET.log" 2>&1 ;;
    esac

    RC=$?; T1=$(date +%s.%N)
    echo "=====END_SET $SET rc=$RC wall=$(echo "$T1 - $T0" | bc)====="
done
echo "SWEEP_DONE"