hardware-counters

#!/bin/bash
#SBATCH --job-name=cweak
#SBATCH --nodes=1
#SBATCH --exclusive
#SBATCH --account=project
#SBATCH --partition=standard
#SBATCH --qos=standard
#SBATCH --output=%x-%j.out

# Cirrus WEAK-SCALING sweep: problem size grows with rank count so work per
# rank stays roughly constant and runtime stays in a useful window.
#
# The earlier fixed-size sweep held sizes identical to ARCHER2 for
# comparability, which handicapped the 288-core machine: miniFE was 90.5%
# sub-second, HPL 65%, CoMD 64%. Below ~1 s fixed overheads dominate and the
# efficiency factor collapses, so no model can beat a constant. This sweep
# tests whether Cirrus is intrinsically hard to predict or whether that was an
# artefact of the size choice.
#
# Env: APP, NCORE, BASE, NTHREAD (default 1)
set -u
B=/work/project/project/user
source "$B/scripts/cirrus_env.sh"

APP=${APP:?}; NCORE=${NCORE:?}; BASE=${BASE:?}; NTHREAD=${NTHREAD:-1}
NRANK=$(( NCORE / NTHREAD ))
[ "$NRANK" -lt 1 ] && { echo "bad decomposition"; exit 1; }

SIZE=$(python3 "$B/scripts/weak_sizes.py" "$APP" "$NRANK" "$BASE")

RUNDIR=$B/runs/weak/${APP}_c${NCORE}_b${BASE}_t${NTHREAD}
rm -rf "$RUNDIR"; mkdir -p "$RUNDIR"; cd "$RUNDIR" || exit 1
echo "META platform=cirrus scaling=weak app=$APP ncore=$NCORE base=$BASE size=$SIZE nthread=$NTHREAD nrank=$NRANK node=$(hostname) jobid=$SLURM_JOB_ID"

export OMP_NUM_THREADS=$NTHREAD
export OMP_PROC_BIND=close
export OMP_PLACES=cores

case $APP in
  hpcg)
    printf "HPCG\nweak\n%d %d %d\n30\n" "$SIZE" "$SIZE" "$SIZE" > hpcg.dat
    cp -f "$B/builds/hpcg/xhpcg+pat" . ;;
  hpl)
    read -r P Q < <(python3 "$B/scripts/pq.py" "$NRANK")
    echo "HPL grid P=$P Q=$Q N=$SIZE"
    { echo "HPLinpack"; echo "ICL"; echo "HPL.out"; echo "6"
      echo "1"; echo "$SIZE"; echo "1"; echo "192"; echo "0"
      echo "1"; echo "$P"; echo "$Q"; echo "16.0"
      echo "1"; echo "2"; echo "1"; echo "4"; echo "1"; echo "2"
      echo "1"; echo "1"; echo "1"; echo "1"; echo "1"; echo "1"
      echo "2"; echo "64"; echo "0"; echo "0"; echo "1"; echo "8"; } > HPL.dat
    cp -f "$B/builds/hpl/xhpl+pat" . ;;
esac

for SET in A B C D E; do
    PAT_RT_PERFCTR=$(bash "$B/scripts/counter_sets.sh" "$SET"); export PAT_RT_PERFCTR
    export PAT_RT_EXPDIR_BASE=$RUNDIR PAT_RT_EXPDIR_NAME=exp_${SET} PAT_RT_EXPDIR_REPLACE=1
    echo "=====BEGIN_SET $SET====="
    T0=$(date +%s.%N)
    SR="srun --ntasks=$NRANK --cpus-per-task=$NTHREAD --hint=nomultithread --distribution=block:block --cpu-bind=cores"

    case $APP in
      stream)
        # STREAM binaries are prebuilt per array length; pick the nearest
        # available size rather than rebuilding inside the job.
        BEST=10000000
        for cand in 10000000 40000000 80000000; do
            [ "$cand" -le "$SIZE" ] && BEST=$cand
        done
        echo "stream using prebuilt size $BEST (requested $SIZE)"
        $SR --ntasks=1 --cpus-per-task="$NCORE" \
            "$B/builds/stream/stream_${BEST}+pat" > "run_$SET.log" 2>&1 ;;
      hpcg)    $SR ./xhpcg+pat  > "run_$SET.log" 2>&1 ;;
      hpl)     $SR ./xhpl+pat   > "run_$SET.log" 2>&1 ;;
      minife)  $SR "$B/builds/minife/minife.x+pat" nx=$SIZE ny=$SIZE nz=$SIZE \
                                > "run_$SET.log" 2>&1 ;;
      lulesh)  $SR "$B/builds/lulesh/lulesh2.0+pat" -s "$SIZE" -i 100 \
                                > "run_$SET.log" 2>&1 ;;
      comd)    read -r I J K < <(python3 "$B/scripts/pq3.py" "$NRANK")
               $SR "$B/builds/comd/CoMD-mpi+pat" -x "$SIZE" -y "$SIZE" -z "$SIZE" \
                   -N 100 -i "$I" -j "$J" -k "$K" > "run_$SET.log" 2>&1 ;;
    esac

    RC=$?; T1=$(date +%s.%N)
    echo "=====END_SET $SET rc=$RC wall=$(echo "$T1 - $T0" | bc)====="
done
echo "SWEEP_DONE"