hardware-counters

#!/bin/bash
#SBATCH --job-name=of
#SBATCH --nodes=1
#SBATCH --exclusive
#SBATCH --account=project
#SBATCH --partition=standard
#SBATCH --qos=standard
#SBATCH --output=%x-%j.out

# OpenFOAM motorBike single-node core sweep with CrayPat counter rotation.
set +u
B=/work/project/project/user
NCORE=${NCORE:?}

module load perftools-base perftools >/dev/null 2>&1
module load openfoam/com/v2212 >/dev/null 2>&1
source /work/y07/shared/apps/core/openfoam/com/v2212/OpenFOAM-v2212/etc/bashrc || true

RUNDIR=$B/runs/sweep/openfoam_c${NCORE}
rm -rf "$RUNDIR"
cp -r $B/builds/openfoam/motorBike "$RUNDIR"
cd "$RUNDIR" || exit 1
echo "META app=openfoam ncore=$NCORE node=$(hostname) jobid=$SLURM_JOB_ID"

export OMP_NUM_THREADS=1

# scotch handles ANY core count (no need for factor triples like hierarchical)
cat > system/decomposeParDict << EOF
FoamFile { version 2.0; format ascii; class dictionary; object decomposeParDict; }
numberOfSubdomains $NCORE;
method scotch;
EOF

mkdir -p constant/triSurface
cp -f "$FOAM_TUTORIALS"/resources/geometry/motorBike.obj.gz constant/triSurface/ 2>/dev/null

# ---- mesh generation ----
surfaceFeatureExtract > log.sfe 2>&1
blockMesh             > log.blockMesh 2>&1; echo "blockMesh rc=$?"
decomposePar -force   > log.decomp 2>&1;    echo "decomposePar rc=$?"

if [ "$NCORE" -gt 1 ]; then
  srun --ntasks=$NCORE --hint=nomultithread snappyHexMesh -overwrite -parallel > log.shm 2>&1
  echo "snappyHexMesh rc=$?"
  reconstructParMesh -constant > log.recon 2>&1
  rm -rf processor*
else
  snappyHexMesh -overwrite > log.shm 2>&1
  echo "snappyHexMesh rc=$?"
fi

# ---- install initial conditions on the NEW mesh, then decompose again ----
rm -rf 0
cp -r 0.orig 0
decomposePar -force > log.decomp2 2>&1; echo "decomposePar2 rc=$?"

# fixed iteration count so runtime is comparable across core counts
sed -i "s/^endTime.*/endTime 100;/"              system/controlDict
sed -i "s/^writeInterval.*/writeInterval 1000;/" system/controlDict

# ---- profiled solve, rotating counter sets ----
for SET in A B C D E; do
  PAT_RT_PERFCTR=$($B/scripts/counter_sets.sh $SET); export PAT_RT_PERFCTR
  export PAT_RT_EXPDIR_BASE=$RUNDIR
  export PAT_RT_EXPDIR_NAME=exp_${SET}
  export PAT_RT_EXPDIR_REPLACE=1
  echo "=====BEGIN_SET $SET====="
  T0=$(date +%s.%N)
  if [ "$NCORE" -gt 1 ]; then
    srun --ntasks=$NCORE --hint=nomultithread pat_run -w -- simpleFoam -parallel > log.solve_$SET 2>&1
  else
    srun --ntasks=1 --hint=nomultithread pat_run -w -- simpleFoam > log.solve_$SET 2>&1
  fi
  RC=$?
  T1=$(date +%s.%N)
  echo "=====END_SET $SET rc=$RC wall=$(echo "$T1 - $T0" | bc)====="
done
echo "SWEEP_DONE"