hardware-counters
build_all.sh
#!/bin/bash
# =============================================================================
# Build all benchmark applications on ARCHER2 with the Cray toolchain and
# instrument them for CrayPat profiling.
#
# bash build_all.sh
#
# Notes / gotchas discovered while getting this working:
# * Binaries MUST be compiled with the `perftools` module loaded, otherwise
# pat_build fails with: Missing required ELF section '.note.link'.
# * The pre-built ARCHER2 `gromacs/2025.4` module CANNOT be instrumented for
# this reason, so GROMACS is built from source here.
# * CP2K and OpenFOAM are only available as pre-built modules; they are
# profiled with `pat_run` (dynamic instrumentation) instead - see the
# slurm/ scripts.
# =============================================================================
set -eu
B=${B:-/work/project/project/user}
mkdir -p "$B"/{src,builds,logs}
module load perftools >/dev/null 2>&1
echo "perftools: $(which pat_build)"
# -----------------------------------------------------------------------------
# 1. STREAM (memory bandwidth)
# -----------------------------------------------------------------------------
echo "### STREAM ###"
cd "$B/src"
[ -f stream.c ] || curl -sS -O https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
mkdir -p "$B/builds/stream" && cd "$B/builds/stream"
cp -f "$B/src/stream.c" .
# 80M elements x 3 arrays x 8 B ~= 1.9 GB, comfortably exceeds LLC
cc -O3 -fopenmp -DSTREAM_ARRAY_SIZE=80000000 -DNTIMES=20 -o stream_c.exe stream.c
pat_build -w -o stream_c.exe+pat stream_c.exe
echo " -> $B/builds/stream/stream_c.exe+pat"
# -----------------------------------------------------------------------------
# 2. HPCG (sparse solver: memory + MPI bound)
# -----------------------------------------------------------------------------
echo "### HPCG ###"
cd "$B/src"
[ -d hpcg ] || git clone -q --depth 1 https://github.com/hpcg-benchmark/hpcg.git
cd hpcg
cat > setup/Make.Cray << 'EOF'
SHELL = /bin/sh
CD = cd
CP = cp
LN_S = ln -s -f
MKDIR = mkdir -p
RM = /bin/rm -f
TOUCH = touch
TOPdir = .
SRCdir = $(TOPdir)/src
INCdir = $(TOPdir)/src
BINdir = $(TOPdir)/bin
MPinc =
MPlib =
HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc)
HPCG_LIBS =
HPCG_OPTS = -DHPCG_NO_LONG_LONG
HPCG_DEFS = $(HPCG_OPTS) $(HPCG_INCLUDES)
CXX = CC
CXXFLAGS = $(HPCG_DEFS) -O3 -ffast-math -ftree-vectorize -fopenmp
LINKER = $(CXX)
LINKFLAGS = $(CXXFLAGS)
ARCHIVER = ar
ARFLAGS = r
RANLIB = echo
EOF
mkdir -p build_cray && cd build_cray
../configure Cray
make -j8
mkdir -p "$B/builds/hpcg" && cd "$B/builds/hpcg"
cp -f "$B/src/hpcg/build_cray/bin/xhpcg" .
pat_build -g mpi,io -w -o xhpcg+pat xhpcg
echo " -> $B/builds/hpcg/xhpcg+pat"
# -----------------------------------------------------------------------------
# 3. HPL (dense LU: compute bound), linked against Cray LibSci
# -----------------------------------------------------------------------------
echo "### HPL ###"
cd "$B/src"
[ -f hpl-2.3.tar.gz ] || curl -sS -O https://www.netlib.org/benchmark/hpl/hpl-2.3.tar.gz
[ -d hpl-2.3 ] || tar xzf hpl-2.3.tar.gz
cd hpl-2.3
cat > Make.Cray << EOF
SHELL = /bin/sh
CD = cd
CP = cp
LN_S = ln -s
MKDIR = mkdir
RM = /bin/rm -f
TOUCH = touch
ARCH = Cray
TOPdir = $B/src/hpl-2.3
INCdir = \$(TOPdir)/include
BINdir = \$(TOPdir)/bin/\$(ARCH)
LIBdir = \$(TOPdir)/lib/\$(ARCH)
HPLlib = \$(LIBdir)/libhpl.a
MPinc =
MPlib =
LAinc =
LAlib = -lsci_cray
F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle
HPL_INCLUDES = -I\$(INCdir) -I\$(INCdir)/\$(ARCH) \$(LAinc) \$(MPinc)
HPL_LIBS = \$(HPLlib) \$(LAlib) \$(MPlib)
HPL_OPTS = -DHPL_CALL_CBLAS
HPL_DEFS = \$(F2CDEFS) \$(HPL_OPTS) \$(HPL_INCLUDES)
CC = cc
CCNOOPT = \$(HPL_DEFS)
CCFLAGS = \$(HPL_DEFS) -O3 -fopenmp
LINKER = cc
LINKFLAGS = \$(CCFLAGS)
ARCHIVER = ar
ARFLAGS = r
RANLIB = echo
EOF
make arch=Cray
mkdir -p "$B/builds/hpl" && cd "$B/builds/hpl"
cp -f "$B/src/hpl-2.3/bin/Cray/xhpl" .
pat_build -g mpi -w -o xhpl+pat xhpl
echo " -> $B/builds/hpl/xhpl+pat"
# -----------------------------------------------------------------------------
# 4. GROMACS 2025.1 (molecular dynamics: mixed compute/memory/PME comms)
# Built from source because the ARCHER2 module lacks .note.link.
# -----------------------------------------------------------------------------
echo "### GROMACS ###"
module load cray-fftw cmake >/dev/null 2>&1
cd "$B/src"
[ -f gromacs-2025.1.tar.gz ] || curl -sS -O https://ftp.gromacs.org/gromacs/gromacs-2025.1.tar.gz
[ -d gromacs-2025.1 ] || tar xzf gromacs-2025.1.tar.gz
cd gromacs-2025.1
rm -rf build_cray && mkdir -p build_cray && cd build_cray
# BUILD_TESTING/GMXAPI/NBLIB must be OFF or cmake errors on the unit-test target
cmake .. \
-DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC \
-DGMX_MPI=ON -DGMX_OPENMP=ON \
-DGMX_SIMD=AVX2_256 \
-DGMX_FFT_LIBRARY=fftw3 \
-DFFTWF_LIBRARY="$FFTW_ROOT/lib/libfftw3f.so" \
-DFFTWF_INCLUDE_DIR="$FFTW_INC" \
-DGMX_BUILD_OWN_FFTW=OFF \
-DBUILD_SHARED_LIBS=OFF -DGMX_PREFER_STATIC_LIBS=ON \
-DCMAKE_INSTALL_PREFIX="$B/builds/gromacs/install" \
-DBUILD_TESTING=OFF -DGMXAPI=OFF -DGMX_INSTALL_NBLIB_API=OFF
make -j16
make install
cd "$B/builds/gromacs"
pat_build -g mpi -w -o gmx_mpi+pat install/bin/gmx_mpi
# benchmark input: 2,136,412-atom ribosome. NB: the download is a ZIP wrapper,
# not a bare .tpr - it segfaults gmx unless extracted first.
if [ ! -f benchRIB.tpr ]; then
curl -sSL -o benchRIB.zip https://www.mpinat.mpg.de/benchRIB.tpr
unzip -o benchRIB.zip
fi
echo " -> $B/builds/gromacs/gmx_mpi+pat"
# -----------------------------------------------------------------------------
# 5. OpenFOAM (CFD) - module only; stage the motorBike tutorial case
# -----------------------------------------------------------------------------
echo "### OpenFOAM (case staging only) ###"
mkdir -p "$B/builds/openfoam"
OF=/work/y07/shared/apps/core/openfoam/com/v2212/OpenFOAM-v2212
[ -d "$B/builds/openfoam/motorBike" ] || \
cp -r "$OF/tutorials/incompressible/simpleFoam/motorBike" "$B/builds/openfoam/"
echo " -> $B/builds/openfoam/motorBike"
echo
echo "All builds complete."