hardware-counters

#!/bin/bash
# Build and instrument all benchmarks on Cirrus (HPE Cray EX4000, EPYC 9825 Turin).
# Mirrors the ARCHER2 build_all.sh; differences are noted inline.
#   bash -l cirrus_build.sh
set -u
B=/work/project/project/user
module load perftools >/dev/null 2>&1
echo "pat_build: $(which pat_build)"
echo "cc: $(cc --version 2>&1 | head -1)"
echo

# ---------------------------------------------------------------- STREAM
echo "### STREAM ###"
mkdir -p "$B/builds/stream" && cd "$B/builds/stream" || exit 1
cp -f "$B/src/stream.c" .
# Cirrus nodes have 768 GB, but keep sizes identical to ARCHER2 so the two
# platforms are directly comparable. 80M elements ~= 1.9 GB.
for S in 10000000 40000000 80000000; do
    cc -O3 -fopenmp -DSTREAM_ARRAY_SIZE=$S -DNTIMES=20 -o "stream_$S" stream.c 2>/dev/null
    pat_build -w -o "stream_$S+pat" "stream_$S" 2>/dev/null
    printf "  %-10s %s\n" "$S" "$([ -f "stream_$S+pat" ] && echo ok || echo FAILED)"
done

# ---------------------------------------------------------------- HPCG
echo "### HPCG ###"
cd "$B/src/hpcg" || exit 1
cat > setup/Make.Cray << 'EOF'
SHELL         = /bin/sh
CD            = cd
CP            = cp
LN_S          = ln -s -f
MKDIR         = mkdir -p
RM            = /bin/rm -f
TOUCH         = touch
TOPdir        = .
SRCdir        = $(TOPdir)/src
INCdir        = $(TOPdir)/src
BINdir        = $(TOPdir)/bin
MPinc         =
MPlib         =
HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc)
HPCG_LIBS     =
HPCG_OPTS     = -DHPCG_NO_LONG_LONG
HPCG_DEFS     = $(HPCG_OPTS) $(HPCG_INCLUDES)
CXX           = CC
CXXFLAGS      = $(HPCG_DEFS) -O3 -ffast-math -ftree-vectorize -fopenmp
LINKER        = $(CXX)
LINKFLAGS     = $(CXXFLAGS)
ARCHIVER      = ar
ARFLAGS       = r
RANLIB        = echo
EOF
mkdir -p build_cray && cd build_cray || exit 1
../configure Cray >/dev/null 2>&1
make -j16 > "$B/logs/hpcg_build.log" 2>&1
if [ -f bin/xhpcg ]; then
    mkdir -p "$B/builds/hpcg" && cd "$B/builds/hpcg" || exit 1
    cp -f "$B/src/hpcg/build_cray/bin/xhpcg" .
    pat_build -g mpi,io -w -o xhpcg+pat xhpcg 2>/dev/null
    echo "  $([ -f xhpcg+pat ] && echo ok || echo FAILED)"
else
    echo "  BUILD FAILED - see logs/hpcg_build.log"
fi

# ---------------------------------------------------------------- HPL
echo "### HPL ###"
cd "$B/src/hpl-2.3" || exit 1
cat > Make.Cray << EOF
SHELL        = /bin/sh
CD           = cd
CP           = cp
LN_S         = ln -s
MKDIR        = mkdir
RM           = /bin/rm -f
TOUCH        = touch
ARCH         = Cray
TOPdir       = $B/src/hpl-2.3
INCdir       = \$(TOPdir)/include
BINdir       = \$(TOPdir)/bin/\$(ARCH)
LIBdir       = \$(TOPdir)/lib/\$(ARCH)
HPLlib       = \$(LIBdir)/libhpl.a
MPinc        =
MPlib        =
LAinc        =
LAlib        = -lsci_cray
F2CDEFS      = -DAdd_ -DF77_INTEGER=int -DStringSunStyle
HPL_INCLUDES = -I\$(INCdir) -I\$(INCdir)/\$(ARCH) \$(LAinc) \$(MPinc)
HPL_LIBS     = \$(HPLlib) \$(LAlib) \$(MPlib)
HPL_OPTS     = -DHPL_CALL_CBLAS
HPL_DEFS     = \$(F2CDEFS) \$(HPL_OPTS) \$(HPL_INCLUDES)
CC           = cc
CCNOOPT      = \$(HPL_DEFS)
CCFLAGS      = \$(HPL_DEFS) -O3 -fopenmp
LINKER       = cc
LINKFLAGS    = \$(CCFLAGS)
ARCHIVER     = ar
ARFLAGS      = r
RANLIB       = echo
EOF
make arch=Cray > "$B/logs/hpl_build.log" 2>&1
if [ -f bin/Cray/xhpl ]; then
    mkdir -p "$B/builds/hpl" && cd "$B/builds/hpl" || exit 1
    cp -f "$B/src/hpl-2.3/bin/Cray/xhpl" .
    pat_build -g mpi -w -o xhpl+pat xhpl 2>/dev/null
    echo "  $([ -f xhpl+pat ] && echo ok || echo FAILED)"
else
    echo "  BUILD FAILED - see logs/hpl_build.log"
fi

# ---------------------------------------------------------------- miniFE
echo "### miniFE ###"
cd "$B/src/miniFE/openmp/src" 2>/dev/null || cd "$B/src/miniFE/ref/src" || exit 1
make -j16 CXX=CC CC=cc CXXFLAGS="-O3 -fopenmp" CFLAGS="-O3 -fopenmp" \
     MPI_CXX=CC MPI_CC=cc > "$B/logs/minife_build.log" 2>&1
SRC=$PWD
if [ -f "$SRC/miniFE.x" ]; then
    mkdir -p "$B/builds/minife" && cd "$B/builds/minife" || exit 1
    cp -f "$SRC/miniFE.x" ./minife.x
    pat_build -g mpi -w -o minife.x+pat minife.x 2>/dev/null
    echo "  $([ -f minife.x+pat ] && echo ok || echo FAILED)"
else
    echo "  BUILD FAILED - see logs/minife_build.log"
fi

# ---------------------------------------------------------------- lulesh
echo "### lulesh ###"
cd "$B/src/LULESH" || exit 1
make -j16 CXX="CC -DUSE_MPI=1" CXXFLAGS="-O3 -fopenmp" LDFLAGS="-fopenmp" \
     > "$B/logs/lulesh_build.log" 2>&1
if [ -f lulesh2.0 ]; then
    mkdir -p "$B/builds/lulesh" && cd "$B/builds/lulesh" || exit 1
    cp -f "$B/src/LULESH/lulesh2.0" .
    pat_build -g mpi -w -o lulesh2.0+pat lulesh2.0 2>/dev/null
    echo "  $([ -f lulesh2.0+pat ] && echo ok || echo FAILED)"
else
    echo "  BUILD FAILED - see logs/lulesh_build.log"
fi

# ---------------------------------------------------------------- CoMD
echo "### CoMD ###"
cd "$B/src/CoMD/src-mpi" || exit 1
[ -f Makefile.vanilla ] && cp -f Makefile.vanilla Makefile
sed -i 's|^CC *=.*|CC = cc|'                     Makefile
sed -i 's|^OPTFLAGS *=.*|OPTFLAGS = -O3 -fopenmp|' Makefile
sed -i 's|^DO_MPI *=.*|DO_MPI = ON|'             Makefile
make -j16 > "$B/logs/comd_build.log" 2>&1
if [ -f "$B/src/CoMD/bin/CoMD-mpi" ]; then
    mkdir -p "$B/builds/comd" && cd "$B/builds/comd" || exit 1
    cp -f "$B/src/CoMD/bin/CoMD-mpi" .
    pat_build -g mpi -w -o CoMD-mpi+pat CoMD-mpi 2>/dev/null
    echo "  $([ -f CoMD-mpi+pat ] && echo ok || echo FAILED)"
else
    echo "  BUILD FAILED - see logs/comd_build.log"
fi

echo
echo "=== instrumented binaries ==="
ls -la "$B"/builds/*/*+pat 2>/dev/null | awk '{print "  " $NF}'