hardware-counters

#!/bin/bash
# Drip-feed the Cirrus sweep, respecting the QOS submit cap.
B=/work/project/project/user
WORK=$B/runs/sweep/worklist.txt
DONE=$B/runs/sweep/dispatched.txt
LIMIT=${LIMIT:-60}
touch "$DONE"

while read -r app nc size nt tl; do
  [ -z "$app" ] && continue
  key="$app $nc $size $nt"
  grep -qxF "$key" "$DONE" && continue
  if grep -qs SWEEP_DONE $B/runs/sweep/x_${app}_c${nc}_s${size}_t${nt}-*.out 2>/dev/null; then
    echo "$key" >> "$DONE"; continue
  fi
  while [ "$(squeue -u user -h 2>/dev/null | wc -l)" -ge "$LIMIT" ]; do sleep 60; done
  jid=$(sbatch --parsable -J x_${app}_c${nc}_s${size}_t${nt} -t "$tl" \
        --export=ALL,APP=$app,NCORE=$nc,SIZE=$size,NTHREAD=$nt \
        $B/scripts/cirrus_sweep.slurm 2>/dev/null)
  if [ -n "$jid" ]; then
    echo "$key" >> "$DONE"; echo "$(date +%H:%M:%S) $jid  $key"
  else
    sleep 120
  fi
done < "$WORK"
echo "drip complete: $(wc -l < "$DONE") dispatched"