hardware-counters

#!/bin/bash
B=/work/project/project/user
WORK=$B/runs/weak/worklist.txt
DONE=$B/runs/weak/dispatched.txt
LIMIT=${LIMIT:-200}
touch "$DONE"
while read -r app nc base nt tl; do
  [ -z "$app" ] && continue
  key="$app $nc $base $nt"
  grep -qxF "$key" "$DONE" && continue
  if grep -qs SWEEP_DONE $B/runs/weak/w_${app}_c${nc}_b${base}_t${nt}-*.out 2>/dev/null; then
    echo "$key" >> "$DONE"; continue
  fi
  while [ "$(squeue -u user -h 2>/dev/null | wc -l)" -ge "$LIMIT" ]; do sleep 60; done
  jid=$(sbatch --parsable -J w_${app}_c${nc}_b${base}_t${nt} -t "$tl" \
        --export=ALL,APP=$app,NCORE=$nc,BASE=$base,NTHREAD=$nt \
        $B/scripts/cirrus_weak.slurm 2>/dev/null)
  if [ -n "$jid" ]; then echo "$key" >> "$DONE"; echo "$(date +%H:%M:%S) $jid $key"
  else sleep 120; fi
done < "$WORK"
echo "drip complete: $(wc -l < "$DONE")"