#!/bin/bash --login
#SBATCH --account=project-gpu
#SBATCH --partition=gpu
#SBATCH --ntasks=8
#SBATCH --ntasks-per-node=8
#SBATCH --gpus-per-task=1
#SBATCH --time=24:00:00
#SBATCH --exclusive
#----
#First preliminar "hack": create a selectGPU wrapper to be used for
# binding only 1 GPU per each task spawned by srun
wrapper=selectGPU_${SLURM_JOBID}
cat << EOF > $wrapper
#!/bin/bash
export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID
exec \$*
EOF
chmod +x ./$wrapper
#----
#Second preliminar "hack": generate an ordered list of CPU-cores (each on a different slurm-socket)
# to be matched with the correct GPU in the srun command using --cpu-bind option.
# Script "generate_CPU_BIND.sh" serves this purpose and is available to all users
# through the module pawseytools, which is loaded by default.
CPU_BIND=$(generate_CPU_BIND.sh map_cpu)
lastResult=$?
if [ $lastResult -ne 0 ]; then
echo "Exiting as the map generation for CPU_BIND failed" 1>&2
rm -f ./$wrapper #deleting the wrapper
exit 1
fi
echo "CPU_BIND=$CPU_BIND"
#----
#MPI & OpenMP settings
export MPICH_GPU_SUPPORT_ENABLED=1 #This allows for GPU-aware MPI communication among GPUs
export OMP_NUM_THREADS=1 #This controls the real CPU-cores per task for the executable
#----
#Execution
srun -c 8 --cpu-bind=${CPU_BIND} ./$wrapper ./program
#----
#Deleting the wrapper
rm -f ./$wrapper |