Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

Column
width900px


Code Block
languagebash
themeEmacs
titleExample 1 : One processes process with a single GPU using shared node access
linenumberstrue
#!/bin/bash --login​
 
​#SBATCH --account=project-gpu​
#SBATCH --partition=gpu​
#SBATCH --ntasks=1​
#SBATCH --ntasks-per-node=1​
#SBATCH --cpus-per-task=8​
#SBATCH --gpus-per-task=1​
#SBATCH --time=24:00:00​
​
export OMP_NUM_THREADS=1
srun -c 8 ./program


Code Block
languagebash
themeEmacs
titleExample 2 : One process with a eight GPUs using exclusive node access
linenumberstrue
#!/bin/bash --login​
 
#SBATCH --account=project-gpu​
#SBATCH --partition=gpu​
#SBATCH --ntasks=1
#SBATCH --ntasks-per-node=1
#SBATCH --gpus-per-task=8​
#SBATCH --time=24:00:00​
#SBATCH --exclusive

export OMP_NUM_THREADS=1 
srun -c 8 ./program 


Code Block
languagebash
themeEmacs
titleExample 3 : Eight processes each with a single GPU using exclusive node access
linenumberstrue
#!/bin/bash --login​
 
#SBATCH --account=project-gpu​
#SBATCH --partition=gpu​
#SBATCH --ntasks=8​
#SBATCH --ntasks-per-node=8
#SBATCH --gpus-per-task=1​
#SBATCH --time=24:00:00​
#SBATCH --exclusive

#----
#First preliminar "hack": create a selectGPU wrapper to be used for
# binding only 1 GPU per each task spawned by srun
wrapper=selectGPU_${SLURM_JOBID}
cat << EOF > $wrapper
#!/bin/bash

export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID
exec \$*
EOF
chmod +x ./$wrapper


#----
#Second preliminar "hack": generate an ordered list of CPU-cores (each on a different slurm-socket)
#                          to be matched with the correct GPU in the srun command using --cpu-bind option.
#                          Script "generate_CPU_BIND.sh" serves this purpose and is available to all users
#                          through the module pawseytools, which is loaded by default.
CPU_BIND=$(generate_CPU_BIND.sh map_cpu)
lastResult=$?
if [ $lastResult -ne 0 ]; then
   echo "Exiting as the map generation for CPU_BIND failed" 1>&2
   rm -f ./$wrapper #deleting the wrapper
   exit 1
fi
echo "CPU_BIND=$CPU_BIND"  

#----
#MPI & OpenMP settings
export MPICH_GPU_SUPPORT_ENABLED=1 #This allows for GPU-aware MPI communication among GPUs
export OMP_NUM_THREADS=1           #This controls the real CPU-cores per task for the executable

#----
#Execution
srun -c 8 --cpu-bind=${CPU_BIND} ./$wrapper ./program

#----
#Deleting the wrapper
rm -f ./$wrapper


...