#!/bin/bash --login
# SLURM directives
#
# Here we specify to SLURM we want two nodes (--nodes=2) with
# a wall-clock time limit of twenty minutes (--time=00:20:00).
#
# Replace [your-project] with the appropriate project name
# following --account (e.g., --account=project123).
#SBATCH --account=[your-project]
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4 #this directive is required on setonix to request 4 tasks on each node
#SBATCH --ntasks-per-socket=2 #this directive indicates that maximum 2 tasks per socket are to be allocated
#SBATCH --ntasks=8 #this directive is required on setonix to request a total of 8 tasks
#SBATCH --cpus-per-task=32 #this directive is required on setonix to request 32 CPU cores for each task for the OpenMP threads
#SBATCH --exclusive #always use when all node resources are needed (or use --mem for shared access)
#SBATCH --time=00:20:00
# Set number of OpenMP threads
export OMP_NUM_THREADS=32
#---
#Specific settings for the cluster you are on
#(Check the specific guide of the cluster for additional settings)
# Launch the job.
# Here we use 8 instances (-n 8) with 4 per node with two instances per socket,
# or NUMA region. Each instance requests 32 cores -c 32 (via -c $OMP_NUM_THREADS)
srun -N 2 -n 8 -c ${OMP_NUM_THREADS} -m block:block:block ./wrapper.sh |