#!/bin/bash

#MSUB -r BNH2hy
#MSUB -N 400                 # Total number of nodes
#MSUB -n 1600                  # Total number of tasks to use
#MSUB -c 16                 # Number of threads per task to use
#MSUB -A gen7594
#MSUB -q knl
#MSUB -T 86000              # Time

#MSUB -o %I.out            # Standard output. %I is the job id
#MSUB -e %I.err            # Error output. %I is the job id
#MSUB -m work,scratch

# Setup the Smilei environment
module purge
bash -c 'LD_PRELOAD=/ccc/products/python-2.7.14/intel--17.0.4.196__openmpi--2.0.2/default/lib/libpython2.7.so.1.0:$LD_PRELOAD smilei'
module load hdf5
module switch flavor/hdf5/serial flavor/hdf5/parallel
export HDF5_ROOT_DIR=${HDF5_ROOT}
export OMPI_MCA_btl_portals4_use_rdma=0

# OMP parameters
export OMP_SCHEDULE=dynamic
#export OMP_PROC_BIND=true
export OMP_NUM_THREADS=32

# Get name of current directory (name of the simulation)
case=${PWD##*/} 
echo "case "$case

# Make scratch folder
mkdir $S/$case

# Find out existing checkpoints
checkpoints=($(ls -d $S/$case/checkpoint* 2>/dev/null))
extra_command=""
if [ ${#checkpoints[@]} -eq 0 ]; then
  # No checkpoint yet: create new folder
  simulation_folder=$S/$case/checkpoint000
else
  # Checkpoints exist
  restart_folder=${checkpoints[-1]}
  next_checkpoint_number=$((${restart_folder##*/checkpoint}+1))
  simulation_folder=$S/$case/checkpoint"$(printf "%03d" $next_checkpoint_number)"
  extra_command="Checkpoints.restart_dir='$restart_folder'"
fi

mkdir $simulation_folder

echo restart_folder $restart_folder
echo next_checkpoint_number $next_checkpoint_number
echo simulation_folder $simulation_folder
echo extra_command $extra_command

# Copy files to scratch
cp /ccc/work/cont003/gen6129/bonvalej/Smilei/Smilei_hub/smileiKNL $simulation_folder
cp BNH2d.py $simulation_folder
cp hydro.txt $simulation_folder
cd $simulation_folder

# Run smilei
#ccc_mprun ./smilei gsi.py $extra_command >out 2>err
ccc_mprun bash -c 'LD_PRELOAD=/ccc/products/python-2.7.14/intel--17.0.4.196__openmpi--2.0.2/default/lib/libpython2.7.so.1.0:$LD_PRELOAD ./smileiKNL BNH2d.py $extra_command >out 2>err'