#!/usr/bin/env bash
# Copyright 2019-2021 Axel Huebl, Rene Widera
#
# This file is part of PIConGPU.
#
# PIConGPU is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PIConGPU is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with PIConGPU.
# If not, see <http://www.gnu.org/licenses/>.
#


# PIConGPU batch script for Summit's LSF (bsub) batch system
#   https://www.olcf.ornl.gov/for-users/system-user-guides/summit/summit-user-guide/#running-jobs

#BSUB -q gpu-short
#BSUB -W 1:00
# Sets batch job's name
#BSUB -J LaserWakefield
#BSUB -nnodes 1
#B SUB -R rusage[mem=32000]
#BSUB -M 8192
# send me mails on job (-B)egin, Fi(-N)ish
#BSUB  -u someone@example.com
#BSUB -cwd /home/labs/malkalab/danle/picOutput/LaserWakefield
#BSUB -P 

#BSUB -o stdout.%J
#BSUB -e stderr.%J


## calculations will be performed by tbg ##

# remove seconds from walltime

# settings that can be controlled by environment variables before submit

# number of available/hosted GPUs per node in the system

# required GPUs per node for the current job

# number of cores to block per GPU - we got 2x22 HW CPU cores per node
#   and we will be accounted those anyway

# use ceil to caculate nodes

## end calculations ##

echo 'Running program...'
echo LaserWakefield

cd /home/labs/malkalab/danle/picOutput/LaserWakefield
echo -n "present working directory:"
pwd


source /home/labs/malkalab/danle/src/spack/opt/spack/linux-ubuntu16.04-broadwell/gcc-7.4.0/picongpu-0.4.3-32hgvt3f654v6zfk7ghipbapi26opvib/etc/picongpu/picongpu.profile 2>/dev/null
if [ $? -ne 0 ] ; then
  echo "Error: PIConGPU environment profile under \"/home/labs/malkalab/danle/src/spack/opt/spack/linux-ubuntu16.04-broadwell/gcc-7.4.0/picongpu-0.4.3-32hgvt3f654v6zfk7ghipbapi26opvib/etc/picongpu/picongpu.profile\" not found!"
  exit 1
fi

mkdir simOutput 2> /dev/null
cd simOutput

#jsrun  -N 1 -n 1 /home/labs/malkalab/danle/picOutput/LaserWakefield/input/bin/cuda_memtest.sh

#if [ $? -eq 0 ] ; then
export OMP_NUM_THREADS=7
mpiexec -n 1 /home/labs/malkalab/danle/picOutput/LaserWakefield/input/bin/picongpu  -d 1 1 1                    -g 192 1024 12                      -s 2048                         --periodic 0 0 1                        --e_png.period 100                                --e_png.axis yx --e_png.slicePoint 0.5            --e_png.folder pngElectronsYX                                 --e_energyHistogram.period 100                     --e_energyHistogram.binCount 1024                  --e_energyHistogram.minEnergy 0 --e_energyHistogram.maxEnergy 1000                  --e_energyHistogram.filter all                           --e_phaseSpace.period 100                                      --e_phaseSpace.space y --e_phaseSpace.momentum py              --e_phaseSpace.min -1.0 --e_phaseSpace.max 1.0                 --e_phaseSpace.filter all                               --e_macroParticlesCount.period 100                         --versionOnce | tee output
# note: instead of the PIConGPU binary, one can also debug starting "js_task_info | sort"
#fi

#this script was created with call /home/labs/malkalab/danle/src/spack/opt/spack/linux-ubuntu16.04-broadwell/gcc-7.4.0/picongpu-0.4.3-32hgvt3f654v6zfk7ghipbapi26opvib/bin/tbg -s bsub -c etc/picongpu/1.cfg -t etc/picongpu/wexac-wis/gpu_batch.tpl /home/labs/malkalab/danle/picOutput/LaserWakefield -f