#!/usr/bin/env bash
# Copyright 2019-2021 Axel Huebl, Rene Widera
#
# This file is part of PIConGPU.
#
# PIConGPU is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PIConGPU is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with PIConGPU.
# If not, see <http://www.gnu.org/licenses/>.
#


# PIConGPU batch script for Summit's LSF (bsub) batch system
#   https://www.olcf.ornl.gov/for-users/system-user-guides/summit/summit-user-guide/#running-jobs

#BSUB -csm y
#BSUB -q gpu-short
#BSUB -W 1:00
# Sets batch job's name
#BSUB -J FoilLCT
#BSUB -n 4 
#BSUB -gpu num=1:j_exclusive=yes 
#BSUB -R rusage[mem=32000]
#BSUB -cwd /home/labs/malkalab/danle/picOutput/FoilLCT
#B SUB -P 

#BSUB -m dgx_hosts

#BSUB -o stdout.%J
#BSUB -e stderr.%J


## calculations will be performed by tbg ##

# remove seconds from walltime

# settings that can be controlled by environment variables before submit

# number of available/hosted GPUs per node in the system

# required GPUs per node for the current job

# number of cores to block per GPU - we got 2x22 HW CPU cores per node
#   and we will be accounted those anyway

# use ceil to caculate nodes

## end calculations ##

echo 'Running program...'
echo FoilLCT

cd /home/labs/malkalab/danle/picOutput/FoilLCT
echo -n "present working directory:"
pwd


source ~/picongpu.profile 2>/dev/null
if [ $? -ne 0 ] ; then
  echo "Error: PIConGPU environment profile under \"//danle/picongpu.profile\" not found!"
  exit 1
fi

mpiexec -n 4 hostname

mkdir simOutput 2> /dev/null
cd simOutput

#jsrun  -N 1 -n 1 /home/labs/malkalab/danle/picOutput/FoilLCT/input/bin/cuda_memtest.sh

#if [ $? -eq 0 ] ; then
export OMP_NUM_THREADS=7

mpiexec -n 4 /home/labs/malkalab/danle/picOutput/FoilLCT/input/bin/picongpu  -d 2 2 1                    -g 256 2560                      -s 5000                         --periodic 1 0                         --e_energyHistogram.period 100 --e_energyHistogram.filter all  --e_energyHistogram.binCount 1024                     --e_energyHistogram.minEnergy 0 --e_energyHistogram.maxEnergy 2000 --H_energyHistogram.period 100 --H_energyHistogram.filter all --H_energyHistogram.binCount 1024                     --H_energyHistogram.minEnergy 0 --H_energyHistogram.maxEnergy 2000 --C_energyHistogram.period 100 --C_energyHistogram.filter all  --C_energyHistogram.binCount 1024                     --C_energyHistogram.minEnergy 0 --C_energyHistogram.maxEnergy 2000 --N_energyHistogram.period 100 --N_energyHistogram.filter all --N_energyHistogram.binCount 1024                     --N_energyHistogram.minEnergy 0 --N_energyHistogram.maxEnergy 2000              --e_phaseSpace.period 250 --e_phaseSpace.filter all --e_phaseSpace.space y --e_phaseSpace.momentum py --e_phaseSpace.min -2.0 --e_phaseSpace.max 2.0 --H_phaseSpace.period 250 --H_phaseSpace.filter all --H_phaseSpace.space y --H_phaseSpace.momentum py --H_phaseSpace.min -0.04 --H_phaseSpace.max 0.04 --C_phaseSpace.period 250 --C_phaseSpace.filter all --C_phaseSpace.space y --C_phaseSpace.momentum py --C_phaseSpace.min -0.02 --C_phaseSpace.max 0.02 --N_phaseSpace.period 250 --N_phaseSpace.filter all --N_phaseSpace.space y --N_phaseSpace.momentum py --N_phaseSpace.min -0.02 --N_phaseSpace.max 0.02                              --fields_energy.period 100                --e_energy.period 100 --e_energy.filter all                --H_energy.period 100 --H_energy.filter all                --C_energy.period 100 --C_energy.filter all                --N_energy.period 100 --N_energy.filter all --chargeConservation.period 100                                           --openPMD.period 250 --openPMD.file simData --openPMD.ext h5                          --versionOnce | tee output
#
#mpiexec -n 9 /home/labs/malkalab/danle/picOutput/FoilLCT/input/bin/picongpu  -d 1 9 1                    -g 192 1024 12                      -s 2048                         --periodic 0 0 1                        --e_png.period 100                                --e_png.axis yx --e_png.slicePoint 0.5            --e_png.folder pngElectronsYX                                 --e_energyHistogram.period 100                     --e_energyHistogram.binCount 1024                  --e_energyHistogram.minEnergy 0 --e_energyHistogram.maxEnergy 1000                  --e_energyHistogram.filter all                           --e_phaseSpace.period 100                                      --e_phaseSpace.space y --e_phaseSpace.momentum py              --e_phaseSpace.min -1.0 --e_phaseSpace.max 1.0                 --e_phaseSpace.filter all                               --e_macroParticlesCount.period 100                         --versionOnce | tee output

# note: instead of the PIConGPU binary, one can also debug starting "js_task_info | sort"
#fi

#this script was created with call cd /home/labs/malkalab/danle/picInput/FoilLCT; /home/labs/malkalab/danle/src/picongpu/bin/tbg -s bsub -c etc/picongpu/1.cfg -t /home/labs/malkalab/danle/picOutput/LaserWakefield_2/tbg/submit.tpl /home/labs/malkalab/danle/picOutput/LWfield