flex_uber

#!/bin/bash

# ------------------------------------------------------------------------------
# modules
# ------------------------------------------------------------------------------
module load bashHelperKennedyRodrigue
source bashHelperKennedyRodrigueFunctions.sh

# ------------------------------------------------------------------------------
# opts
# ------------------------------------------------------------------------------
set -e
study_list=(all jlbs pams)

declare -A wave_list
wave_list[jlbs]="all 1 2 3"
wave_list[pams]="all"

date_str=`date +%Y-%m-%d`
logs_pwd="${root_dir}/server/logs/${USER}/${date_str}"

declare -a slurm_opts
# slurm_opts[0]="''"
slurm_opts[1]="--nodes=1 \
--ntasks=1 \
--cpus-per-task=1 \
--mem=4G \
--partition=kenrod \
--output=jid-%A_jname-%x.log \
--time=0-08:00:00 \
--export=NONE \
-D ${logs_pwd}"
slurm_opts[2]="--nodes=1 \
--ntasks=1 \
--cpus-per-task=1 \
--mem=8G \
--partition=kenrod \
--output=jid-%A_jname-%x.log \
--time=0-08:00:00 \
--export=NONE \
-D ${logs_pwd}"
slurm_opts[3]="--nodes=1 \
--ntasks=1 \
--cpus-per-task=1 \
--mem=4G \
--partition=kenrod \
--nodelist=compute-02 \
--output=jid-%A_jname-%x.log \
--time=0-08:00:00 \
--export=NONE \
-D ${logs_pwd}"

declare -a slurm_opts_desc
# slurm_opts_desc[0]="[none]"
slurm_opts_desc[1]="[default]"
slurm_opts_desc[2]="[8GB mem]"
slurm_opts_desc[3]="[gpu]\t"

# slurm_opts_desc[4]="[compute-02]"

# slurm_opts+=("custom")

# slurm_default_opts="--nodes=1 \
# --ntasks=1 \
# --cpus-per-task=1 \
# --mem=4G \
# --partition=kenrod \
# --output=jid-%A_jname-%x.log \
# --time=0-08:00:00 \
# --export=NONE \
# -D ${logs_pwd}"

# sge_default_opts="-V \
# -S /bin/bash \
# -o jid-\$JOB_ID-\$TASK_ID_jname-\$JOB_NAME.log \
# -j y"

sge_default_opts="-j y \
-wd ${logs_pwd}"

# ------------------------------------------------------------------------------
# paths
# ------------------------------------------------------------------------------
root_dir=`get_root_dir kenrod`

declare -A in_paths
in_paths[preproc]="${root_dir}/software/scripts/eep170030/script_sets.txt"
in_paths[ids]="${root_dir}/software/scripts/eep170030/ids_long-format_study-all.csv"

in_scripts=(`cat ${in_paths[preproc]} | sort`)

# ------------------------------------------------------------------------------
# main
# ------------------------------------------------------------------------------

# ------------------------------------------------------------------------------
# pipelines
# ------------------------------------------------------------------------------
echo ""
warning_msg "please choose carefully, options are likely to change"
echo -e "\npipelines (sorted):"
for i in ${!in_scripts[@]}; do
    i_str=$((i+1))
    echo -e "\t${i_str}:\t${in_scripts[$i]}"
done
scripts_idx_opts=`seq 1 $((${#in_scripts[@]}))`
scripts_idx=$(get_valid_input \
-p "which pipeline do you want to run? (enter index number)" \
-o ${scripts_idx_opts} \
-e 0)

scripts_idx1=${scripts_idx}
scripts_idx=$((scripts_idx-1))
echo -e "pipeline[${scripts_idx1}]:\t${in_scripts[$scripts_idx]}"

# ------------------------------------------------------------------------------
# script
# ------------------------------------------------------------------------------
echo -e "\nscripts:"
in_script_set=`eval echo ${in_scripts[scripts_idx]}`
in_script=(`cat ${in_script_set}`)
if [[ ${#in_script[@]} -lt 1 ]]; then
    error_msg "no scripts found for ${in_script_set}"
fi
for i in ${!in_script[@]}; do
    i_str=$((i+1))
    echo -e "\t${i_str}:\t${in_script[i]}"
done
script_idx_opts=`seq 1 $((${#in_script[@]}))`
script_idx=$(get_valid_input \
-p "which script do you want to run? (enter index number)" \
-o ${script_idx_opts} \
-e 0)


scripts_idx_tmp=$((script_idx-1))
echo -e "script[${script_idx}]:\t${in_script[$scripts_idx_tmp]}"

# ------------------------------------------------------------------------------
# study
# ------------------------------------------------------------------------------
echo -e "\nstudies:"
for i in ${!study_list[@]}; do
    echo -e "\t${i}:\t${study_list[i]}"
done
study_idx_opts=`seq 0 $((${#study_list[@]}-1))`
study_idx=$(get_valid_input \
-p "which study do you want to run? (enter index number or hit enter to use default; default: 0 [all])" \
-o ${study_idx_opts} \
-e 1 \
-d 0)
study=${study_list[$study_idx]}
echo -e "study[${study_idx}]:\t${study}"

# ------------------------------------------------------------------------------
# session
# ------------------------------------------------------------------------------
echo -e "\nsessions:"
ses_list=(`echo ${wave_list[${study}]}`)
if [[ ${study} == 'all' ]]; then
    ses_list=(`echo ${wave_list[jlbs]} ${wave_list[pams]}`)
    ses_idx=0
elif [[ ${#ses_list[@]} -lt 1 ]]; then
    error_msg "no sessions found for ${study}"
else
    for i in ${!ses_list[@]}; do
        echo -e "\t${i}:\t${ses_list[$i]}"
    done
    ses_idx_opts=`seq 0 $((${#ses_list[@]}-1))`
    ses_idx=$(get_valid_input \
    -p "which session do you want to run? (enter index number or hit enter to use default; default: 0 [all])" \
    -o ${ses_idx_opts} \
    -e 1 \
    -d 0)
fi
echo "session[${ses_idx}]: ${ses_list[$ses_idx]}"
ses=${ses_list[$ses_idx]}
if [[ ${ses} =~ ^[0-9]$ ]]; then
    ses=`printf "%02d" ${ses}`
fi

# ------------------------------------------------------------------------------
# ids using filter
# ------------------------------------------------------------------------------
in_ids=(`filter_ids --study ${study} --ses ${ses}`)
if [[ ${#in_ids[@]} -lt 1 ]]; then
    error_msg "no ids found for ${study}"
fi
echo ""
ids_idx_opts=()
for i in ${!in_ids[@]}; do
    i_str=`echo ${in_ids[$i]} | awk -F, '{print $1}'`
    ids_idx_opts+=(${i_str})
    echo -e "\t${i_str}:\t${in_ids[i]}"
done
echo -e "\tall:\tall"
echo -e "\ttest-1:\ttest first subject"
echo -e "\ttest-3:\ttest first three subjects"

# note: can't use get_valid_input because of the comma and semicolon input options
echo -en "\n[PROMPT]\twhich subjects do you want to run? (enter index number or hit enter to use default; default: all)\n"
read -r ids_idx
if [[ -z ${ids_idx} ]]; then
    echo "ids_idx: all"
    ids_idx=`filter_ids --study ${study} --ses ${ses} --idx-only 1`
fi

if [[ ${ids_idx} == 'all' ]]; then
    echo "ids_idx: all"
    ids_idx=`filter_ids --study ${study} --ses ${ses} --idx-only 1`
elif [[ ${ids_idx} == 'test-1' ]]; then
    echo "ids_idx: test-1"
    ids_idx=`filter_ids --study ${study} --ses ${ses} --idx-only 1`
    ids_idx=(`echo ${ids_idx} | tr ',' ' '`)
    ids_idx=${ids_idx[0]}
elif [[ ${ids_idx} == 'test-3' ]]; then
    echo "ids_idx: test-3"
    ids_idx=`filter_ids --study ${study} --ses ${ses} --idx-only 1`
    ids_idx=(`echo ${ids_idx} | tr ',' ' '`)
    ids_idx="${ids_idx[0]},${ids_idx[1]},${ids_idx[2]}"
fi
echo "ids_idx: ${ids_idx}"

# ------------------------------------------------------------------------------
# extra options
# ------------------------------------------------------------------------------
echo -en "\n[PROMPT]\textra options? (enter number or hit enter to use default; default: none)\n"
read -r extra_opts
if [[ -z ${extra_opts} ]] || [[ ${extra_opts} == 'none' ]]; then
    extra_opts="''"
else
    extra_opts="'${extra_opts}'"
fi
echo -e "extra_opts:\t${extra_opts}"

# ------------------------------------------------------------------------------
# parallel
# ------------------------------------------------------------------------------
echo -e "\nparallel engines:"
pe_list=(0 sge slurm)
for i in ${!pe_list[@]}; do
    echo -e "\t${i}:\t${pe_list[i]}"
done
pe_idx_opts=`seq 0 $((${#pe_list[@]}-1))`
pe_idx=$(get_valid_input \
-p "which parallel environment do you want to run? (enter index number or hit enter to use default; default: 0 [none])" \
-o ${pe_idx_opts} \
-e 1 \
-d 0)

echo "parellel engine[${pe_idx}]: ${pe_list[$pe_idx]}"

if [[ ${pe_idx} -gt 0 ]]; then
    echo -e "\nBefore running parallel jobs using the parallel engine, please consider the following:"
    echo -e "- please ensure that the parallel settings are properly configured in the script, especially if using none. for more details, please see the following link:\n\thttps://kennedy-rodrigue-wiki.readthedocs.io/en/latest/server.html#parallel-jobs"
    echo -e "- please test a single subject first to ensure the job is running as expected"
    echo -e "\nparallel engine arguments:\n\t0:\tnone"
    if [[ ${pe_idx} -eq 1 ]]; then
        echo -e "${pe_opts_str}\t1:\t${sge_default_opts}"
    elif [[ ${pe_idx} -eq 2 ]]; then
        for i in ${!slurm_opts[@]}; do
            echo -e "\t${i}:\t${slurm_opts_desc[$i]}\t${slurm_opts[$i]}"
        done
        # echo -e "${pe_opts_str}\t1:\t${slurm_default_opts}"
    fi
    echo -en "\n[PROMPT]\tparallel opts? (enter additional parallel engine arguments, use one of the predefined options, or hit enter to use the default option; default: 1) \n"
    read -r pe_opts

    if [[ ${pe_idx} -eq 1 ]] && [[ -z ${pe_opts} ]]; then
    pe_opts="'${sge_default_opts}'"
    elif [[ ${pe_idx} -eq 1 ]] && [[ $(is_numeric ${pe_opts}) -eq 1 ]]; then
        if [[ ${pe_opts} -gt ${#sge_opts[@]} ]]; then
            warning_msg "not a valid sge option (${pe_opts}), using default (1)"
            pe_opts="'${sge_default_opts}'"
        fi
    elif [[ ${pe_idx} -eq 2 ]] && [[ -z ${pe_opts} ]]; then
        pe_opts="'${slurm_opts[1]}'"
    elif [[ ${pe_idx} -eq 2 ]] && [[ $(is_numeric ${pe_opts}) -eq 1 ]]; then
        if [[ ${pe_opts} -gt ${#slurm_opts[@]} ]]; then
            warning_msg "not a valid slurm option (${pe_opts}), using default (1)"
            pe_opts="'${slurm_opts[1]}'"
        fi
    else
        pe_opts="'${pe_opts}'"
    fi
    echo -e "parallel opts:\t${pe_opts}"
fi


if [[ ${pe_idx} -gt 0 ]]; then
    pe_job_limit_opts=`seq 0 32`
    pe_job_limit=$(get_valid_input \
    -p "parallel job limit? (enter number or hit enter to use default; default: 0)" \
    -o ${pe_job_limit_opts} \
    -e 1 \
    -d 0)
    echo -e "parallel job limit: ${pe_job_limit}"
fi

# ------------------------------------------------------------------------------
# overwrite
# ------------------------------------------------------------------------------
overwrite=$(get_valid_input \
-p "do you want to overwrite output files? (enter 0 or 1 or hit enter to use default; default: 0 [no])" \
-o 0 1 \
-e 1 \
-d 0)
echo -e "overwrite:\t${overwrite}"

# ------------------------------------------------------------------------------
# cmd
# ------------------------------------------------------------------------------
in_script_run=`eval echo ${in_scripts[scripts_idx]}`
cmd="flex_wrapper \
--in-scripts ${in_script_run} \
--in-ids ${in_paths[ids]} \
--scripts-idx ${script_idx} \
--ids-idx ${ids_idx} \
--overwrite ${overwrite}"
if [[ ! -z ${extra_opts} ]]; then
    cmd="${cmd} \
    --extra-opts ${extra_opts}"
fi
if [[ ${pe_idx} -gt 0 ]]; then
    cmd="${cmd} \
    --parallel ${pe_list[${pe_idx}]} \
    --parallel-opts ${pe_opts} \
    --parallel-job-limit ${pe_job_limit}"
fi
cmd="${cmd} \
--print 1"
echo -e "\ncommand:\n${cmd}\n"
eval ${cmd} | head -n 100
echo -e "...\n[INFO]\t\tdisplaying only the first 100 lines"

# ------------------------------------------------------------------------------
# execute
# ------------------------------------------------------------------------------
cmd=`echo ${cmd} | sed 's/--print 1/--print 0/'`
run=$(get_valid_input \
-p "ok to run? (enter 0 or 1 or hit enter to use default; default: 0 [no])" \
-o 0 1 \
-e 1 \
-d 0)
echo -e "run:\t${run}"

if [[ ${run} -eq 1 ]]; then
    eval ${cmd}
else
    warning_msg "not running"
fi