Update scripts for running on WCOSS2. (NOAA-EMC#434)

Added scripts to combine surface and atmosphere grib2 files when uses
GEFS forecast as initial condition on wcoss2.

Co-authored-by: ming hu <ming.hu@clogin03.cactus.wcoss2.ncep.noaa.gov>

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -214,9 +214,11 @@ get_extrn_mdl_file_dir_info \
   lbs_spec_intvl_hrs="${lbs_spec_intvl_hrs}" \
   boundary_len_hrs="${boundary_len_hrs}"\
   time_offset_hrs="${time_offset_hrs}" \
+  varname_extrn_mdl_memhead="${GEFS_INPUT_SUBDIR}" \
   varname_extrn_mdl_cdate="extrn_mdl_cdate" \
   varname_extrn_mdl_lbc_spec_fhrs="extrn_mdl_lbc_spec_fhrs" \
   varname_extrn_mdl_fns_on_disk="extrn_mdl_fns_on_disk" \
+  varname_extrn_mdl_fns_on_disk2="extrn_mdl_fns_on_disk2" \
   varname_extrn_mdl_fns_in_arcv="extrn_mdl_fns_in_arcv" \
   varname_extrn_mdl_sysdir="extrn_mdl_sysdir" \
   varname_extrn_mdl_arcv_fmt="extrn_mdl_arcv_fmt" \
@@ -284,6 +286,7 @@ fi
 #
 extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")"
 extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")"
+extrn_mdl_fns_on_disk_str2="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk2[@]}" )")"
 extrn_mdl_fns_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")"
 extrn_mdl_arcv_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fns[@]}" )")"
 extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")"
@@ -294,6 +297,7 @@ $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \
   extrn_mdl_cdate="${extrn_mdl_cdate}" \
   extrn_mdl_lbc_spec_fhrs="${extrn_mdl_lbc_spec_fhrs_str}" \
   extrn_mdl_fns_on_disk="${extrn_mdl_fns_on_disk_str}" \
+  extrn_mdl_fns_on_disk2="${extrn_mdl_fns_on_disk_str2}" \
   extrn_mdl_fns_in_arcv="${extrn_mdl_fns_in_arcv_str}" \
   extrn_mdl_source_dir="${extrn_mdl_source_dir}" \
   extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \

scripts/exregional_bufrsnd.sh

@@ -85,36 +85,9 @@ print_input_args valid_args
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-
-# Specify computational resources.
-    export NODES=2
-    export ntasks=48
-    export ptile=24
-    export threads=1
-    export MP_LABELIO=yes
-    export OMP_NUM_THREADS=$threads
-
-    APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth"
-    ;;
-
-  "WCOSS_DELL_P3")
-
-# Specify computational resources.
-    module list
-    export NODES=1
-    export ntasks=28
-    export ptile=28
-    export threads=1
-    export MP_LABELIO=yes
-    export OMP_NUM_THREADS=$threads
-    module unload netcdf/4.7.4
-    module load NetCDF/4.5.0
-
-    module list
-
-    APRUNC="mpirun"
-    APRUNS="time"
+  "WCOSS2")
+    ncores=$(( NNODES_RUN_BUFRSND*PPN_RUN_BUFRSND))
+    APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_BUFRSND}"
     ;;
 
   "HERA")

scripts/exregional_get_extrn_mdl_files.sh

@@ -61,6 +61,7 @@ valid_args=( \
 "extrn_mdl_cdate" \
 "extrn_mdl_lbc_spec_fhrs" \
 "extrn_mdl_fns_on_disk" \
+"extrn_mdl_fns_on_disk2" \
 "extrn_mdl_fns_in_arcv" \
 "extrn_mdl_source_dir" \
 "extrn_mdl_staging_dir" \
@@ -92,7 +93,9 @@ print_input_args valid_args
 #
 num_files_to_copy="${#extrn_mdl_fns_on_disk[@]}"
 prefix="${extrn_mdl_source_dir}/"
+prefix2="${extrn_mdl_source_dir}/../pgrb2ap5"
 extrn_mdl_fps_on_disk=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" )
+extrn_mdl_fps_on_disk2=( "${extrn_mdl_fns_on_disk2[@]/#/$prefix2}" )
 #
 #-----------------------------------------------------------------------
 #
@@ -232,7 +235,13 @@ model files on disk (extrn_mdl_fns_on_disk) in the source directory
   extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\"
   extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str}"
 
-    ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]}
+    cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir}/.
+    if [ "${ics_or_lbcs}" = "ICS" ]; then
+      cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir}/.
+      more ${extrn_mdl_fps_on_disk2[@]} >>  ${extrn_mdl_staging_dir}/${extrn_mdl_fns_on_disk[@]}
+    else
+      ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]}
+    fi
 
   else
 

scripts/exregional_jedienvar_ioda.sh

@@ -83,23 +83,11 @@ print_input_args valid_args
 #
 case $MACHINE in
 #
-"WCOSS_C" | "WCOSS")
-#
-  . /apps/lmod/lmod/init/sh
-  module purge
-  module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles
-  module load intel/16.1.150 impi/5.1.1.109 netcdf/4.3.0 
-  module list
-
-  ulimit -s unlimited
-  ulimit -a
-  APRUN="mpirun -l -np 1"
-  ;;
-#
-"WCOSS_DELL_P3")
+"WCOSS2")
   ulimit -s unlimited
   ulimit -a
-  APRUN="mpirun -l -np ${IO_LAYOUT_Y}"
+  ncores=$(( NNODES_RUN_JEDIENVAR_IODA*PPN_RUN_JEDIENVAR_IODA ))
+  APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_JEDIENVAR_IODA}"
   ;;
 #
 "HERA")

scripts/exregional_make_grid.sh

@@ -96,25 +96,11 @@ export OMP_STACKSIZE=2048m
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-    { save_shell_opts; set +x; } > /dev/null 2>&1
-    . $MODULESHOME/init/sh
-    module load PrgEnv-intel cfp-intel-sandybridge/1.1.0
-    module list
-    { restore_shell_opts; } > /dev/null 2>&1
-    export NODES=1
-    export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth"
-    export KMP_AFFINITY=disabled
+  "WCOSS2")
     ulimit -s unlimited
     ulimit -a
-    ;;
-
-  "WCOSS_DELL_P3")
-    { save_shell_opts; set +x; } > /dev/null 2>&1
-    module list
-    { restore_shell_opts; } > /dev/null 2>&1
-    export APRUN="mpirun"
-    ulimit -s unlimited
+    ncores=$(( NNODES_MAKE_GRID*PPN_MAKE_GRID))
+    APRUN="mpiexec -n ${ncores} -ppn ${PPN_MAKE_GRID}"
     ;;
 
   "HERA")

scripts/exregional_make_ics.sh

@@ -407,7 +407,7 @@ case "${EXTRN_MDL_NAME_ICS}" in
     fn_grib2="${EXTRN_MDL_FNS[0]}"
     input_type="grib2"
     convert_nst=False
-    if [ "$MACHINE" == "WCOSS_DELL_P3" ]; then
+    if [ "$MACHINE" == "WCOSS2" ]; then
       fn_atm_nemsio="${EXTRN_MDL_FNS[0]}"
       fn_sfc_nemsio="${EXTRN_MDL_FNS[1]}"
     fi

scripts/exregional_make_orog.sh

@@ -96,23 +96,11 @@ export OMP_STACKSIZE=2048m
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-    { save_shell_opts; set +x; } > /dev/null 2>&1
-    . $MODULESHOME/init/sh
-    module load PrgEnv-intel cfp-intel-sandybridge/1.1.0
-    module list
-    { restore_shell_opts; } > /dev/null 2>&1
-    export NODES=1
-    export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth"
-    export KMP_AFFINITY=disabled
+  "WCOSS2")
     ulimit -s unlimited
     ulimit -a
-    ;;
-
-  "WCOSS_DELL_P3")
-    ulimit -s unlimited
-    ulimit -a
-    APRUN="mpirun"
+    ncores=$(( NNODES_MAKE_OROG*PPN_MAKE_OROG ))
+    APRUN="mpiexec -n ${ncores} -ppn ${PPN_MAKE_ORO}"
     ;;
 
   "HERA")
@@ -436,7 +424,7 @@ cp_vrfy "${raw_orog_fp}" "${filtered_orog_fp}"
 # point it to the actual grid file specified by grid_fp.
 #
 
-if [ "${MACHINE}" = "WCOSS_CRAY" ]; then
+if [ "${MACHINE}" = "WCOSS2" ]; then
   ln_vrfy -fs "${grid_fp}" "${filter_dir}/${grid_fn}"
 else
   ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}"

scripts/exregional_make_sfc_climo.sh

@@ -128,19 +128,9 @@ EOF
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-    APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"}
-    ;;
-
-  "WCOSS_DELL_P3")
-# Specify computational resources.
-    export NODES=2
-    export ntasks=48
-    export ptile=24
-    export threads=1
-    export MP_LABELIO=yes
-    export OMP_NUM_THREADS=$threads
-    APRUN="mpirun"
+  "WCOSS2")
+    ncores=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO))
+    APRUN="mpiexec -n ${ncores} -ppn ${PPN_MAKE_SFC_CLIMO}"
     ;;
 
   "HERA")

scripts/exregional_run_enkf.sh

@@ -66,22 +66,16 @@ cycle_type=${cycle_type:-prod}
 
 case $MACHINE in
 #
-"WCOSS_C" | "WCOSS")
+"WCOSS2")
 #
-  module load NCO/4.7.0
   module list
   ulimit -s unlimited
   ulimit -a
-  APRUN="mpirun -l -np ${PE_MEMBER01}"
-  ;;
-#
-"WCOSS_DELL_P3")
-#
-  module load NCO/4.7.0
-  module list
-  ulimit -s unlimited
-  ulimit -a
-  APRUN="mpirun -l -np ${PE_MEMBER01}"
+  export FI_OFI_RXM_SAR_LIMIT=3145728
+  export OMP_STACKSIZE=500M
+  export OMP_NUM_THREADS=1
+  ncores=$(( NNODES_RUN_ENKF*PPN_RUN_ENKF ))
+  APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --cpu-bind core --depth ${OMP_NUM_THREADS}"
   ;;
 #
 "THEIA")

scripts/exregional_run_fcst.sh

@@ -110,7 +110,8 @@ case $MACHINE in
     export MPICH_OFI_STARTUP_CONNECT=1
     export MPICH_OFI_VERBOSE=1
     export MPICH_OFI_NIC_VERBOSE=1
-    ncores=$(( NNODES_RUN_FCST*PPN_RUN_FCST))
+    #ncores=$(( NNODES_RUN_FCST*PPN_RUN_FCST))
+    ncores=${PE_MEMBER01}
     APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_FCST} --cpu-bind core --depth ${OMP_NUM_THREADS}"
     ;;
 
@@ -192,9 +193,6 @@ the grid and (filtered) orography files ..."
 cd_vrfy ${run_dir}/INPUT
 
 relative_or_null=""
-if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ]; then
-  relative_or_null="--relative"
-fi
 
 # Symlink to mosaic file with a completely different name.
 target="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" # must use *mosaic.halo3.nc
@@ -256,9 +254,6 @@ fi
 
 
 relative_or_null=""
-if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then
-  relative_or_null="--relative"
-fi
 
 # Symlink to halo-0 orography file with "${CRES}_" and "halo0" stripped from name.
 target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc"
@@ -438,9 +433,6 @@ static) files in the FIXam directory:
   run_dir = \"${run_dir}\""
 
 relative_or_null=""
-if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then
-  relative_or_null="--relative"
-fi
 
 regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$"
 num_symlinks=${#CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}
@@ -491,9 +483,6 @@ Creating links in the current run directory to cycle-independent model
 input files in the main experiment directory..."
 
 relative_or_null=""
-if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then
-  relative_or_null="--relative"
-fi
 
 ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${run_dir}
 ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${run_dir}

scripts/exregional_run_prepstart_ensfcst.sh

@@ -76,15 +76,9 @@ print_input_args valid_args
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-    export ntasks=1
-    export ptile=1
-    export threads=1
-    APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth"
-    ;;
-
-  "WCOSS_DELL_P3")
-    APRUN="mpirun"
+  "WCOSS2")
+    ncores=$(( NNODES_RUN_PREPSTART*PPN_RUN_PREPSTART))
+    APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_PREPSTART}"
     ;;
 
   "HERA")

scripts/exregional_run_prepstart_fromext.sh

@@ -76,15 +76,8 @@ print_input_args valid_args
 #
 case $MACHINE in
 
-  "WCOSS_CRAY")
-    export ntasks=1
-    export ptile=1
-    export threads=1
-    APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth"
-    ;;
-
-  "WCOSS_DELL_P3")
-    APRUN="mpirun"
+  "WCOSS2")
+    APRUN="mpiexec -n 1 -ppn 1"
     ;;
 
   "HERA")

scripts/exregional_run_recenter.sh

@@ -74,22 +74,16 @@ print_input_args valid_args
 #
 case $MACHINE in
 #
-"WCOSS_C" | "WCOSS")
+"WCOSS2")
 #
-  module load NCO/4.7.0
   module list
   ulimit -s unlimited
   ulimit -a
-  APRUN="mpirun -l -np ${PE_MEMBER01}"
-  ;;
-#
-"WCOSS_DELL_P3")
-#
-  module load NCO/4.7.0
-  module list
-  ulimit -s unlimited
-  ulimit -a
-  APRUN="mpirun -l -np ${PE_MEMBER01}"
+  export FI_OFI_RXM_SAR_LIMIT=3145728
+  export OMP_STACKSIZE=500M
+  export OMP_NUM_THREADS=1
+  ncores=$(( NNODES_RUN_RECENTER*PPN_RUN_RECENTER ))
+  APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_RECENTER} --cpu-bind core --depth ${OMP_NUM_THREADS}"
   ;;
 #
 "THEIA")