From 837f06aa08216a04980f80c8003f4bb44bd4b726 Mon Sep 17 00:00:00 2001 From: "edward.safford" Date: Wed, 16 Mar 2022 18:08:31 +0000 Subject: [PATCH 1/2] Github issue #313. Store data files for global sources in tar files. Completes issue #313 --- util/Radiance_Monitor/CMakeLists.txt | 7 +- .../sorc/radmon_mk_base.fd/CMakeLists.txt | 2 +- .../sorc/radmon_validate_tm.fd/CMakeLists.txt | 2 +- .../data_extract/ush/RadMon_CP_glb.sh | 4 +- .../data_extract/ush/RadMon_DE_glb.sh | 4 +- .../data_extract/ush/RadMon_DE_rgn.sh | 4 +- .../data_extract/ush/nu_find_cycle.pl | 26 +- .../src/radmon_ig_angle.fd/CMakeLists.txt | 2 +- .../src/radmon_ig_bcoef.fd/CMakeLists.txt | 2 +- .../src/radmon_ig_horiz.fd/CMakeLists.txt | 2 +- .../src/radmon_ig_summary.fd/CMakeLists.txt | 2 +- .../src/radmon_ig_time.fd/CMakeLists.txt | 2 +- .../image_gen/ush/RadMon_IG_glb.sh | 24 +- .../image_gen/ush/RadMon_IG_rgn.sh | 4 +- .../image_gen/ush/mk_angle_plots.sh | 164 ++++++++---- .../image_gen/ush/mk_bcoef_plots.sh | 89 +++++-- .../image_gen/ush/mk_bcor_plots.sh | 127 +++++---- .../image_gen/ush/mk_digital_bcoef.sh | 3 +- .../image_gen/ush/mk_horiz_plots.sh | 17 +- .../image_gen/ush/mk_time_plots.sh | 250 +++++++++++------- .../image_gen/ush/nu_find_cycle.pl | 26 +- .../image_gen/ush/plot_angle.sh | 87 +++--- .../image_gen/ush/plot_bcoef.sh | 99 +++---- .../image_gen/ush/plot_bcor.sh | 124 +++------ .../image_gen/ush/plot_summary.sh | 86 +++--- .../image_gen/ush/plot_time.sh | 67 ++--- .../gdas_radmon/fix/gdas_radmon_satype.txt | 2 +- .../gdas_radmon/fix/gdas_radmon_scaninfo.txt | 1 + .../sorc/verf_radang.fd/CMakeLists.txt | 2 +- .../sorc/verf_radbcoef.fd/CMakeLists.txt | 2 +- .../sorc/verf_radbcor.fd/CMakeLists.txt | 2 +- .../sorc/verf_radtime.fd/CMakeLists.txt | 2 +- .../radmon_shared/ush/radmon_verf_angle.sh | 19 +- .../radmon_shared/ush/radmon_verf_bcoef.sh | 16 +- .../radmon_shared/ush/radmon_verf_bcor.sh | 16 +- .../radmon_shared/ush/radmon_verf_time.sh | 17 +- .../nwprod/radmon_shared/ush/rstprod.sh | 18 ++ util/Radiance_Monitor/parm/RadMon_config | 2 + 38 files changed, 744 insertions(+), 581 deletions(-) create mode 100755 util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh diff --git a/util/Radiance_Monitor/CMakeLists.txt b/util/Radiance_Monitor/CMakeLists.txt index 1bc451ea6..e11f186d0 100644 --- a/util/Radiance_Monitor/CMakeLists.txt +++ b/util/Radiance_Monitor/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) # I am top-level project. if( NOT DEFINED ENV{CC} ) @@ -66,6 +66,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) endif() cmake_policy(SET CMP0009 NEW) + cmake_policy(SET CMP0074 NEW) find_package(OpenMP) message("found openmp with flag ${OPENMP_Fortran_FLAGS}") @@ -84,8 +85,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) find_package( W3NCO ) set(BUILD_NCDIAG ON) - set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag/include") - add_subdirectory(src/ncdiag) + set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag") + add_subdirectory(${PROJECT_SOURCE_DIR}/../../src/ncdiag ${PROJECT_BINARY_DIR}/src/ncdiag) set(NCDIAG_LIBRARIES ncdiag ) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) endif() diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt index 7089ff113..603d4acfb 100644 --- a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_MK_BASE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_MK_BASE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_mk_base ) diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt index be546a960..071e872e8 100644 --- a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_VALIDATE_TM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_VALIDATE_TM_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_validate_tm.x ) diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh index b24927322..a5c62347b 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh @@ -192,8 +192,8 @@ if [[ -d ${DATA_LOCATION} ]]; then -o ${logfile} --ntasks=1 --mem=5g ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/CP.${PDY}.${CYC}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi else echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}" diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh index ef689e36a..2a8ca89a4 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh @@ -248,8 +248,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then -o ${logfile} --ntasks=1 --mem=5g ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${pdy}.${cyc}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi else # radstat and/or biascr not found diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh index 97b0e2499..0eac566ac 100755 --- a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh +++ b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh @@ -306,8 +306,8 @@ elif [[ $MY_MACHINE = "hera" ]]; then -j oe -o ${logfile} ${job} elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${PDY}.${cyc}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi diff --git a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl index af13f64d8..10a505adb 100755 --- a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl +++ b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl @@ -89,7 +89,7 @@ } my @mmdirs = grep { /$search_string/ } @alldirs; - + #----------------------------------------------------------------------- # If there are no $run.yyyymmdd subdirectories, then exit without # returning any date string. @@ -157,13 +157,26 @@ $hr_ctr = $hr_ctr - 1; $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}"; -# print " newdir = $newdir \n"; - if( -d $newdir ) { opendir DIR, $newdir or die "Cannot open the current directory: $!"; - my @timefiles = grep { /ieee_d/ } readdir DIR; + my @dirfiles = readdir DIR; + my @timefiles = grep { /ieee_d/ } @dirfiles; + + # If no *ieee_d* files were found then look for the compressed format of + # radmon_*.tar files instead. If found then use the parent directories + # to establish the cycle time. + # + if( $#timefiles < 0 ) { + my @tarfiles = grep { /radmon_time.tar/ } @dirfiles; + + if( $#tarfiles >= 0 ) { + my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}"; + push( @timefiles, $cycle ); + } + } + if( $#timefiles >= 0 ) { my @sorttime = sort( @timefiles ); @@ -201,18 +214,13 @@ } } } - } } while $hr_ctr > 0 && $found_cycle == 0; -# print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n"; - if( $cyc == 0 && $ctr >= $end_ctr ){ -# print " exiting from if\n"; $exit_flag = 1; } elsif( $cyc == 1 && $ctr <= $end_ctr ){ -# print " exiting from elsif\n"; $exit_flag = 1; } diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt index 44fcad075..e585be7a3 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_angle ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt index ca50c0821..06e447e3d 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_bcoef ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt index d74885b27..07e20110d 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_HORIZ_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_HORIZ_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_horiz ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt index dde3b6613..98dd52d74 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_SUMMARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_SUMMARY_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_summary ) diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt index a14d8566e..3f4bfa4cc 100644 --- a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_IG_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_IG_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_time ) diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh index c5b2f87d4..6ab9b4f70 100755 --- a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh +++ b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh @@ -109,6 +109,10 @@ if [[ $? -ne 0 ]]; then fi +if [[ ! -d ${IMGNDIR} ]]; then + mkdir -p ${IMGNDIR} +fi + #-------------------------------------------------------------------- # Determine cycle to plot. Exit if cycle is > last available # data. @@ -183,7 +187,6 @@ export PDY=`echo $PDATE|cut -c1-8` #-------------------------------------------------------------------- # Locate ieee_src in $TANKverf and verify data files are present # - ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then @@ -199,7 +202,16 @@ if [[ ! -d ${ieee_src} ]]; then exit 5 fi +#----------------------------------------------------- +# check $ieee_src for data files. If none are found +# check contents of the radmon_angle.tar file. +# nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'` +if [[ $nfile_src -le 0 ]]; then + if [[ -e ${ieee_src}/radmon_angle.tar ]]; then + nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l` + fi +fi if [[ $nfile_src -le 0 ]]; then echo " Missing ieee_src files, nfile_src = ${nfile_src}, aborting plot" @@ -238,6 +250,9 @@ fi # the $satype_file can't be found. # test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*` +if [[ $test_list = "" ]]; then + test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d` +fi for test in ${test_list}; do this_file=`basename $test` @@ -332,11 +347,12 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then export WEBDIR=${WEBDIR}/${RADMON_SUFFIX}/${RUN}/pngs if [[ $MY_MACHINE = "wcoss2" ]]; then - cmdfile=transfer_cmd + cmdfile="${PLOT_WORK_DIR}/transfer_cmd" echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile + chmod 755 $cmdfile - $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \ - -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} + $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err \ + -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} else $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \ -R affinity[core] -J ${jobname} -cwd ${PWD} -b $run_time ${job} diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh index c8eec4d35..665486150 100755 --- a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh +++ b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh @@ -286,8 +286,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then cmdfile=transfer_cmd echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile - $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \ - -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} + $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err + -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile} else $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \ diff --git a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh index 72b9348e8..a46eaf64a 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh @@ -39,54 +39,92 @@ allmissing=1 cycdy=$((24/$CYCLE_INTERVAL)) # number cycles per day ndays=$(($NUM_CYCLES/$cycdy)) # number of days in plot period -echo SATYPE=$SATYPE - for type in ${SATYPE}; do found=0 - finished=0 - ctr=$ndays test_day=$PDATE - - while [[ $found -eq 0 && $finished -ne 1 ]]; do - - if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next + ctr=$ndays + + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do + + if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` + else pdy=`echo $test_day|cut -c1-8` - ieee_src=${TANKverf}/radmon.${pdy} + cyc=`echo $test_day|cut -c9-10` + fi + + + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then + found=1 + else - pdy=`echo $test_day|cut -c1-8` - ieee_src=${TANKverf}/${RUN}.${pdy}/${CYC}/${MONITOR} + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${pdy} fi - fi - - echo "ieee_src = with pdy = $pdy" - - if [[ -s ${ieee_src}/angle.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/angle.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/angle.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/angle.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #---------------------------------------------------- + # Determine if the angle files are in a tar file + # and, if so, extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_angle.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_angle.tar ${ctl_list} + cd ${cwd} + fi + fi + + #------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'angle' from the file name. + # + ctl_files=`ls $ieee_src/angle.$type*.ctl*` + prefix='angle.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #---------------------------------------------------------------- + # If there's a radmon_angle.tar archive in ${ieee_src} then + # delete the extracted *ctl* files to leave just the tar files. + # + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/angle.${type}.ctl* + rm -f ${ieee_src}/angle.${type}_anl.ctl* + fi - elif [[ -s ${ieee_src}/angle.${type}.ctl ]]; then - $NCP ${ieee_src}/angle.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/angle.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/angle.${type}_anl.ctl ${imgndir}/${type}_anl.ctl - fi - found=1 fi - - if [[ $found -eq 0 ]]; then + + if [[ ${found} -eq 0 ]]; then + #------------------------------------------ + # Step to the previous day and try again. + # if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` - ctr=$(($ctr-1)) - else - finished=1 + ctr=$(($ctr-1)) fi fi done @@ -95,8 +133,10 @@ for type in ${SATYPE}; do allmissing=0 found=1 fi + done + if [[ $allmissing = 1 ]]; then echo ERROR: Unable to plot. All angle control files are missing from ${TANKverf} for requested date range. exit 2 @@ -107,26 +147,26 @@ fi # Update the time definition (tdef) line in the angle control # files. # -for type in ${SATYPE}; do - if [[ -s ${imgndir}/${type}.ctl.${Z} ]]; then - ${UNCOMPRESS} ${imgndir}/${type}.ctl.${Z} +for sat in ${SATYPE}; do + if [[ -s ${imgndir}/${sat}.ctl.${Z} ]]; then + ${UNCOMPRESS} ${imgndir}/${sat}.ctl.${Z} fi - ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} -done + ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${sat}.ctl ${START_DATE} ${NUM_CYCLES} - -#------------------------------------------------------------------- -# Separate the sources with a large number of channels. These will -# be submitted in dedicated jobs, while the sources with a smaller -# number of channels will be submitted together. -# -for sat in ${SATYPE}; do + #------------------------------------------------------------------- + # Separate the sources with a large number of channels. These will + # be submitted in dedicated jobs, while the sources with a smaller + # number of channels will be submitted together. + # nchanl=`cat ${imgndir}/${sat}.ctl | gawk '/title/{print $NF}'` + if [[ $nchanl -lt 100 ]]; then satlist=" $sat $satlist " else big_satlist=" $sat $big_satlist " fi + + ${COMPRESS} ${imgndir}/${sat}.ctl done echo "" @@ -180,9 +220,7 @@ done chmod 755 ${cmdfile} echo "CMDFILE: ${cmdfile}" -ntasks=`cat $cmdfile|wc -l ` wall_tm="0:20" - if [[ ${MY_MACHINE} = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M 500 -W ${wall_tm} \ -R "affinity[core]" -J ${jobname} -cwd ${PWD} $cmdfile @@ -200,8 +238,8 @@ elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then -J ${jobname} -cwd ${PWD} $cmdfile elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} fi @@ -218,7 +256,7 @@ echo "starting big_satlist" for sat in ${big_satlist}; do echo processing $sat in $big_satlist - if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then + if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then cmdfile=${PLOT_WORK_DIR}/cmdfile_pangle_${sat} if [[ -e ${cmdfile} ]]; then @@ -229,23 +267,32 @@ for sat in ${big_satlist}; do jobname=plot_${RADMON_SUFFIX}_ang_${sat} logfile=${LOGdir}/plot_angle_${sat}.log - ntasks=`cat $cmdfile|wc -l ` - echo "ntasks = $ntasks" + if [[ -e ${logfile} ]]; then + rm ${logfile} + fi wall_tm="0:30" if [[ $MY_MACHINE = "wcoss_d" ]]; then - mem="12000" - $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M ${mem} -W ${wall_tm} \ - -R "affinity[core]" -J ${jobname} -cwd ${PWD} $cmdfile + $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -W ${wall_tm} \ + -R "affinity[core]" -R "rusage[mem=10000]" -J ${jobname} -cwd ${PWD} $cmdfile elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M 600 -W ${wall_tm} \ -J ${jobname} -cwd ${PWD} $cmdfile - fi + elif [[ $MY_MACHINE = "wcoss2" ]]; then + errfile=${LOGdir}/plot_angle_${sat}.err + if [[ -e ${errfile} ]]; then + rm ${errfile} + fi + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${sat}.err \ + -V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile} + fi - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || $MY_MACHINE = "s4" ]]; then # hera|jet|s4, submit 1 job for each sat/list item + #--------------------------------------------------- + # hera|jet|s4, submit 1 job for each sat/list item + elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || $MY_MACHINE = "s4" ]]; then ii=0 logfile=${LOGdir}/plot_angle_${sat}.log @@ -260,7 +307,10 @@ for sat in ${big_satlist}; do (( ii=ii+1 )) done - if [[ ! $MY_MACHINE = "jet" ]]; then + if [[ $MY_MACHINE = "hera" ]]; then + $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + --mem=0 --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ $MY_MACHINE = "s4" ]]; then $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ --wrap "srun -l --multi-prog ${cmdfile}" else diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh index 05e175470..8eedb783a 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh @@ -38,50 +38,83 @@ test_day=$PDATE for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ $found -eq 0 && $finished -ne 1 ]]; do + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${PDY}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${PDY} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${PDY} - fi - + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then + found=1 + else + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi - if [[ -s ${ieee_src}/bcoef.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcoef.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/bcoef.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcoef.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} + using_tar=0 + #---------------------------------------------------- + # Determine if the bcoef files are in an tar file. + # if so extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_bcoef.tar ${ctl_list} + cd ${cwd} + fi fi - found=1 - elif [[ -s ${ieee_src}/bcoef.${type}.ctl ]]; then - $NCP ${ieee_src}/bcoef.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/bcoef.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/bcoef.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'bcoef.' from the file name. + # + ctl_files=`ls $ieee_src/bcoef.$type*.ctl*` + prefix='bcoef.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------- + # If there's a radmon_bcoef.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/bcoef.${type}.ctl* + rm -f ${ieee_src}/bcoef.${type}_anl.ctl* fi - found=1 + fi - if [[ $found -eq 0 ]]; then + if [[ $found -eq 0 ]]; then if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi done @@ -135,7 +168,7 @@ elif [[ $MY_MACHINE = "jet" ]]; then -p ${RADMON_PARTITION} -o ${logfile} -D . $IG_SCRIPTS/plot_bcoef.sh elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e $LOGdir/plot_bcoef.err -V \ -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} $IG_SCRIPTS/plot_bcoef.sh fi diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh index ba41fe9d9..2751d4501 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh @@ -29,7 +29,6 @@ fi # are found or we run out of dates to check. Report an error to # the log file and exit if no ctl files are found. # -allmissing=1 pdy=`echo $PDATE|cut -c1-8` cyc=`echo $PDATE|cut -c9-10` @@ -40,65 +39,96 @@ test_day=$PDATE for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ $found -eq 0 && $finished -ne 1 ]]; do - if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do + + if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - echo "Unable to locate ieee_src directory, exiting mk_time_plots.sh." - exit 12 - fi - - - if [[ -s ${ieee_src}/bcor.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcor.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/bcor.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/bcor.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi + #------------------------------------------------------------------ + # Check to see if the *ctl* files for this $type are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || + ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then found=1 - elif [[ -s ${ieee_src}/bcor.${type}.ctl ]]; then - $NCP ${ieee_src}/bcor.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/bcor.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/bcor.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + else + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #--------------------------------------------------------- + # Determine if the bcor files are in a tar file. + # if so extract the ctl files for this $type. + # + # Note that the ctl files are moved back to ${ieee_src} + # so the code block that follows will work with both + # tarred and non-tarred storage schemes. + # + if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + tar -xf ./radmon_bcor.tar ${ctl_list} + cd ${cwd} + fi + fi + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'bcor.' from the file name. + # + ctl_files=`ls $ieee_src/bcor.$type*.ctl*` + prefix='bcor.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------ + # If there's a radmon_bcor.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/bcor.${type}*.ctl* + fi + fi - if [[ $found -eq 0 ]]; then + if [[ ${found} -eq 0 ]]; then # if not found try previous day if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi - done - if [[ -s ${imgndir}/${type}.ctl.${Z} || -s ${imgndir}/${type}.ctl ]]; then - allmissing=0 - found=1 - fi + done + done -if [[ $allmissing = 1 ]]; then +nctl=`ls ${imgndir}/*ctl* -1 | wc -l` +if [[ $nctl -le 0 ]]; then echo ERROR: Unable to plot. All bcor control files are missing. exit 14 fi @@ -112,8 +142,8 @@ for type in ${SATYPE}; do if [[ -s ${imgndir}/${type}.ctl.${Z} ]]; then ${UNCOMPRESS} ${imgndir}/${type}.ctl.${Z} fi - ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} + ${IG_SCRIPTS}/update_ctl_tdef.sh ${imgndir}/${type}.ctl ${START_DATE} ${NUM_CYCLES} done for sat in ${SATYPE}; do @@ -142,9 +172,8 @@ cd ${PLOT_WORK_DIR} #------------------------------------------------------------------------- -# Loop over satellite/instruments. Submit poe job to make plots. Each task handles -# a single satellite/insrument. - +# Loop over satellite/instruments and submit job. +# suffix=a cmdfile=cmdfile_pbcor_${suffix} jobname=plot_${RADMON_SUFFIX}_bcor_${suffix} @@ -189,6 +218,10 @@ elif [[ $MY_MACHINE = "jet" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} \ -p ${RADMON_PARTITION} --time=2:00:00 --wrap "srun -l --multi-prog ${cmdfile}" +elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} + fi @@ -246,8 +279,8 @@ for sat in ${bigSATLIST}; do -p ${RADMON_PARTITION} --time=1:00:00 --wrap "srun -l --multi-prog ${cmdfile}" elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \ + -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} fi echo "submitted $sat" @@ -255,4 +288,4 @@ done echo "end mk_bcor_plots.sh" -exit 0 +exit diff --git a/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh b/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh index 9da421879..1d3af5aa4 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_digital_bcoef.sh @@ -91,8 +91,7 @@ EOF #------------------------------ # mv output files to IMGNDIR # - cp -f ${type}.*.bcoef.txt ${IMGNDIR}/bcoef/. -# cp -f ${type}.chan.txt ${IMGNDIR}/time/. + mv ${type}.*.bcoef.txt ${IMGNDIR}/bcoef/. done diff --git a/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh index 009adfed9..d6b5a0b53 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh @@ -160,7 +160,7 @@ done # submit the plot jobs # -if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then +if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then cmdfile="./cmdfile_horiz_${RADMON_SUFFIX}_${PID}" logfile=${LOGdir}/horiz_${PID}.log rm -f $cmdfile @@ -177,9 +177,11 @@ if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 0:45 -J ${jobname} -cwd ${PWD} $cmdfile - else + + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 0:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + fi else # hera|jet|s4 @@ -225,9 +227,14 @@ for sat in ${bigSATLIST}; do if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \ + -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile} else $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \ -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile @@ -248,9 +255,15 @@ for sat in ${bigSATLIST}; do if [[ $MY_MACHINE = "wcoss_d" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \ -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile + elif [[ $MY_MACHINE = "wcoss_c" ]]; then $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \ -J ${jobname} -cwd ${PWD} $cmdfile + + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \ + -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile} + else $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \ -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile diff --git a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh index c1ccf2c11..66d626c36 100755 --- a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh +++ b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh @@ -27,64 +27,104 @@ fi # then $TANKverf/radmon.$pdy. # allmissing=1 -pdy=`echo $PDATE|cut -c1-8` -cyc=`echo $PDATE|cut -c9-10` cycdy=$((24/$CYCLE_INTERVAL)) # number cycles per day ndays=$(($NUM_CYCLES/$cycdy)) # number days in plot period test_day=$PDATE +#-------------------------------------------------------- +# Verify there are control files available in $imgndir +# for everything in $SATYPE. +# for type in ${SATYPE}; do found=0 - finished=0 test_day=$PDATE ctr=$ndays - while [[ ${found} -eq 0 && $finished -ne 1 ]]; do + while [[ ${found} -eq 0 && $ctr -gt 0 ]]; do if [[ $REGIONAL_RR -eq 1 ]]; then # REGIONAL_RR stores hrs 18-23 in next tdate=`$NDATE +6 ${test_day}` # day's radmon.yyymmdd directory - pdy=`echo $test_day|cut -c1-8` + pdy=`echo $tdate|cut -c1-8` + cyc=`echo $tdate|cut -c9-10` else pdy=`echo $test_day|cut -c1-8` + cyc=`echo $test_day|cut -c9-10` fi - ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${RUN}.${pdy} - fi - if [[ ! -d ${ieee_src} ]]; then - ieee_src=${TANKverf}/${MONITOR}.${pdy} - fi - if [[ -s ${ieee_src}/time.${type}.ctl.${Z} ]]; then - $NCP ${ieee_src}/time.${type}.ctl.${Z} ${imgndir}/${type}.ctl.${Z} - if [[ -s ${ieee_src}/time.${type}_anl.ctl.${Z} ]]; then - $NCP ${ieee_src}/time.${type}_anl.ctl.${Z} ${imgndir}/${type}_anl.ctl.${Z} - fi + #--------------------------------------------------- + # Check to see if the *ctl* files are in $imgndir + # + nctl=`ls ${imgndir}/${type}*ctl* -1 | wc -l` + if [[ ( $USE_ANL -eq 1 && $nctl -ge 2 ) || ( $USE_ANL -eq 0 && $nctl -ge 1 ) ]]; then found=1 - elif [[ -s ${ieee_src}/time.${type}.ctl ]]; then - $NCP ${ieee_src}/time.${type}.ctl ${imgndir}/${type}.ctl - if [[ -s ${ieee_src}/time.${type}_anl.ctl ]]; then - $NCP ${ieee_src}/time.${type}_anl.ctl ${imgndir}/${type}_anl.ctl + + else + + #------------------------- + # Locate $ieee_src + # + ieee_src=${TANKverf}/${RUN}.${pdy}/${cyc}/${MONITOR} + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy}/${MONITOR} fi - found=1 + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${RUN}.${pdy} + fi + if [[ ! -d ${ieee_src} ]]; then + ieee_src=${TANKverf}/${MONITOR}.${pdy} + fi + + using_tar=0 + #-------------------------------------------------- + # Determine if the time files are in an tar file. + # if so extract the ctl files for this $type. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + using_tar=1 + ctl_list=`tar -tf ${ieee_src}/radmon_time.tar | grep $type | grep ctl` + if [[ ${ctl_list} != "" ]]; then + cwd=`pwd` + cd ${ieee_src} + ctl_list=`tar -tf ./radmon_time.tar | grep $type | grep ctl` + tar -xf ${ieee_src}/radmon_time.tar ${ctl_list} + cd ${cwd} + fi + fi + + #-------------------------------------------------- + # Copy the *ctl* files to $imgndir, dropping + # 'time.' from the file name. + # + ctl_files=`ls $ieee_src/time.$type*.ctl*` + prefix='time.' + for file in $ctl_files; do + newfile=`basename $file | sed -e "s/^$prefix//"` + $NCP ${file} ${imgndir}/${newfile} + found=1 + done + + #------------------------------------------------------ + # If there's a radmon_time.tar archive in ${ieee_src} + # then delete the extracted *ctl* files. + if [[ $using_tar -eq 1 ]]; then + rm -f ${ieee_src}/time.${type}.ctl* + rm -f ${ieee_src}/time.${type}_anl.ctl* + fi + fi if [[ ${found} -eq 0 ]]; then if [[ $ctr -gt 0 ]]; then test_day=`$NDATE -24 ${pdy}00` ctr=$(($ctr-1)) - else - finished=1 fi fi done + if [[ -s ${imgndir}/${type}.ctl.${Z} || -s ${imgndir}/${type}.ctl ]]; then allmissing=0 found=1 @@ -152,7 +192,7 @@ elif [[ ${MY_MACHINE} = "jet" ]]; then --partition ${RADMON_PARTITION} -o ${logfile} ${IG_SCRIPTS}/plot_summary.sh elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_summary.err -V \ -l select=1:mem=1g -l walltime=10:00 -N ${jobname} ${IG_SCRIPTS}/plot_summary.sh fi @@ -182,58 +222,58 @@ list="count penalty omgnbc total omgbc" # Build command file and submit plot job for intruments not on # the bigSAT list. # +suffix=a +jobname=plot_${RADMON_SUFFIX}_tm_${suffix} - suffix=a - cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${suffix} - jobname=plot_${RADMON_SUFFIX}_tm_${suffix} - logfile=${LOGdir}/plot_time_${suffix}.log - +cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${suffix} +if [[ -e ${cmdfile} ]]; then rm -f $cmdfile +fi + +logfile=${LOGdir}/plot_time_${suffix}.log +if [[ -e ${logfile} ]]; then rm ${logfile} +fi >$cmdfile - ctr=0 - - for sat in ${SATLIST}; do - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then - echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile - else - echo "$IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile - fi - ((ctr=ctr+1)) - done - - chmod 755 $cmdfile +ctr=0 - if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then - wall_tm="2:30" +for sat in ${SATLIST}; do + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then + echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile else - wall_tm="0:45" + echo "$IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile fi + ((ctr=ctr+1)) +done +chmod 755 $cmdfile - if [[ $MY_MACHINE = "wcoss_d" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ - -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} +wall_tm="0:45" +if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then + wall_tm="2:30" +fi - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then - echo "using ctr = ${ctr}" - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - --wrap "srun -l --multi-prog ${cmdfile}" +if [[ $MY_MACHINE = "wcoss_d" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ + -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} - elif [[ $MY_MACHINE = "jet" ]]; then - echo "using ctr = ${ctr}" - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ - -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" +elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ + --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ - -J ${jobname} -cwd ${PWD} ${cmdfile} +elif [[ $MY_MACHINE = "jet" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ + -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} - fi +elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ + -J ${jobname} -cwd ${PWD} ${cmdfile} + +elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${suffix}.err -V \ + -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile} +fi @@ -244,54 +284,62 @@ list="count penalty omgnbc total omgbc" # data that a separate job for each provides a faster solution. # #--------------------------------------------------------------------------- - for sat in ${bigSATLIST}; do - - cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${sat} - jobname=plot_${RADMON_SUFFIX}_tm_${sat} - logfile=${LOGdir}/plot_time_${sat}.log +for sat in ${bigSATLIST}; do + jobname=plot_${RADMON_SUFFIX}_tm_${sat} - rm -f ${logfile} + cmdfile=${PLOT_WORK_DIR}/cmdfile_ptime_${sat} + if [[ -e ${cmdfile} ]]; then rm -f ${cmdfile} + fi - ctr=0 - for var in $list; do - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then - echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile - else - echo "$IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile - fi - ((ctr=ctr+1)) - done - chmod 755 $cmdfile + logfile=${LOGdir}/plot_time_${sat}.log + if [[ -e ${logfile} ]]; then + rm -f ${logfile} + fi - if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then - wall_tm="2:30" + ctr=0 + for var in $list; do + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" ]]; then + echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile else - wall_tm="1:00" + echo "$IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile fi + ((ctr=ctr+1)) + done + chmod 755 $cmdfile - if [[ $MY_MACHINE = "wcoss_d" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ - -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} + wall_tm="1:00" + if [[ $PLOT_ALL_REGIONS -eq 1 || $ndays -gt 30 ]]; then + wall_tm="2:30" + fi - elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then - $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ - -J ${jobname} -cwd ${PWD} ${cmdfile} + if [[ $MY_MACHINE = "wcoss_d" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -R affinity[core] -o ${logfile} \ + -W ${wall_tm} -J ${jobname} -cwd ${PWD} ${cmdfile} - elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ - --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then + $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W ${wall_tm} \ + -J ${jobname} -cwd ${PWD} ${cmdfile} + + elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "jet" ]]; then - $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ - -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" + elif [[ $MY_MACHINE = "jet" ]]; then + $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ + -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "wcoss2" ]]; then - $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \ - -l select=1:mem=1g -l walltime=1:30:00 -N ${jobname} ${cmdfile} + elif [[ $MY_MACHINE = "wcoss2" ]]; then + logfile=${LOGdir}/plot_time_${sat}.log + if [[ -e ${logfile} ]]; then + rm ${logfile} fi - done + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${sat}.err -V \ + -l select=1:mem=1g -l walltime=1:30:00 -N ${jobname} ${cmdfile} + fi + +done echo End mk_time_plots.sh diff --git a/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl b/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl index af13f64d8..10a505adb 100755 --- a/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl +++ b/util/Radiance_Monitor/image_gen/ush/nu_find_cycle.pl @@ -89,7 +89,7 @@ } my @mmdirs = grep { /$search_string/ } @alldirs; - + #----------------------------------------------------------------------- # If there are no $run.yyyymmdd subdirectories, then exit without # returning any date string. @@ -157,13 +157,26 @@ $hr_ctr = $hr_ctr - 1; $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}"; -# print " newdir = $newdir \n"; - if( -d $newdir ) { opendir DIR, $newdir or die "Cannot open the current directory: $!"; - my @timefiles = grep { /ieee_d/ } readdir DIR; + my @dirfiles = readdir DIR; + my @timefiles = grep { /ieee_d/ } @dirfiles; + + # If no *ieee_d* files were found then look for the compressed format of + # radmon_*.tar files instead. If found then use the parent directories + # to establish the cycle time. + # + if( $#timefiles < 0 ) { + my @tarfiles = grep { /radmon_time.tar/ } @dirfiles; + + if( $#tarfiles >= 0 ) { + my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}"; + push( @timefiles, $cycle ); + } + } + if( $#timefiles >= 0 ) { my @sorttime = sort( @timefiles ); @@ -201,18 +214,13 @@ } } } - } } while $hr_ctr > 0 && $found_cycle == 0; -# print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n"; - if( $cyc == 0 && $ctr >= $end_ctr ){ -# print " exiting from if\n"; $exit_flag = 1; } elsif( $cyc == 1 && $ctr <= $end_ctr ){ -# print " exiting from elsif\n"; $exit_flag = 1; } diff --git a/util/Radiance_Monitor/image_gen/ush/plot_angle.sh b/util/Radiance_Monitor/image_gen/ush/plot_angle.sh index 75d1baddb..94dfd55e8 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_angle.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_angle.sh @@ -52,18 +52,10 @@ edate0=`echo $edate|cut -c1-8` #-------------------------------------------------------------------- # Copy control files to $wrkdir - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/angle -else - ctldir=$TANKverf/angle -fi - -echo ctldir = $ctldir +ctldir=$IMGNDIR/angle for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ done ${UNCOMPRESS} *.ctl.${Z} @@ -74,20 +66,30 @@ ${UNCOMPRESS} *.ctl.${Z} # for type in ${SATYPE2}; do + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} + cdate=$bdate - while [[ $cdate -le $edate ]] ; do + + #------------------------------------- + # Locate and copy data files. + # + while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` + day=`echo $cdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} fi @@ -98,46 +100,39 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - - if [[ -d ${ieee_src} ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.angle.${type}.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/angle.${type}.${cdate}.ieee_d - else - test_file=${ieee_src}/angle.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.angle.${type}_anl.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/angle.${type}_anl.${cdate}.ieee_d - else - test_file2=${ieee_src}/angle.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_angle.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_angle.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_angle.tar ${files} fi + else + files=`ls ${ieee_src}/angle.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} $wrkdir/*.ieee_d.${Z} + #----------------------------------------------- + # Remove 'angle.' from the *ieee_d file names. + # + prefix="angle." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + #----------------------------------------------------------------------- # mk_digital_ang.sh produces the text files used by the js/html files # to generate the interactive charts diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh index 0a21b5a55..d9c783c1f 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_bcoef.sh @@ -33,46 +33,39 @@ edate0=`echo $edate|cut -c1-8` #-------------------------------------------------------------------- # Copy executable and control files to $tmpdir - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/bcoef -else - ctldir=$TANKverf/bcoef -fi - -echo ctldir = $ctldir - +ctldir=$IMGNDIR/bcoef +$NCP ${IG_SCRIPTS}/mk_digital_bcoef.sh . #-------------------------------------------------------------------- # Loop over satellite types. Copy data files, create plots and # place on the web server. # -# Data file location may either be in angle, bcoef, bcor, and time -# subdirectories under $TANKverf, or in the Operational organization -# of radmon.YYYYMMDD directories under $TANKverf - - -$NCP ${IG_SCRIPTS}/mk_digital_bcoef.sh . - for type in ${SATYPE}; do - $NCP $ctldir/${type}.ctl* ./ - ${UNCOMPRESS} ${type}.ctl.${Z} + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate + + #------------------------------------- + # Locate and copy data files. + # while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` + day=`echo $cdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} @@ -84,46 +77,39 @@ for type in ${SATYPE}; do ieee_src=${TANKverf}/${RUN}.${day} fi - if [[ -d ${ieee_src} ]]; then - - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.bcoef.${type}.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/bcoef.${type}.${cdate}.ieee_d - else - test_file=${ieee_src}/bcoef.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then -# test_file=${ieee_src}/${rgnHH}.bcoef.${type}_anl.${cdate}.ieee_d.${rgnTM} - test_file=${ieee_src}/bcoef.${type}_anl.${cdate}.ieee_d - else - test_file2=${ieee_src}/bcoef.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the ieee_d files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_bcoef.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_bcoef.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_bcoef.tar ${files} fi - + else + files=`ls ${ieee_src}/bcoef.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'bcoef.' from the *ieee_d file names. + # + prefix="bcoef." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then list="mean atmpath clw lapse2 lapse cos_ssmis sin_ssmis emiss ordang4 ordang3 ordang2 ordang1" for var in $list; do @@ -140,23 +126,24 @@ EOF if [[ ! -d ${IMGNDIR}/bcoef ]]; then mkdir -p ${IMGNDIR}/bcoef fi - cp -f *.png ${IMGNDIR}/bcoef + mv *.png ${IMGNDIR}/bcoef fi #-------------------------------------------------------------------------- # mk_digital_bcoef.sh produces the data files needed by the js/html files # to generate the interactive charts. + # ./mk_digital_bcoef.sh ${type} - + rm -f ${type}*ieee_d done #-------------------------------------------------------------------- # Clean $tmpdir. Submit done job. - +# cd $tmpdir cd ../ rm -rf $tmpdir diff --git a/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh b/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh index 83702c9e1..feb17576f 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_bcor.sh @@ -17,7 +17,7 @@ echo "begin plot_bcor.sh" #------------------------------------------------------------------ # Set environment variables. - +# word_count=`echo $PTYPE | wc -w` echo word_count = $word_count @@ -43,47 +43,39 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` - -#-------------------------------------------------------------------- -# Set ctldir to point to control file directory - -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/bcor -else - ctldir=$TANKverf/bcor -fi - -echo ctldir = $ctldir - +ctldir=$IMGNDIR/bcor #-------------------------------------------------------------------- # Loop over satellite types. Copy data files, create plots and # place on the web server. # -# Data file location may either be in angle, bcoef, bcor, and time -# subdirectories under $TANKverf, or in the Operational organization -# of radmon.YYYYMMDD directories under $TANKverf - for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate + + #------------------------------------- + # Locate and copy data files. + # while [[ $cdate -le $edate ]]; do + if [[ $REGIONAL_RR -eq 1 ]]; then tdate=`$NDATE +6 $cdate` - day=`echo $tdate | cut -c1-8 ` + day=`echo $tdate | cut -c1-8` cyc=`echo $cdate | cut -c9-10` . ${IG_SCRIPTS}/rr_set_tz.sh $cyc else - day=`echo $cdate | cut -c1-8 ` - cyc=`echo $cdate | cut -c9-10 ` + day=`echo $cdate | cut -c1-8` + cyc=`echo $cdate | cut -c9-10` fi + #---------------------------------------------------- + # Attempt to locate the parent directory for the + # extracted ieee data files. + # ieee_src=${TANKverf}/${RUN}.${day}/${cyc}/${MONITOR} - if [[ ! -d ${ieee_src} ]]; then ieee_src=${TANKverf}/${RUN}.${day}/${MONITOR} fi @@ -94,39 +86,38 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - nfile_src=`ls -l ${ieee_src}/*${cdate}*ieee_d* | egrep -c '^-'` - - echo "nfile_src = $nfile_src" - - if [[ -d ${ieee_src} ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then - test_file=${ieee_src}/${rgnHH}.bcor.${type}.${cdate}.ieee_d.${rgnTM} - else - test_file=${ieee_src}/bcor.${type}.${cdate}.ieee_d - fi - - if [[ $USE_ANL = 1 ]]; then - if [[ $REGIONAL_RR -eq 1 ]]; then - test_file=${ieee_src}/${rgnHH}.bcor.${type}_anl.${cdate}.ieee_d.${rgnTM} - else - test_file2=${ieee_src}/bcor.${type}_anl.${cdate}.ieee_d - fi - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi + + #----------------------------------------------------------- + # Now locate this cycle's data files, first checking for + # a tar file, and copy them locally. + # + if [[ -s ${ieee_src}/radmon_bcor.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_bcor.tar | grep ${type} | grep ieee_d` + tar -xf ${ieee_src}/radmon_bcor.tar ${files} + + else + files=`ls ${ieee_src}/bcor.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi - adate=`$NDATE +${CYCLE_INTERVAL} $cdate` + adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` cdate=$adate done + ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'bcor.' from the *ieee_d file names. + # + prefix="bcor." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + for var in ${PTYPE}; do echo $var if [ "$var" = 'count' ]; then @@ -147,47 +138,20 @@ EOF $GRADS -bpc "run ${tmpdir}/${type}_${var}.gs" done -#-------------------------------------------------------------------- -# Delete data files - -# rm -f ${type}.ieee_d -# rm -f ${type}.ctl - done #-------------------------------------------------------------------- # Copy image files to $IMGNDIR to set up for mirror to web server. -# Delete images and data files. - -if [[ ! -d ${IMGNDIR}/bcor ]]; then - mkdir -p ${IMGNDIR}/bcor -fi -cp -r *.png ${IMGNDIR}/bcor - +# +mv *.png ${IMGNDIR}/bcor/. #-------------------------------------------------------------------- # Clean $tmpdir - +# cd $tmpdir cd ../ rm -rf $tmpdir - -#-------------------------------------------------------------------- -# If this is the last bcor plot job to finish then rm PLOT_WORK_DIR. -# - -#count=`ls ${LOADLQ}/*plot*_${RADMON_SUFFIX}* | wc -l` -#complete=`grep "COMPLETED" ${LOADLQ}/*plot*_${RADMON_SUFFIX}* | wc -l` - -#running=`expr $count - $complete` - -#if [[ $running -eq 1 ]]; then -# cd ${PLOT_WORK_DIR} -# cd ../ -# rm -rf ${PLOT_WORK_DIR} -#fi - echo "end plot_bcor.sh" exit diff --git a/util/Radiance_Monitor/image_gen/ush/plot_summary.sh b/util/Radiance_Monitor/image_gen/ush/plot_summary.sh index bc394bb8e..61455bd6b 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_summary.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_summary.sh @@ -29,17 +29,7 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` -#-------------------------------------------------------------------- -# Set ctldir to point to correct control file source -# -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/time -else - ctldir=$TANKverf/time -fi - - +ctldir=$IMGNDIR/time usef="use.txt" timesf="times.txt" chanf="chan.txt" @@ -52,7 +42,7 @@ chanf="chan.txt" # for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ + $NCP $ctldir/${type}*.ctl* ./ ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate @@ -88,24 +78,20 @@ for type in ${SATYPE2}; do echo "rgnHH, rgnTM = $rgnHH, $rgnTM" - test_file=${ieee_src}/time.${type}.${cdate}.ieee_d - - if [[ $USE_ANL = 1 ]]; then - test_file2=${ieee_src}/time.${type}_anl.${cdate}.ieee_d - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_time.tar ${files} + fi + else + files=`ls ${ieee_src}/time.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi adate=`$NDATE +${CYCLE_INTERVAL} ${cdate}` @@ -114,17 +100,26 @@ for type in ${SATYPE2}; do ${UNCOMPRESS} *.ieee_d.${Z} + #----------------------------------------------- + # Remove 'time.' from the *ieee_d file names. + # + prefix="time." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done -#-------------------------------------------------------------------- -# Plotting is moving towards dynamic, interactive images drawn -# on-the-fly in the client browser. These images require small text -# files instead of static images produced by GrADS. The flag -# PLOT_STATIC_IMGS controls the conditional plotting of the older -# static images. -# -# At present this only affects the summary plots, but will eventually -# include most radiance images. -# + #-------------------------------------------------------------------- + # Plotting is moving towards dynamic, interactive images drawn + # on-the-fly in the client browser. These images require small text + # files instead of static images produced by GrADS. The flag + # PLOT_STATIC_IMGS controls the conditional plotting of the older + # static images. + # + # At present this only affects the summary plots, but will eventually + # include most radiance images. + # if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then outfile=${tmpdir}/${type}.gs @@ -204,7 +199,7 @@ EOF echo "END data file generation:" rm -f ${input} - + rm -f ${type}*ieee_d done @@ -215,13 +210,14 @@ done if [[ ! -d ${IMGNDIR}/summary ]]; then mkdir -p ${IMGNDIR}/summary fi -$NCP *summary.png ${IMGNDIR}/summary/. + +if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then + $NCP *summary.png ${IMGNDIR}/summary/. +fi for type in ${SATYPE2}; do - $NCP ${type}.sum.txt ${IMGNDIR}/summary/${type}.${PDATE}.sum.txt + mv ${type}.sum.txt ${IMGNDIR}/summary/${type}.${PDATE}.sum.txt done -$NCP *.sum.txt ${IMGNDIR}/summary/. - #-------------------------------------------------------------------- diff --git a/util/Radiance_Monitor/image_gen/ush/plot_time.sh b/util/Radiance_Monitor/image_gen/ush/plot_time.sh index e33528acd..8d45e1b2c 100755 --- a/util/Radiance_Monitor/image_gen/ush/plot_time.sh +++ b/util/Radiance_Monitor/image_gen/ush/plot_time.sh @@ -45,17 +45,7 @@ edate=$PDATE bdate0=`echo $bdate|cut -c1-8` edate0=`echo $edate|cut -c1-8` -#-------------------------------------------------------------------- -# Set ctldir to point to correct control file source -# -imgdef=`echo ${#IMGNDIR}` -if [[ $imgdef -gt 0 ]]; then - ctldir=$IMGNDIR/time -else - ctldir=$TANKverf/time -fi - -echo ctldir = $ctldir +ctldir=$IMGNDIR/time #-------------------------------------------------------------------- @@ -68,15 +58,8 @@ echo ctldir = $ctldir # for type in ${SATYPE2}; do - $NCP $ctldir/${type}.ctl* ./ - if [[ -s ./${type}.ctl.${Z} ]]; then - ${UNCOMPRESS} ./${type}.ctl.${Z} - fi - - if [[ $USE_ANL = 1 ]]; then - $NCP $ctldir/${type}_anl.ctl* ./ - ${UNCOMPRESS} ./${type}_anl.ctl.${Z} - fi + $NCP $ctldir/${type}*.ctl* ./ + ${UNCOMPRESS} *.ctl.${Z} cdate=$bdate while [[ $cdate -le $edate ]]; do @@ -102,27 +85,20 @@ for type in ${SATYPE2}; do ieee_src=${TANKverf}/${RUN}.${day} fi - - if [[ -d ${ieee_src} ]]; then - test_file=${ieee_src}/time.${type}.${cdate}.ieee_d - - if [[ $USE_ANL = 1 ]]; then - test_file2=${ieee_src}/time.${type}_anl.${cdate}.ieee_d - else - test_file2= - fi - - if [[ -s $test_file ]]; then - $NCP ${test_file} ./${type}.${cdate}.ieee_d - elif [[ -s ${test_file}.${Z} ]]; then - $NCP ${test_file}.${Z} ./${type}.${cdate}.ieee_d.${Z} - fi - - if [[ -s $test_file2 ]]; then - $NCP ${test_file2} ./${type}_anl.${cdate}.ieee_d - elif [[ -s ${test_file2}.${Z} ]]; then - $NCP ${test_file2}.${Z} ./${type}_anl.${cdate}.ieee_d.${Z} + #----------------------------------------------------------- + # Locate the data files, first checking for a tar file, + # and copy them locally. + # + if [[ -s ${ieee_src}/radmon_time.tar ]]; then + files=`tar -tf ${ieee_src}/radmon_time.tar | grep ${type} | grep ieee_d` + if [[ ${files} != "" ]]; then + tar -xf ${ieee_src}/radmon_time.tar ${files} fi + else + files=`ls ${ieee_src}/time.*${type}*ieee_d*` + for f in ${files}; do + $NCP ${f} . + done fi adate=`$NDATE +${CYCLE_INTERVAL} $cdate` @@ -131,6 +107,17 @@ for type in ${SATYPE2}; do ${UNCOMPRESS} ./*.ieee_d.${Z} + #----------------------------------------------- + # Remove 'time.' from the *ieee_d file names. + # + prefix="time." + dfiles=`ls *.ieee_d` + for file in $dfiles; do + newfile=`basename $file | sed -e "s/^$prefix//"` + mv ./${file} ./${newfile} + done + + if [[ $PLOT_STATIC_IMGS -eq 1 ]]; then for var in ${PTYPE}; do echo $var diff --git a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt index 30836d410..b8c4d0aa9 100644 --- a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt +++ b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt @@ -1,2 +1,2 @@ -abi_g16 abi_g17 ahi_himawari8 amsua_metop-b amsua_metop-c amsua_n15 amsua_n18 amsua_n19 atms_npp atms_n20 avhrr_metop-b avhrr_n18 avhrr_n19 cris-fsr_npp cris-fsr_n20 hirs4_n19 iasi_metop-b iasi_metop-c mhs_metop-b mhs_metop-c mhs_n19 seviri_m08 seviri_m11 ssmis_f17 ssmis_f18 +abi_g16 ahi_himawari8 amsua_metop-b amsua_metop-c amsua_n15 amsua_n18 amsua_n19 atms_npp atms_n20 avhrr_metop-b avhrr_n18 avhrr_n19 cris-fsr_npp cris-fsr_n20 hirs4_n19 iasi_metop-b iasi_metop-c mhs_metop-b mhs_metop-c mhs_n19 seviri_m08 seviri_m11 ssmis_f17 ssmis_f18 diff --git a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt index ecb51f7b6..122b82825 100644 --- a/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt +++ b/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt @@ -47,6 +47,7 @@ mhs_metop-c -49.444 1.111 90 iasi_metop-a -48.330 3.334 60 iasi_metop-b -48.330 3.334 60 + iasi_metop-c -48.330 3.334 60 cris_npp -48.330 3.333 30 cris-fsr_npp -48.330 3.333 30 cris-fsr_n20 -48.330 3.333 30 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt index 19aba6306..a881d1581 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radang.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_angle ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt index 506824445..116bcfe31 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcoef.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_bcoef ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt index 59ac8532d..7b2bfe03a 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radbcor.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_BCOR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_BCOR_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_bcor ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt index 3bb4e4b2e..27cac8025 100644 --- a/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt +++ b/util/Radiance_Monitor/nwprod/radmon_shared/sorc/verf_radtime.fd/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 2.8.12) file(GLOB RADMON_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 ) set(RADMON_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ") set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_time ) diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh index b592c7699..2b0bf1ecd 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh @@ -162,10 +162,7 @@ else fi if [[ $REGIONAL_RR -eq 1 ]]; then -# angl_file=${rgnHH}.angle.${data_file}.${rgnTM} angl_file=${rgnHH}.${data_file}.${rgnTM} -# else -# angl_file=angle.${data_file} fi @@ -228,17 +225,19 @@ EOF done # for type in ${SATYPE} loop - cwd=`pwd` - tar_file=radmon_angle.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_angle.tar tar -cf $tar_file angle*.ieee_d* angle*.ctl* - mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - cd ${cwd} + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=3 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh index 14a5905a8..bd4642d77 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh @@ -210,17 +210,19 @@ EOF done # type in $SATYPE loop - cwd=`pwd` - tar_file=radmon_bcoef.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_bcoef.tar tar -cf $tar_file bcoef*.ieee_d* bcoef*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=5 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh index 553cb8111..6f2bb0388 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh @@ -202,18 +202,20 @@ EOF done # dtype in $gesanl loop done # type in $SATYPE loop - cwd=`pwd` + + ${USHradmon}/rstprod.sh tar_file=radmon_bcor.tar tar -cf $tar_file bcor*.ieee_d* bcor*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd - + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then err=7 diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh index 4603dad21..97f53307a 100755 --- a/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh @@ -260,18 +260,19 @@ EOF done - cwd=`pwd` - tar_file=radmon_time.tar + ${USHradmon}/rstprod.sh + tar_file=radmon_time.tar tar -cf $tar_file time*.ieee_d* time*.ctl* mv $tar_file ${TANKverf_rad} - cd ${TANKverf_rad} - tar -xf ${tar_file} - rm ${tar_file} - - cd $cwd - + if [[ $RAD_AREA = "rgn" ]]; then + cwd=`pwd` + cd ${TANKverf_rad} + tar -xf ${tar_file} + rm ${tar_file} + cd ${cwd} + fi if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then echo "fail, ctr = $fail, $ctr" diff --git a/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh b/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh new file mode 100755 index 000000000..dd5a09ec0 --- /dev/null +++ b/util/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +#--------------------------------------------------------- +# rstprod.sh +# +# Restrict data from select sensors and satellites +#--------------------------------------------------------- + +# Restrict select sensors and satellites +echo "--> rstprod.sh" + +export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} +rlist="saphir abi_g16" +for rtype in $rlist; do + ${CHGRP_CMD} *${rtype}* +done + +echo "<-- rstprod.sh" diff --git a/util/Radiance_Monitor/parm/RadMon_config b/util/Radiance_Monitor/parm/RadMon_config index d411da390..381c4767b 100644 --- a/util/Radiance_Monitor/parm/RadMon_config +++ b/util/Radiance_Monitor/parm/RadMon_config @@ -222,6 +222,8 @@ elif [[ $MY_MACHINE = "wcoss2" ]]; then # module use /apps/test/modules module load GrADS/2.2.1-cce-11.0.4 + export GRADS=`which grads` + export STNMAP=`which stnmap` #------------------------ export SUB="qsub" From b73a7961644c2b9d1f36c17c3770d716b7dbf473 Mon Sep 17 00:00:00 2001 From: "Edward.Safford" Date: Wed, 30 Mar 2022 18:27:17 +0000 Subject: [PATCH 2/2] Github issue #347. Port non-operational components of OznMon to wcoss2. Completes #347. --- util/Ozone_Monitor/CMakeLists.txt | 6 +-- util/Ozone_Monitor/OznMon_install.pl | 29 +++++++---- util/Ozone_Monitor/build_OznMon_cmake.sh | 49 ++++++++----------- .../Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh | 6 ++- .../Ozone_Monitor/data_xtrct/ush/OznMon_DE.sh | 5 ++ .../data_xtrct/ush/OznMon_MkBase.sh | 12 +---- util/Ozone_Monitor/get_hostname.pl | 41 ---------------- util/Ozone_Monitor/get_machine.sh | 38 ++++++++++++++ .../image_gen/ush/OznMon_Transfer.sh | 20 +++++--- util/Ozone_Monitor/image_gen/ush/mk_horiz.sh | 5 +- .../Ozone_Monitor/image_gen/ush/mk_summary.sh | 5 ++ util/Ozone_Monitor/image_gen/ush/mk_time.sh | 6 +++ util/Ozone_Monitor/image_gen/ush/transfer.sh | 2 +- util/Ozone_Monitor/parm/OznMon_config | 28 +++++++++-- 14 files changed, 144 insertions(+), 108 deletions(-) delete mode 100755 util/Ozone_Monitor/get_hostname.pl create mode 100755 util/Ozone_Monitor/get_machine.sh diff --git a/util/Ozone_Monitor/CMakeLists.txt b/util/Ozone_Monitor/CMakeLists.txt index 1b4acf7c4..35111689a 100644 --- a/util/Ozone_Monitor/CMakeLists.txt +++ b/util/Ozone_Monitor/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.8.12) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) # I am top-level project. if( NOT DEFINED ENV{CC} ) @@ -84,8 +84,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) find_package( W3NCO ) set(BUILD_NCDIAG ON) - set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag/include") - add_subdirectory(src/ncdiag) + set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag") + add_subdirectory(${PROJECT_SOURCE_DIR}/../../src/ncdiag ${PROJECT_BINARY_DIR}/src/ncdiag) set(NCDIAG_LIBRARIES ncdiag ) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) endif() diff --git a/util/Ozone_Monitor/OznMon_install.pl b/util/Ozone_Monitor/OznMon_install.pl index 0c7906d68..bf8248f80 100755 --- a/util/Ozone_Monitor/OznMon_install.pl +++ b/util/Ozone_Monitor/OznMon_install.pl @@ -5,18 +5,19 @@ # # This script makes sets all necessary configuration definitions # and calls the makeall.sh script to build all the necessary -# executables. This script works for hera, wcoss_c, and -# wcoss_d machines. +# executables. This script works for hera, wcoss_c, wcoss_d, +# and wcoss2 machines. # #------------------------------------------------------------------- use IO::File; use File::Copy qw(move); - my $machine = `/usr/bin/perl get_hostname.pl`; + my $machine = `./get_machine.sh`; + $machine =~ s/^\s+|\s+$//g; # strip any whitespace in $machine my $my_machine="export MY_MACHINE=$machine"; - if( $machine ne "wcoss_c" && $machine ne "hera" && $machine ne "wcoss_d" ) { + if( $machine ne "wcoss_c" && $machine ne "hera" && $machine ne "wcoss_d" && $machine ne "wcoss2" ) { die( "ERROR --- Unrecognized machine hostname, $machine. Exiting now...\n" ); } else { @@ -62,6 +63,9 @@ elsif( $machine eq "wcoss_c" ){ $tankdir = "/gpfs/hps/emc/da/noscrub/$user_name/nbns"; } + elsif( $machine eq "wcoss2" ){ + $tankdir = "/lfs/h2/emc/da/noscrub/$user_name/nbns"; + } else { $tankdir = "/global/save/$user_name/nbns"; } @@ -209,6 +213,10 @@ $my_ptmp="export OZN_PTMP=\${OZN_PTMP:-/gpfs/hps2/ptmp}"; $my_stmp="export OZN_STMP=\${OZN_STMP:-/gpfs/hps2/stmp}"; } + elsif( $machine eq "wcoss2" ) { + $my_ptmp="export OZN_PTMP=\${OZN_PTMP:-/lfs/h2/emc/ptmp}"; + $my_stmp="export OZN_STMP=\${OZN_STMP:-/lfs/h2/emc/stmp}"; + } elsif( $machine eq "hera" ){ $ptmp = "/scratch2/NCEPDEV/stmp3"; @@ -325,18 +333,21 @@ print "\n"; print "Updating parm/OznMon_user_settings\n"; - my $account = "export ACCOUNT=\${ACCOUNT:-fv3-cpu}"; - if( $machine ne "theia" && $machine ne "hera" ) { - $account = "export ACCOUNT=\${ACCOUNT:-}"; + my $account = "export ACCOUNT=\${ACCOUNT:-}"; + if( $machine eq "hera" ) { + $account = "export ACCOUNT=\${ACCOUNT:-fv3-cpu}"; + } + elsif( $machine eq "wcoss2" ){ + $account = "export ACCOUNT=\${ACCOUNT:-GFS-DEV}"; } my $project = "export PROJECT=\${PROJECT:-GFS-DEV}"; - if( $machine ne "wcoss_c" && $machine ne "wcoss_d" ) { + if( $machine ne "wcoss_c" && $machine ne "wcoss_d" && $machine ne "wcoss2" ) { $project="export PROJECT="; } my $job_queue="export JOB_QUEUE="; - if( $machine eq "wcoss_c" ) { + if( $machine eq "wcoss_c" || $machine eq "wcoss2" ) { $job_queue="export JOB_QUEUE=\${JOB_QUEUE:-dev}"; } elsif( $machine eq "wcoss" || $machine eq "wcoss_d" ){ $job_queue = "export JOB_QUEUE=\${JOB_QUEUE:-dev_shared}"; diff --git a/util/Ozone_Monitor/build_OznMon_cmake.sh b/util/Ozone_Monitor/build_OznMon_cmake.sh index b5768f284..681fed999 100755 --- a/util/Ozone_Monitor/build_OznMon_cmake.sh +++ b/util/Ozone_Monitor/build_OznMon_cmake.sh @@ -22,36 +22,23 @@ echo "top_level = ${top_level}" export MY_OZNMON=${MY_OZNMON:-$top_level} echo "MY_OZNMON = ${MY_OZNMON}" -#module purge - -if [[ -d /dcom && -d /hwrf ]] ; then - . /usrx/local/Modules/3.2.10/init/sh - target=wcoss - . $MODULESHOME/init/sh -elif [[ -d /cm ]] ; then -# MODULESHOME=/opt/modules/3.2.10.3 - . $MODULESHOME/init/sh - target=wcoss_c -elif [[ -d /ioddev_dell ]]; then - . $MODULESHOME/init/sh - target=wcoss_d -elif [[ -d /scratch1 ]] ; then - . /apps/lmod/lmod/init/sh - target=hera -elif [[ -d /work ]]; then - . $MODULESHOME/init/sh - target=orion -elif [[ -d /data/prod ]]; then - . $MODULESHOME/init/sh - target=s4 -elif [[ -d /jetmon ]]; then - . /apps/lmod/lmod/init/sh - target=jet +target=`./get_machine.sh` +echo "target = $target" + +if [[ $target = "wcoss_c" || $target = "wcoss_d" || + $target = "orion" || $target = "wcoss2" || + $target = "s4" ]] ; then + . $MODULESHOME/init/sh +elif [[ $target = "hera" ]] ; then + . /apps/lmod/lmod/init/sh +elif [[ $target = "jet" ]] ; then + . /apps/lmod/lmod/init/sh else - echo "unknown target = $target" - exit 9 + echo "unknown target = $target" + exit 9 fi + GSI_Pkg=${top_level}/../.. echo "GSI_Pkg = ${GSI_Pkg}" @@ -71,7 +58,8 @@ fi if [[ ${target} = "hera" || ${target} = "wcoss_c" \ || ${target} = "wcoss_d" || ${target} = "orion" \ - || ${target} = "jet" || ${target} = "s4" ]]; then + || ${target} = "jet" || ${target} = "s4" \ + || ${target} = "wcoss2" ]]; then echo Building nwprod executables on ${target} echo @@ -97,6 +85,10 @@ if [[ ${target} = "hera" || ${target} = "wcoss_c" \ module purge module use -a $dir_modules module load $dir_modules/modulefile.ProdGSI.$target + elif [ $target = wcoss2 ]; then + module purge + module use -a $dir_modules + module load modulefile.ProdGSI.$target.lua fi @@ -117,7 +109,6 @@ if [[ ${target} = "hera" || ${target} = "wcoss_c" \ #------------------------------ # source OznMon_config #------------------------------ - . ${top_level}/parm/OznMon.ver . ${top_level}/parm/OznMon_config #------------------------------------------------------- diff --git a/util/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh b/util/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh index a40867709..dc88bd039 100755 --- a/util/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh +++ b/util/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh @@ -39,7 +39,7 @@ echo start OznMon_CP.sh exit_value=0 nargs=$# -if [[ $nargs -le 0 || $nargs -gt 7 ]]; then +if [[ $nargs -le 0 || $nargs -gt 9 ]]; then usage exit 1 fi @@ -193,6 +193,10 @@ if [[ ${nfile_src} -gt 0 ]]; then elif [[ $MY_MACHINE = "hera" ]]; then $SUB --account=${ACCOUNT} --time=10 -J ${jobname} -D . \ -o ${logfile} --ntasks=1 --mem=5g ${job} + + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${OZN_LOGdir}/CP.${PDY}.${CYC}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job} fi else echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}" diff --git a/util/Ozone_Monitor/data_xtrct/ush/OznMon_DE.sh b/util/Ozone_Monitor/data_xtrct/ush/OznMon_DE.sh index abe0d3680..e29078899 100755 --- a/util/Ozone_Monitor/data_xtrct/ush/OznMon_DE.sh +++ b/util/Ozone_Monitor/data_xtrct/ush/OznMon_DE.sh @@ -229,6 +229,11 @@ elif [[ $MY_MACHINE = "wcoss_c" ]]; then -R "select[mem>100] rusage[mem=100]" \ -M 100 -W 0:05 -J ${job} -cwd ${PWD} $jobfile +elif [[ $MY_MACHINE = "wcoss2" ]]; then + + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${OZN_LOGdir}/DE.${PDY}.${cyc}.log \ + -e ${OZN_LOGdir}/DE.${PDY}.${cyc}.err \ + -V -l select=1:mem=5000M -l walltime=20:00 -N ${job} ${jobfile} fi diff --git a/util/Ozone_Monitor/data_xtrct/ush/OznMon_MkBase.sh b/util/Ozone_Monitor/data_xtrct/ush/OznMon_MkBase.sh index d8dd20ba2..b22ae2895 100755 --- a/util/Ozone_Monitor/data_xtrct/ush/OznMon_MkBase.sh +++ b/util/Ozone_Monitor/data_xtrct/ush/OznMon_MkBase.sh @@ -79,14 +79,6 @@ export GLB_AREA=${GLB_AREA:-1} #------------------------------------------------------------------- top_parm=${this_dir}/../../parm -oznmon_version=${oznmon_version:-${top_parm}/OznMon.ver} -if [[ -s ${oznmon_version} ]]; then - . ${oznmon_version} -else - echo "Unable to source ${oznmon_version} file" - exit 2 -fi - oznmon_user_settings=${oznmon_user_settings:-${top_parm}/OznMon_user_settings} if [[ -s ${oznmon_user_settings} ]]; then . ${oznmon_user_settings} @@ -205,7 +197,7 @@ for type in ${SATYPE}; do test_dir=${OZN_STATS_TANKDIR}/${RUN}.${pdy}/${cyc}/oznmon/time if [[ -d ${test_dir} ]]; then - test_file=${test_dir}/${type}.${cdate}.ieee_d + test_file=${test_dir}/${type}.ges.${cdate}.ieee_d if [[ -s $test_file ]]; then $NCP ${test_file} ./${type}.${cdate}.ieee_d @@ -220,7 +212,7 @@ for type in ${SATYPE}; do if [[ $have_ctl -eq 0 ]]; then - test_file=${test_dir}/${type}.ctl + test_file=${test_dir}/${type}.ges.ctl if [[ -s ${test_file} ]]; then $NCP ${test_file} ./${type}.ctl have_ctl=1 diff --git a/util/Ozone_Monitor/get_hostname.pl b/util/Ozone_Monitor/get_hostname.pl deleted file mode 100755 index 145b3a91c..000000000 --- a/util/Ozone_Monitor/get_hostname.pl +++ /dev/null @@ -1,41 +0,0 @@ -#! /usr/bin/perl - -#------------------------------------------------------------------- -# get_hostname.pl -# -# This script determines the hostname of the current machine. The -# possiblities are wcoss_c, wcoss_d, hera, or "" if the host is not -# one of those three. -#------------------------------------------------------------------- - - - my $machine = ""; - - # - # hera login nodes are hfe1-hfeN, and hostname command only returns the node name, - # while wcoss_c and wcoss_d return [hostname].ncep.noaa.gov. Keep only the - # actual hostname and see if it matches the node names for hera, wcoss_d, or wcoss_c. - # - my $host = ""; - $host = ` hostname `; - chomp( $host ); - - if( $host =~ /\./ ) { - my @hostnames = split( '\.', $host ); - $host = $hostnames[0]; - } - - if( $host =~ /hfe/ ) { - $machine = "hera"; - } - elsif( $host =~ /login/ ) { - $machine = "wcoss_c"; - } - elsif( $host =~ /v/ || $host =~ /m/ ){ # wcoss_d nodes are vXXaY and mXXaY - $machine = "wcoss_d"; - } - - print "$machine"; - - exit 0; - diff --git a/util/Ozone_Monitor/get_machine.sh b/util/Ozone_Monitor/get_machine.sh new file mode 100755 index 000000000..7cbfadc61 --- /dev/null +++ b/util/Ozone_Monitor/get_machine.sh @@ -0,0 +1,38 @@ +#! /bin/bash + +#------------------------------------------------------------------ +# get_machine.sh +# +# Determine which platform we're running on. System name will +# be journaled to stdout so calling scripts can get it there. +# If system is not recognized an empty string will be returned +# so it's up to the calling scripts to correctly interpret that. +#------------------------------------------------------------------ + +target="" + +if [[ -d /cm ]] ; then + . $MODULESHOME/init/sh + target=wcoss_c +elif [[ -d /ioddev_dell ]]; then + . $MODULESHOME/init/sh + target=wcoss_d +elif [[ -d /scratch1 ]] ; then + . /apps/lmod/lmod/init/sh + target=hera +elif [[ -d /data ]] ; then + . /usr/share/lmod/lmod/init/sh + target=s4 +elif [[ -d /work ]]; then + . $MODULESHOME/init/sh + target=orion +elif [[ -d /jetmon ]] ; then + . /apps/lmod/lmod/init/sh + target=jet +elif [[ -d /lfs ]] ; then + . $MODULESHOME/init/sh + target=wcoss2 +fi + +echo $target +exit diff --git a/util/Ozone_Monitor/image_gen/ush/OznMon_Transfer.sh b/util/Ozone_Monitor/image_gen/ush/OznMon_Transfer.sh index 4a9bb5602..2ba4b3bf3 100755 --- a/util/Ozone_Monitor/image_gen/ush/OznMon_Transfer.sh +++ b/util/Ozone_Monitor/image_gen/ush/OznMon_Transfer.sh @@ -90,20 +90,24 @@ fi transfer_script=${OZN_IG_SCRIPTS}/transfer.sh job=${OZNMON_SUFFIX}_ozn_transfer +job_queue="dev_transfer" -if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then - - job_queue="dev_transfer" +echo "PROJECT = $PROJECT" +echo "logf = $logf" +echo "errf = $errf" +echo "transfer_script = $transfer_script" - echo "PROJECT = $PROJECT" - echo "logf = $logf" - echo "errf = $errf" - echo "transfer_script = $transfer_script" +if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then $SUB -P $PROJECT -q $job_queue -o ${logf} -e ${errf} -M 50 -W 0:20 \ -R affinity[core] -J ${job} -cwd ${OZN_IG_SCRIPTS} \ ${transfer_script} - + +elif [[ $MY_MACHINE = "wcoss2" ]]; then + + $SUB -q $job_queue -A $ACCOUNT -o ${logf} -e ${errf} \ + -V -l select=1:mem=1000M -l walltime=20:00 \ + -N ${job} ${transfer_script} fi exit diff --git a/util/Ozone_Monitor/image_gen/ush/mk_horiz.sh b/util/Ozone_Monitor/image_gen/ush/mk_horiz.sh index e0cb39991..a316cc56b 100755 --- a/util/Ozone_Monitor/image_gen/ush/mk_horiz.sh +++ b/util/Ozone_Monitor/image_gen/ush/mk_horiz.sh @@ -101,7 +101,10 @@ for dsrc in ${data_source}; do -o ${logf} -e ${errf} -W 0:05 -J ${job} -cwd ${WORKDIR} \ ${WORKDIR}/${cmdfile} - + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logf} -e ${errf} \ + -V -l select=1:mem=5000M -l walltime=20:00 \ + -N ${job} ${WORKDIR}/${cmdfile} fi done diff --git a/util/Ozone_Monitor/image_gen/ush/mk_summary.sh b/util/Ozone_Monitor/image_gen/ush/mk_summary.sh index 80560d657..191825f03 100755 --- a/util/Ozone_Monitor/image_gen/ush/mk_summary.sh +++ b/util/Ozone_Monitor/image_gen/ush/mk_summary.sh @@ -116,6 +116,11 @@ for ptype in ${process_type}; do -o ${logf} -e ${errf} -W 0:05 -J ${job} \ -cwd ${WORKDIR} ${WORKDIR}/${cmdfile} + elif [[ $MY_MACHINE = "wcoss2" ]]; then + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logf} -e ${errf} \ + -V -l select=1:mem=5000M -l walltime=5:00 \ + -N ${job} ${WORKDIR}/${cmdfile} + fi done diff --git a/util/Ozone_Monitor/image_gen/ush/mk_time.sh b/util/Ozone_Monitor/image_gen/ush/mk_time.sh index cb200a174..25a70667c 100755 --- a/util/Ozone_Monitor/image_gen/ush/mk_time.sh +++ b/util/Ozone_Monitor/image_gen/ush/mk_time.sh @@ -93,6 +93,12 @@ for dsrc in ${data_source}; do $SUB -q ${JOB_QUEUE} -P ${PROJECT} -M 50 -R affinity[core] \ -o ${logf} -e ${errf} -W 0:05 -J ${job} -cwd ${WORKDIR} ${WORKDIR}/${cmdfile} + elif [[ $MY_MACHINE = "wcoss2" ]]; then + + $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logf} -e ${errf} \ + -V -l select=1:mem=5000M -l walltime=20:00 \ + -N ${job} ${WORKDIR}/${cmdfile} + fi diff --git a/util/Ozone_Monitor/image_gen/ush/transfer.sh b/util/Ozone_Monitor/image_gen/ush/transfer.sh index d34921879..cbc688395 100755 --- a/util/Ozone_Monitor/image_gen/ush/transfer.sh +++ b/util/Ozone_Monitor/image_gen/ush/transfer.sh @@ -13,7 +13,7 @@ echo "RSYNCH = $RSYNC" if [[ ${OZN_IMGN_TANKDIR} != "/" ]]; then # sanity check - if [[ $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss_d" ]]; then + if [[ $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss2" ]]; then WEB_DIR=${WEB_DIR}/${OZNMON_SUFFIX}/${RUN} ssh ${WEB_USER}@${WEB_SVR}.ncep.noaa.gov "mkdir -p ${WEB_DIR}" diff --git a/util/Ozone_Monitor/parm/OznMon_config b/util/Ozone_Monitor/parm/OznMon_config index bc00092c9..36f04db3b 100644 --- a/util/Ozone_Monitor/parm/OznMon_config +++ b/util/Ozone_Monitor/parm/OznMon_config @@ -18,7 +18,7 @@ if [[ $OZNMON_CONFIG -ne 1 ]]; then export OZNMON_CONFIG=1 - export MY_MACHINE=wcoss_d + export MY_MACHINE=wcoss2 #------------------------------------------------------------------- # Note: The file header in the oznstat file has slightly changed. @@ -38,7 +38,7 @@ if [[ $OZNMON_CONFIG -ne 1 ]]; then # you checked out only the Ozone_Monitor portion of the branch then # MY_RADMON should point to that. # - export MY_OZNMON=${MY_OZNMON:-/gpfs/dell2/emc/modeling/noscrub/Edward.Safford/GSI/util/Ozone_Monitor} + export MY_OZNMON=${MY_OZNMON:-/lfs/h2/emc/da/noscrub/Edward.Safford/git/gsi/GSI/util/Ozone_Monitor} export OZN_SCRIPTS=${OZN_SCRIPTS:-${MY_OZNMON}/scripts} export OZN_DE_SCRIPTS=${OZN_DE_SCRIPTS:-${MY_OZNMON}/data_xtrct/ush} @@ -66,7 +66,7 @@ if [[ $OZNMON_CONFIG -ne 1 ]]; then # and the succedding scripts will construct and use subdirectories under # this location. # - export OZN_TANKDIR=${OZN_TANKDIR:-/u/Edward.Safford/nbns} + export OZN_TANKDIR=${OZN_TANKDIR:-/lfs/h2/emc/da/noscrub/Edward.Safford/nbns} if [[ $GLB_AREA -eq 1 ]]; then OZN_IMGN_TANKDIR=${OZN_TANKDIR}/imgn/${OZNMON_SUFFIX} @@ -93,9 +93,9 @@ if [[ $OZNMON_CONFIG -ne 1 ]]; then # # Log and work space definitions # - export OZN_PTMP=${OZN_PTMP:-/gpfs/dell2/ptmp} + export OZN_PTMP=${OZN_PTMP:-/lfs/h2/emc/ptmp} export PTMP_USER=${PTMP_USER:-${OZN_PTMP}/${LOGNAME}} - export OZN_STMP=${OZN_STMP:-/gpfs/dell2/stmp} + export OZN_STMP=${OZN_STMP:-/lfs/h2/emc/stmp} export STMP_USER=${STMP_USER:-${OZN_STMP}/${LOGNAME}} @@ -213,6 +213,24 @@ if [[ $OZNMON_CONFIG -ne 1 ]]; then export PARTITION_OZNMON=${PARTITION_OZNMON:-vjet} + elif [[ $MY_MACHINE = "wcoss2" ]]; then + module load prod_util/2.0.13 + module load util_shared/1.4.0 + + #------------------------ + # temporary workaround + # + module use /apps/test/modules + module load GrADS/2.2.1-cce-11.0.4 + export GRADS=`which grads` + export STNMAP=`which stnmap` + #------------------------ + + export SUB="qsub" + export COMPRESS=/usr/bin/gzip + export UNCOMPRESS="/usr/bin/gunzip -f" + export RSYNC=`which rsync` + fi else