Github issue #313. Store data files for global sources in tar files. …

…Completes issue #313
EdwardSafford-NOAA · Mar 16, 2022 · 837f06a · 837f06a
1 parent 47ee70b
commit 837f06a
Show file tree

Hide file tree

Showing 38 changed files with 744 additions and 581 deletions.
diff --git a/util/Radiance_Monitor/CMakeLists.txt b/util/Radiance_Monitor/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 2.8.12)
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     # I am top-level project.
     if( NOT DEFINED ENV{CC} )
@@ -66,6 +66,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     endif()
 
     cmake_policy(SET CMP0009 NEW)
+    cmake_policy(SET CMP0074 NEW)
     find_package(OpenMP)
     message("found openmp with flag ${OPENMP_Fortran_FLAGS}")
 
@@ -84,8 +85,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     find_package( W3NCO )
 
     set(BUILD_NCDIAG ON)
-    set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag/include")
-    add_subdirectory(src/ncdiag)
+    set(NCDIAG_INCS "${PROJECT_BINARY_DIR}/src/ncdiag")
+    add_subdirectory(${PROJECT_SOURCE_DIR}/../../src/ncdiag ${PROJECT_BINARY_DIR}/src/ncdiag)
     set(NCDIAG_LIBRARIES ncdiag )
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 endif()

diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_mk_base.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_MK_BASE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_MK_BASE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_mk_base )

diff --git a/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt b/util/Radiance_Monitor/data_extract/sorc/radmon_validate_tm.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_VALIDATE_TM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_VALIDATE_TM_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_validate_tm.x )

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh
@@ -192,8 +192,8 @@ if [[  -d ${DATA_LOCATION} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job}
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-           -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/CP.${PDY}.${CYC}.err \
+	   -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 else
    echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}"

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh
@@ -248,8 +248,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job} 
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${pdy}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 
 else  # radstat and/or biascr not found

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh
@@ -306,8 +306,8 @@ elif [[ $MY_MACHINE = "hera" ]]; then
         -j oe -o ${logfile} ${job}
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${PDY}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
 fi
 
 

diff --git a/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl b/util/Radiance_Monitor/data_extract/ush/nu_find_cycle.pl
@@ -89,7 +89,7 @@
    }
 
    my @mmdirs = grep { /$search_string/ } @alldirs;
-
+   
    #-----------------------------------------------------------------------   
    #  If there are no $run.yyyymmdd subdirectories, then exit without 
    #    returning any date string.
@@ -157,13 +157,26 @@
          $hr_ctr = $hr_ctr - 1;
 
          $newdir = "${dirpath}/${sortmm[$ctr]}/${hrs[$hr_ctr]}/${lcm}";
-#         print " newdir = $newdir \n";
-
 
          if( -d $newdir ) {
             opendir DIR, $newdir or die "Cannot open the current directory: $!";
 
-            my @timefiles = grep { /ieee_d/ } readdir DIR;
+            my @dirfiles = readdir DIR;
+            my @timefiles = grep { /ieee_d/ } @dirfiles;
+
+            #  If no *ieee_d* files were found then look for the compressed format of
+            #  radmon_*.tar files instead.  If found then use the parent directories
+            #  to establish the cycle time.
+            #
+            if( $#timefiles < 0 ) {       
+               my @tarfiles = grep { /radmon_time.tar/ } @dirfiles;
+
+               if( $#tarfiles >= 0 ) { 
+                  my $cycle = "${sortmm[$ctr]}${hrs[$hr_ctr]}";
+                  push( @timefiles, $cycle );
+               }
+            } 
+
 
             if( $#timefiles >= 0 ) {
                my @sorttime = sort( @timefiles );
@@ -201,18 +214,13 @@
                   }
                }
             }
-
          } 
 
       } while $hr_ctr > 0 && $found_cycle == 0;
 
-#      print " found_cycle, ctr, end_ctr = $found_cycle, $ctr, $end_ctr \n";
-
       if( $cyc == 0 && $ctr >= $end_ctr ){  
-#         print " exiting from if\n";
          $exit_flag = 1;
       } elsif( $cyc == 1 && $ctr <= $end_ctr ){
-#         print " exiting from elsif\n";
          $exit_flag = 1;
       }
 

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_angle.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_ANGLE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_ANGLE_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_angle )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_bcoef.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_BCOEF_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_BCOEF_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_bcoef )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_horiz.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_HORIZ_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_HORIZ_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_horiz )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_summary.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_SUMMARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_SUMMARY_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_summary )

diff --git a/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt b/util/Radiance_Monitor/image_gen/src/radmon_ig_time.fd/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8.12)
   file(GLOB RADMON_IG_TIME_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*90 )
   set(RADMON_IG_TIME_Fortran_FLAGS "-fp-model strict -assume byterecl -convert big_endian -O3 -traceback -D_REAL8_ ")
   set(Util_MODULE_DIR ${PROJECT_BINARY_DIR}/include/radmon_ig_time )

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh
@@ -109,6 +109,10 @@ if [[ $? -ne 0 ]]; then
 fi
 
 
+if [[ ! -d ${IMGNDIR} ]]; then
+   mkdir -p ${IMGNDIR}
+fi
+
 #--------------------------------------------------------------------
 # Determine cycle to plot.  Exit if cycle is > last available
 # data.
@@ -183,7 +187,6 @@ export PDY=`echo $PDATE|cut -c1-8`
 #--------------------------------------------------------------------
 #  Locate ieee_src in $TANKverf and verify data files are present
 #
-
 ieee_src=${TANKverf}/${RUN}.${PDY}/${CYC}/${MONITOR}
 
 if [[ ! -d ${ieee_src} ]]; then
@@ -199,7 +202,16 @@ if [[ ! -d ${ieee_src} ]]; then
    exit 5
 fi
 
+#-----------------------------------------------------
+# check $ieee_src for data files.  If none are found
+# check contents of the radmon_angle.tar file.
+#
 nfile_src=`ls -l ${ieee_src}/*${PDATE}*ieee_d* | egrep -c '^-'`
+if [[ $nfile_src -le 0 ]]; then
+   if [[ -e ${ieee_src}/radmon_angle.tar ]]; then
+      nfile_src=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d | wc -l`
+   fi
+fi
 
 if [[ $nfile_src -le 0 ]]; then
    echo " Missing ieee_src files, nfile_src = ${nfile_src}, aborting plot"
@@ -238,6 +250,9 @@ fi
 #  the $satype_file can't be found.
 #
 test_list=`ls ${ieee_src}/angle.*${PDATE}.ieee_d*`
+if [[ $test_list = "" ]]; then
+   test_list=`tar -tf ${ieee_src}/radmon_angle.tar | grep ieee_d` 
+fi
 
 for test in ${test_list}; do
    this_file=`basename $test`
@@ -332,11 +347,12 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
       export WEBDIR=${WEBDIR}/${RADMON_SUFFIX}/${RUN}/pngs
 
       if [[ $MY_MACHINE = "wcoss2" ]]; then 
-         cmdfile=transfer_cmd
+         cmdfile="${PLOT_WORK_DIR}/transfer_cmd"
          echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
+         chmod 755 $cmdfile
 
-         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-              -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err \
+	      -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
       else
          $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \
               -R affinity[core] -J ${jobname} -cwd ${PWD} -b $run_time ${job}

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh
@@ -286,8 +286,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
 	    cmdfile=transfer_cmd
 	    echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
 
-	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-	         -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err 
+	         -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
 
 	 else
             $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \