adding support for TAMU Grace

abishekg7 · Mar 2, 2021 · 7830672 · 7830672
1 parent 67ba5e1
commit 7830672
Show file tree

Hide file tree

Showing 8 changed files with 151 additions and 27 deletions.
diff --git a/Config/config_postprocess.xml b/Config/config_postprocess.xml
@@ -62,47 +62,47 @@
       <entry id="ATM_GRID"
 	     type="string"
 	     valid_values="0.9x1.25"  
-	     value="ne30"
+	     value="ne120np4"
 	     group="case_info"
 	     desc="Atmosphere model grid"
 	     ></entry> 
 
       <entry id="ICE_GRID"
 	     type="string"
 	     valid_values="gx1v6" 
-	     value="gx1v6"
+	     value="tx0.1v2"
 	     group="case_info"
 	     desc="Sea-Ice model grid"
 	     ></entry> 
 
       <entry id="ICE_NX"
 	     type="string"
 	     valid_values="320"  
-	     value="320"
+	     value="3600"
 	     group="case_info"
 	     desc="Sea-Ice model grid NX - default 320 for 1-degree ocean grid"
 	     ></entry> 
 
       <entry id="ICE_NY"
 	     type="string"
 	     valid_values="384"
-	     value="384"
+	     value="2400"
 	     group="case_info"
 	     desc="Sea-Ice model grid NY - default 384 for 1-degree ocean grid"
 	     ></entry> 
 
       <entry id="LND_GRID"
 	     type="string"
 	     valid_values="0.9x1.25"
-	     value="ne30"
+	     value="ne120np4"
 	     group="case_info"
 	     desc="Land model grid"
 	     ></entry> 
 
       <entry id="OCN_GRID"
 	     type="string"
 	     valid_values="gx1v6"   
-	     value="gx1v6"
+	     value="tx0.1v2"
 	     group="case_info"
 	     desc="Ocean model grid"
 	     ></entry> 

diff --git a/Machines/grace_modules b/Machines/grace_modules
@@ -0,0 +1,27 @@
+#!/bin/bash -l
+
+echo "Loading modules for CESM_postprocessing tool..."
+
+module purge
+ml intel/2019b
+ml ANTLR
+ml HDF5
+#ml NCL/6.6.2
+#ml NCO/4.7.9
+#ml netCDF/4.6.1
+ml Python/2.7.16
+
+export NCARG_ROOT=/scratch/group/ihesp/software/share/ncl-6.6.2
+export NCO_PATH=/scratch/group/ihesp/software/share/nco-4.9.4/nco-4.9.7
+export NETCDF4_PATH=/scratch/group/ihesp/software/share/netcdf-c-4.7.4/install2019b_2
+export LD_LIBRARY_PATH=$NCARG_ROOT/lib:$NCO_PATH/lib:$NETCDF4_PATH/lib:$LD_LIBRARY_PATH
+export PATH=$NCARG_ROOT/bin:$NCO_PATH/bin:$NETCDF4_PATH/bin:$PATH
+
+
+#source /ihesp/user/agopal/CESM_postprocessing/cesm-env2/bin/activate
+#export PYTHONPATH=/ihesp/user/agopal/CESM_postprocessing/pynio/site-packages:$PYTHONPATH
+#export PYTHONPATH=/ihesp/user/agopal/CESM_postprocessing/pynio/build/lib.linux-x86_64-2.7/PyNIO:$PYTHONPATH
+#export PYTHONPATH=/ihesp/user/agopal/CESM_postprocessing/cesm-env2/lib/python2.7/site-packages/PyNIO:$PYTHONPATH
+#export PYNIOPATH=/ihesp/user/agopal/CESM_postprocessing/pynio/site-packages/PyNIO
+
+
diff --git a/Machines/machine_postprocess.xml b/Machines/machine_postprocess.xml
@@ -2,6 +2,78 @@
 
 <machine_postprocess>
 
+  <machine name="grace" hostname="grace">
+    <timeseries_pes queue="long"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</timeseries_pes>
+    <xconform_pes  queue="long" nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</xconform_pes>
+    <ocn_remap_pes  queue="long" nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</ocn_remap_pes>
+    <mpi_command>mpirun</mpi_command>
+    <pythonpath></pythonpath>
+    <f2py fcompiler="gfortran" f77exec="/sw/eb/sw/GCCcore/8.3.0/bin/gfortran">f2py</f2py>
+    <za>
+      <compiler>gfortran</compiler>
+      <flags>-c -g -O2</flags>
+      <include>-I/scratch/group/ihesp/software/share/netcdf-c-4.7.4/install2019b_2/include -I/scratch/group/ihesp/software/share/netcdf-fortran-4.5.3/install2019b_2/include</include>
+      <libs>-L/scratch/group/ihesp/software/share/netcdf-c-4.7.4/install2019b_2/lib -L/scratch/group/ihesp/software/share/netcdf-fortran-4.5.3/install2019b_2/lib -lnetcdff -L/sw/eb/sw/GCCcore/8.3.0/lib64 -L/sw/eb/sw/GCCcore/8.3.0/lib -L/sw/eb/sw/OpenBLAS/0.3.7-GCC-8.3.0/lib -L/sw/eb/sw/ScaLAPACK/2.0.2-gompi-2019b/lib64 -L/sw/eb/sw/FFTW/3.3.8-intel-2019b/lib -lnetcdf -lm -lpthread</libs>
+    </za>
+    <reset_modules>
+      <module>module purge</module>
+    </reset_modules>
+    <modules>
+      <module>module load intel/2019b</module>
+      <module>module load ANTLR</module>
+      <module>module load HDF5</module>
+      <module>module load GSL/2.6</module>
+      <module>module load Python/2.7.16</module>
+      <module>export NCARG_ROOT=/scratch/group/ihesp/software/share/ncl-6.6.2</module>
+      <module>export NCO_PATH=/scratch/group/ihesp/software/share/nco-4.9.4/nco-4.9.7/install</module>
+      <module>export NETCDF4_PATH=/scratch/group/ihesp/software/share/netcdf-c-4.7.4/install2019b_2</module>
+      <module>export LD_LIBRARY_PATH=$NCARG_ROOT/lib:$NCO_PATH/lib:$NETCDF4_PATH/lib:$LD_LIBRARY_PATH</module>
+      <module>export PATH=$NCARG_ROOT/bin:$NCO_PATH/bin:$NETCDF4_PATH/bin:$PATH</module>
+      <module>export PYTHONPATH=/scratch/group/iheso/software/CESM_postprocessing/cesm-env2/lib/python2.7/site-packages/PyNIO:$PYTHONPATH</module>
+<!--      <module>ml NCL/6.6.2-foss-2018b</module>
+                <module>ml NCO/4.7.9-foss-2018b</module>
+      <module>ml netCDF/4.6.1-foss-2018b-cdf5</module>
+      <module>export PYTHONPATH=/ihesp/altuntas/CESM_postprocessing_cdf5/pynio/site-packages:$PYTHONPATH</module>
+      <module>export PYTHONPATH=/ihesp/altuntas/CESM_postprocessing_cdf5/pynio/build/lib.linux-x86_64-2.7/PyNIO:$PYTHONPATH</module>
+      <module>export PYNIOPATH=/ihesp/altuntas/CESM_postprocessing_cdf5/pynio/site-packages/PyNIO</module>-->
+    </modules>
+    <components>
+      <component name="atm">
+      	<averages_pes queue="long" nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</averages_pes>
+      	<diagnostics_pes queue="long" nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</diagnostics_pes>
+      	<regrid_pes queue="long" nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</regrid_pes>
+      	<obs_root>/scratch/group/ihesp/obs_root/amwg/amwg_data</obs_root> 
+      </component>
+      <component name="ice">
+      	<averages_pes queue="long"  nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</averages_pes>
+      	<diagnostics_pes queue="long"  nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</diagnostics_pes>
+      	<obs_root>/scratch/group/ihesp/obs_root/pcwg/ice/data</obs_root>
+      </component>
+      <component name="lnd">
+      	<averages_pes queue="long"  nodes="1" pes_per_node="12" wallclock="20:00:00" memory="18000">12</averages_pes>
+      	<diagnostics_pes queue="long"  nodes="1" pes_per_node="12" wallclock="20:00:00" memory="18000">12</diagnostics_pes>
+      	<regrid_pes queue="long" nodes="1" pes_per_node="12" wallclock="20:00:00" memory="18000">12</regrid_pes>
+      	<obs_root>/scratch/group/ihesp/obs_root/lnd_diag_data</obs_root>
+      </component>
+      <component name="ocn">
+      	<averages_pes queue="long"  nodes="1" pes_per_node="12" wallclock="20:00:00" memory="360000">12</averages_pes>
+      	<diagnostics_pes queue="long"  nodes="1" pes_per_node="10" wallclock="25:00:00" memory="360000">10</diagnostics_pes>
+      	<obs_root>/scratch/group/ihesp/obs_root</obs_root>
+      </component>
+      <component name="ilamb">
+	<diagnostics_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>
+	<initialize_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="06:00:10" memory="18000">3</initialize_pes>
+	<obs_root>/glade/p/cesm/lmwg_dev/oleson/ILAMB/ILAMB_all</obs_root>
+      </component>
+      <component name="iomb">
+	<diagnostics_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>
+	<initialize_pes queue="mn_small"  nodes="2" pes_per_node="1" wallclock="00:00:10" memory="18000">3</initialize_pes>
+	<obs_root>/glade/p/cesm/omwg/obs_data/IOMB</obs_root>
+      </component>
+    </components>
+  </machine>
+
+
   <machine name="dav" hostname="dav,geyser,caldera,pronghorn,casper">
     <timeseries_pes  nodes="4" pes_per_node="16" wallclock="04:00:00" memory="100G">64</timeseries_pes>
     <xconform_pes  nodes="4" pes_per_node="16" wallclock="06:00:00" memory="100G">64</xconform_pes>

diff --git a/Templates/batch_grace.tmpl b/Templates/batch_grace.tmpl
@@ -0,0 +1,15 @@
+#! /bin/bash -l
+
+#SBATCH -n {{ pes }}
+#SBATCH -N {{ nodes }}
+#SBATCH --ntasks-per-node={{ ppn }}
+#SBATCH -t {{ wallclock }}
+#SBATCH -p medium
+#SBATCH -J {{ processName }}
+#SBATCH -A {{ project }}
+#SBATCH --mem {{ memory }}
+#SBATCH -m block
+#SBATCH --mail-type=end          # send email when job ends
+#SBATCH --mail-type=fail         # send email if job fails
+#SBATCH --mail-user=agopal@tamu.edu
+
diff --git a/cesm_utils/cesm_utils/create_postprocess b/cesm_utils/cesm_utils/create_postprocess
@@ -117,7 +117,7 @@ def commandline_options():
                         help='show exception backtraces as extra debugging output')
 
     parser.add_argument('-debug', '--debug', nargs=1, required=False, type=int, default=0,
-                        help='debugging verbosity level output: 0 = none, 1 = minimum, 2 = maximum. 0 is default')
+                        help='debugging verbosity level output: 0 = none, 1 = minimum, 2 = maximum. 2 is default')
 
     parser.add_argument('-caseroot', '--caseroot', nargs=1, required=True, 
                         help='fully quailfied path to case root directory (required)')
@@ -131,6 +131,9 @@ def commandline_options():
     parser.add_argument('-username', '--username', nargs=1, required=False,
                         help='User name (optional). Defaults to user login name.')
 
+    parser.add_argument('-machine', '--machine', nargs=1, required=False,
+                        help='Machine name (optional). Defaults to grace')
+
     parser.add_argument('-add-dav', '--add-dav', dest='add_dav', nargs=1, required=False,
                         help='Fully qualified path to the root of the CESM postprocessing ' \
                         'virtualenv for the NCAR DAV cluster. This option sets the XML ' \
@@ -471,7 +474,7 @@ def create_batch(ppDir, pes, batchTmpl, runTmpl, postProcessCmd, mpiCmd, outFile
     if not rc:
         raise OSError(err_msg)
 
-    debug = '--debug 0'
+    debug = '--debug 2'
     if options.debug:
         debug = '--debug {0}'.format(options.debug[0])
 
@@ -597,6 +600,11 @@ def initialize_main(envDict, options, standalone):
     if options.cesmtag:
         envDict['CESM_TAG'] = options.cesmtag[0]
 
+    # set the project code
+    envDict['MACH_TMP'] = os.getenv('MACH')
+    if options.machine:
+        envDict['MACH_TMP'] = options.machine[0]
+
     # set the project code
     envDict['PROJECT'] = os.getenv('PROJECT')
     if options.project:
@@ -650,7 +658,9 @@ def main(options):
     envDict['PP_CASE_PATH'] = os.path.abspath(pp_case_path)
 
     # get the machine name from the hostname
-    hostname = cesmEnvLib.get_hostname()
+    #hostname = cesmEnvLib.get_hostname()
+    hostname = envDict['MACH_TMP']
+    print hostname
     envDict['MACH'] = cesmEnvLib.get_machine_name(hostname, '{0}/Machines/machine_postprocess.xml'.format(envDict['POSTPROCESS_PATH']))
 
     # check for undefined machine

diff --git a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml
@@ -262,7 +262,7 @@ Applies to both model and control cases."
   <entry id="ICEDIAG_VAR_NAME_TYPE_DIFF"
          type="char"
          valid_values="CSIM/CICE/SIMIP"
-         value="SIMIP"
+         value="CICE"
          group="diff_dataset"
          desc="OLD for u, v var names. NEW for uvel, vvel var names."
          ></entry>
@@ -346,15 +346,15 @@ Applies to both model and control cases."
   <entry id="ICEDIAG_GRIDFILECONT"
          type="char"
          valid_values=""
-         value="$OCNDIAG_OBSROOTPATH/grids/($ICEDIAG_GRID_CONT)_grid.nc"
+         value="/scratch/group/ihesp/obs_root/omwg/grids/($ICEDIAG_GRID_CONT)_grid.nc"
          group="input_path"
          desc="Grid file for control case. Note - this is an ocean grid file."
          ></entry>
 
   <entry id="ICEDIAG_GRIDFILEDIFF"
          type="char"
          valid_values=""
-         value="$OCNDIAG_OBSROOTPATH/grids/($ICEDIAG_GRID_DIFF)_grid.nc"
+         value="/scratch/group/ihesp/obs_root/omwg/grids/($ICEDIAG_GRID_DIFF)_grid.nc"
          group="input_path"
          desc="Grid file for diff case. Note - this is an ocean grid file."
          ></entry>

diff --git a/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml b/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml
@@ -200,7 +200,7 @@ Applies to both model and control cases."
       <entry id="OCNDIAG_TIMESERIES_OBSPATH"
 	     type="char"
 	     valid_values=""
-	     value="$OCNDIAG_DIAGOBSROOT/omwg/timeseries_obs"
+	     value="$OCNDIAG_DIAGOBSROOT/omwg/timeseries_obs_($OCN_GRID)_($OCNDIAG_VERTICAL)lev"
 	     group="observations"
 	     desc="This value points to the local machine's fully qualified directory path location of the OMWG timeseries observational data. This path is used as the input mean_diff_rms_obs_dir to the averager specifier class to create the regional horizontal means averages for SALT and TEMP."
 	     ></entry>
@@ -232,15 +232,15 @@ Applies to both model and control cases."
       <entry id="OCNDIAG_FLUXOBSDIR_CPL7"
 	     type="char"
 	     valid_values=""
-	     value="$OCNDIAG_OBSROOTPATH/fluxes/Data/a.b27.03"
+	     value="$OCNDIAG_OBSROOTPATH/fluxes/Data_hires/a.e13.AIAF.T62_t12.001"
 	     group="observations"
 	     desc="Local machine directory to observation fluxes when CPL value is set to 7"
 	     ></entry>
 
       <entry id="OCNDIAG_FLUXOBSFILE_CPL7"
 	     type="char"
 	     valid_values=""
-	     value="a.b27.03.mean.1984-2006.nc"
+	     value="a.e13.AIAF.T62_t12.001.cpl.ha.1982-2007.nc"
 	     group="observations"
 	     desc="Local machine file to observation fluxes when CPL value is set to 7"
 	     ></entry>
@@ -304,7 +304,7 @@ Applies to both model and control cases."
       <entry id="OCNDIAG_TOBSFILE"
 	     type="char"
 	     valid_values=""
-	     value="PHC2_TEMP_($OCN_GRID)_ann_avg.nc"
+	     value="PHC2_TEMP_($OCN_GRID)_($OCNDIAG_VERTICAL)lev_ann_avg.nc"
 	     group="observations"
 	     desc="Local machine file for temperature observations"
 	     ></entry>

diff --git a/env_reqs.txt b/env_reqs.txt
@@ -1,11 +1,11 @@
-Cython
-scipy
-numpy
-pandas
-netCDF4
-mpi4py
-cf_units
-cftime
-matplotlib
-Jinja2
-sympy
+Cython==0.29.21
+scipy==1.2.3
+numpy==1.16.6
+pandas==0.24.2
+netCDF4==1.5.4
+mpi4py==3.0.3
+cf_units==2.1.4
+cftime==1.2.1
+matplotlib==2.2.5
+Jinja2==2.11.2
+sympy==1.5.1