Fixes to installation process and template defaults

abishekg7 · Oct 6, 2020 · 74a1cfa · 74a1cfa
1 parent 6d6b14a
commit 74a1cfa
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 24 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -34,9 +34,9 @@ local_path = ilamb/ilamb
 required = True
 
 [ocean_remap]
-tag = master
+branch = devel
 protocol = git
-repo_url = https://github.com/bertinia/ocean_remap
+repo_url = https://github.com/NCAR/ocean_remap.git
 local_path = ocean_remap/ocean_remap
 required = True
 

diff --git a/Machines/ada_modules b/Machines/ada_modules
@@ -3,14 +3,13 @@
 echo "Loading modules for CESM_postprocessing tool..."
 
 module purge
-ml Anaconda/2-5.0.1
+#ml Anaconda/2-5.0.1
+ml Miniconda2/4.3.21
 ml NCL/6.6.2-foss-2018b
 ml NCO/4.7.9-foss-2018b
 ml netCDF/4.6.1-foss-2018b-cdf5
 #ml python/2.7.13-generic
 #ml Python/2.7.15-foss-2018b
-#ml Anaconda3/5.2.0
-#ml Miniconda2/4.3.21
 
 #source /ihesp/user/agopal/CESM_postprocessing/cesm-env2/bin/activate
 #export PYTHONPATH=/ihesp/user/agopal/CESM_postprocessing/pynio/site-packages:$PYTHONPATH

diff --git a/Machines/machine_postprocess.xml b/Machines/machine_postprocess.xml
@@ -325,7 +325,7 @@
       <module>module purge</module>
     </reset_modules>
     <modules>
-      <module>ml Anaconda/2-5.0.1</module>	    
+      <module>ml Miniconda2/4.3.21</module>	    
       <module>ml NCL/6.6.2-foss-2018b</module>
       <module>ml NCO/4.7.9-foss-2018b</module>
       <module>ml netCDF/4.6.1-foss-2018b-cdf5</module>
@@ -339,12 +339,12 @@
 	<averages_pes queue="mn_small" nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</averages_pes>
 	<diagnostics_pes queue="mn_small" nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>
 	<regrid_pes queue="mn_small" nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</regrid_pes>
-	<obs_root>/glade/p/cesm/amwg/amwg_data</obs_root> <!-- TODO: update this and other obs_roots -->
+	<obs_root>/scratch/training/CESM_post/obs_root/amwg/amwg_data</obs_root> <!-- TODO: update this and other obs_roots -->
       </component>
       <component name="ice">
 	<averages_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</averages_pes>
 	<diagnostics_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>
-	<obs_root>/glade/p/cesm/pcwg/ice/data</obs_root>
+	<obs_root>/scratch/training/CESM_post/obs_root/pcwg/ice/data</obs_root>
       </component>
       <component name="lnd">
 	<averages_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</averages_pes>
@@ -355,7 +355,7 @@
       <component name="ocn">
 	<averages_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</averages_pes>
 	<diagnostics_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>
-	<obs_root>/scratch/user/altuntas/obsroot</obs_root> <!-- TODO: move to a permanent dir -->
+	<obs_root>/scratch/training/CESM_post/obs_root</obs_root> <!-- TODO: move to a permanent dir -->
       </component>
       <component name="ilamb">
 	<diagnostics_pes queue="mn_small"  nodes="1" pes_per_node="3" wallclock="6:00" memory="18000">3</diagnostics_pes>

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 ## 0. Load all necessary modules
 ```
 module purge
-ml Anaconda/2-5.0.1
+ml Miniconda2/4.3.21
 ml NCL/6.6.2-foss-2018b
 ml NCO/4.7.9-foss-2018b
 ml netCDF/4.6.1-foss-2018b-cdf5
@@ -14,6 +14,7 @@ ml netCDF/4.6.1-foss-2018b-cdf5
 ```
 git clone https://github.com/abishekg7/CESM_postprocessing.git
 cd CESM_postprocessing
+export POSTPROCESS_PATH=`pwd`
 ```
 
 ## 2. Install virtual environment:
@@ -23,8 +24,7 @@ cd CESM_postprocessing
 ## 3. Activate virtual environment:
 ```
 ml purge 
-ml Anaconda/2-5.0.1
-export POSTPROCESS_PATH=<path to your CESM_Postprocessing root>
+ml Miniconda2/4.3.21
 source activate $POSTPROCESS_PATH/cesm-env2
 ```
 ## 4. Create your postprocessing instance

diff --git a/Templates/postprocess.tmpl b/Templates/postprocess.tmpl
@@ -28,10 +28,11 @@ pp_dav_path=`./pp_config --get POSTPROCESS_PATH_DAV --value`
 
 ## activate the virtualenv that contains all the non-bootstrapped dependencies
 
-cd {{ virtualEnvDir }}
-echo "Running from virtualenv directory:"
-pwd
-. activate
+#cd {{ virtualEnvDir }}
+#echo "Running from virtualenv directory:"
+#pwd
+#. activate
+
 
 ## load the boot-strap modules 
 
@@ -40,6 +41,8 @@ pwd
 {% endfor %}
 
 
+source activate {{ virtualEnvRoot }}
+
 {% if pythonpath|length > 0 %}
 ## The PYTHONPATH setting here is for libraries that are not available via modules.
 PYTHONPATH={{ pythonpath }}:$PYTHONPATH

diff --git a/cesm_utils/cesm_utils/create_postprocess b/cesm_utils/cesm_utils/create_postprocess
@@ -63,6 +63,7 @@ except KeyError:
 
 cesm_pp_path = os.environ["POSTPROCESS_PATH"]
 
+print ('Using PP Environment at {0}'.format(cesm_pp_path))
 #
 # activate the virtual environment that was created by create_python_env
 #
@@ -464,12 +465,13 @@ def create_batch(ppDir, pes, batchTmpl, runTmpl, postProcessCmd, mpiCmd, outFile
         raise OSError(err_msg)
 
     virtualEnvDir = '{0}/cesm-env2/bin'.format(ppDir)
+    virtualEnvRoot = '{0}/cesm-env2'.format(ppDir)
     # check that the postProcessCmd exists in the virtualenv
     rc, err_msg = cesmEnvLib.checkFile('{0}/{1}'.format(virtualEnvDir, postProcessCmd), 'read')
     if not rc:
         raise OSError(err_msg)
 
-    debug = '--debug 0'
+    debug = '--debug 2'
     if options.debug:
         debug = '--debug {0}'.format(options.debug[0])
 
@@ -511,6 +513,7 @@ def create_batch(ppDir, pes, batchTmpl, runTmpl, postProcessCmd, mpiCmd, outFile
                      'postProcessCmd' : postProcessCmd,
                      'caseRoot' : caseRoot,
                      'virtualEnvDir' : virtualEnvDir,
+                     'virtualEnvRoot' : virtualEnvRoot,
                      'debug' : debug,
                      'backtrace' : backtrace,
                      'standalone' : standalone,
@@ -644,7 +647,7 @@ def main(options):
 
     # initialize the environment dictionary
     envDict = initialize_main(envDict, options, standalone)
-    envDict['PP_CASE_PATH'] = pp_case_path
+    envDict['PP_CASE_PATH'] = os.path.abspath(pp_case_path)
 
     # get the machine name from the hostname
     hostname = cesmEnvLib.get_hostname()

diff --git a/create_python_env b/create_python_env
@@ -145,8 +145,8 @@ if [ -d $env ]; then
 else
    export CONDA_ENVS_PATH=${pp_dir}
    export CONDA_PKGS_DIRS=${pp_dir}/conda_pkgs
-   #conda create python=2.7 -n cesm-env2
-   virtualenv --system-site-packages -p python2 cesm-env2
+   conda create python=2.7 -n cesm-env2
+   #virtualenv --system-site-packages -p python2 cesm-env2
 fi
 
 #----------------------------------------------------------------------
@@ -166,15 +166,14 @@ echo "$progname - activating virtual environment in ${pp_dir}/cesm-env2."
 source activate ${pp_dir}/cesm-env2
 export PATH=${pp_dir}/cesm-env2/bin:$PATH
 
-pip install numpy==1.16.5
-pip install mpi4py==2.0
-#conda install -c conda-forge scipy==1.1.0
+conda  install -c conda-forge numpy==1.16.5 mpi4py==2.0 cftime
+conda install -c conda-forge scipy==1.1.0
 #conda install -c conda-forge pandas==0.23.1
 #conda install -c conda-forge netCDF4==1.5.3
 #conda install -c conda-forge mpi4py==2.0.0
 #conda install -c conda-forge matplotlib
 #conda install -c conda-forge Jinja2==2.7.3
-#conda install -c conda-forge sympy
+conda install -c conda-forge matplotlib sympy Jinja2 markupsafe
 #cesm-env2/bin/pip install -r env_reqs.txt 
 
 cp Templates/activate_this.py ${pp_dir}/cesm-env2/bin/