batch script for generating parameter file from a diff file

parameter_files/patch_default_bciopt224.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<all>
+  <notes>This parameter dataset was created by Ryan Knox rgknox@lbl.gov. Please contact if using in published work. The calibration uses the following datasets: [1] Ely et al. 2019. Leaf mass area, Panama. NGEE-Tropics data collection.http://dx.doi.org/10.15486/ngt/1411973 and [2] Condit et al. 2019. Complete data from the Barro Colorado 50-ha plot. https://doi.org/10.15146/5xcp-0d46. [3] Koven et al. 2019. Benchmarking and parameter sensitivity of physiological and vegetation dynamics using the functionally assembled terrestrial ecosystem simulator. Biogeosciences. The ECA nutrient aquisition parmeters are unconstrained, the file output naming convention vmn6phi is shorthand for vmax for nitrogen uptake is order e-6 and for phosphorus is excessively high.  These parameters were calibrated with the special fates modification in main/EDTypesMod.F90: nclmax = 3</notes>
+  <base_file>fates_params_default.cdl</base_file>
+  <new_file>fates_params_opt224_vmn6phi_080621.cdl</new_file>
+  <pft_list>1</pft_list>
+  <parameters>
+	 <fates_prescribed_nuptake> 0 </fates_prescribed_nuptake>
+	 <fates_prescribed_puptake> 0 </fates_prescribed_puptake>
+	 <fates_prt_alloc_priority> 1,1,3,4 </fates_prt_alloc_priority>
+	 <fates_prt_nitr_stoich_p1> 0.03347526,0.024,1e-08,0.0047 </fates_prt_nitr_stoich_p1>
+	 <fates_prt_nitr_stoich_p2> 0.03347526,0.024,1e-08,0.0047 </fates_prt_nitr_stoich_p2>
+	 <fates_turnover_carb_retrans> 0.025,0,0,0 </fates_turnover_carb_retrans>
+	 <fates_turnover_nitr_retrans> 0.45,0.25,0,0 </fates_turnover_nitr_retrans>
+	 <fates_leaf_long> 0.8012471 </fates_leaf_long>
+	 <fates_leaf_vcmax25top> 30.94711 </fates_leaf_vcmax25top>
+	 <fates_allom_agb1> 0.0673 </fates_allom_agb1>
+	 <fates_allom_agb2> 0.976 </fates_allom_agb2>
+	 <fates_allom_agb3> -9 </fates_allom_agb3>
+	 <fates_allom_agb4> -9 </fates_allom_agb4>
+	 <fates_allom_amode> 3 </fates_allom_amode>
+	 <fates_allom_d2bl1> 0.1266844 </fates_allom_d2bl1>
+	 <fates_allom_d2bl2> 1.281329 </fates_allom_d2bl2>
+	 <fates_allom_d2bl3> -9 </fates_allom_d2bl3>
+	 <fates_allom_d2ca_coefficient_max> 0.768654 </fates_allom_d2ca_coefficient_max>
+	 <fates_allom_d2ca_coefficient_min> 0.768654 </fates_allom_d2ca_coefficient_min>
+	 <fates_allom_d2h1> 57.6 </fates_allom_d2h1>
+	 <fates_allom_d2h2> 0.74 </fates_allom_d2h2>
+	 <fates_allom_d2h3> 21.6 </fates_allom_d2h3>
+	 <fates_allom_dbh_maxheight> 200 </fates_allom_dbh_maxheight>
+	 <fates_allom_fmode> 2 </fates_allom_fmode>
+	 <fates_allom_hmode> 5 </fates_allom_hmode>
+	 <fates_allom_l2fr> 0.4863088 </fates_allom_l2fr>
+	 <fates_allom_lmode> 3 </fates_allom_lmode>
+	 <fates_eca_vmax_nh4> 3e-06 </fates_eca_vmax_nh4>
+	 <fates_eca_vmax_no3> 3e-06 </fates_eca_vmax_no3>
+	 <fates_eca_vmax_p> 3e-07 </fates_eca_vmax_p>
+	 <fates_eca_vmax_ptase> 3e-08 </fates_eca_vmax_ptase>
+	 <fates_leaf_slamax> 0.03991654 </fates_leaf_slamax>
+	 <fates_leaf_slatop> 0.01995827 </fates_leaf_slatop>
+	 <fates_mort_bmort> 0.01303514 </fates_mort_bmort>
+	 <fates_mort_scalar_cstarvation> 0.02955703 </fates_mort_scalar_cstarvation>
+	 <fates_nitr_store_ratio> 3 </fates_nitr_store_ratio>
+	 <fates_phos_store_ratio> 3 </fates_phos_store_ratio>
+	 <fates_seed_alloc> 0.04680188 </fates_seed_alloc>
+	 <fates_seed_suppl> 0.001 </fates_seed_suppl>
+	 <fates_wood_density> 0.8374751 </fates_wood_density>
+	 <fates_comp_excln> -1 </fates_comp_excln>
+	 <fates_mort_disturb_frac> 0.5 </fates_mort_disturb_frac>
+	 <fates_mort_understorey_death> 1 </fates_mort_understorey_death>
+  </parameters>
+</all>

parameter_files/patch_default_e3smtest.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<all>
+  <base_file>fates_params_default.cdl</base_file>
+  <new_file>fates_params_e3smtest.cdl</new_file>
+  <pft_list>1,2,3,4,5,6,7,8,9,10,11,12</pft_list>
+  <parameters>
+	 <fates_prescribed_nuptake>0,0,0,0,0,0,0,0,0,0,0,0</fates_prescribed_nuptake>
+	 <fates_prescribed_puptake>0,0,0,0,0,0,0,0,0,0,0,0</fates_prescribed_puptake>
+  </parameters>
+</all>

tools/BatchPatchParams.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+#### this script modifies the default FATES parameter file to generate
+#    a file used in testing E3SM
+#    Parser code was based off of modify_fates_paramfile.py
+
+import os
+import argparse
+import code  # For development: code.interact(local=dict(globals(), **locals()))
+from scipy.io import netcdf
+
+
+# ---------------------------------------------------------------------------------------
+
+class param_type:
+    def __init__(self,name,values_text):
+        self.name = name
+        self.values = values_text.replace(" ","") #[float(x) for x in values_text.split(',')]
+
+# ---------------------------------------------------------------------------------------
+
+
+def load_xml(xmlfile): 
+
+    import xml.etree.ElementTree as et
+
+    xmlroot = et.parse(xmlfile).getroot()
+    print("\nOpenend: "+xmlfile)
+
+    base_cdl = xmlroot.find('base_file').text
+    new_cdl = xmlroot.find('new_file').text
+
+    pftparams = xmlroot.find('pft_list').text.replace(" ","")
+
+    paramroot = xmlroot.find('parameters') 
+    paramlist = []
+    for param in paramroot:
+        print("parsing "+param.tag)
+        paramlist.append(param_type(param.tag,param.text))
+
+
+
+    return(base_cdl,new_cdl,pftparams,paramlist)
+
+
+
+# Little function for assembling the call to the system to make the modification
+# ----------------------------------------------------------------------------------------
+
+def parse_syscall_str(fnamein,fnameout,param_name,param_val):
+
+    sys_call_str = "../tools/modify_fates_paramfile.py"+" --fin " + fnamein + \
+                   " --fout " + fnameout + " --var " + param_name + " --silent " +\
+                   " --val " + param_val + " --overwrite --all"
+
+    return(sys_call_str)
+
+
+
+def main():
+
+    # Parse arguments
+    parser = argparse.ArgumentParser(description='Parse command line arguments to this script.')
+    parser.add_argument('--f', dest='xmlfile', type=str, help="XML control file  Required.", required=True)
+    args = parser.parse_args()
+
+
+    # Load the xml file, which contains the base cdl, the output cdl,
+    #  and the parameters to be modified
+    [base_cdl,new_cdl,pftlist,paramlist] = load_xml(args.xmlfile)
+
+
+    # Convert the base cdl file into a temp nc binary
+    base_nc = os.popen('mktemp').read().rstrip('\n')
+    gencmd = "ncgen -o "+base_nc+" "+base_cdl
+    os.system(gencmd)
+
+    # Generate a temp output file name
+    new_nc = os.popen('mktemp').read().rstrip('\n')
+
+
+    # Use FatesPFTIndexSwapper.py to prune out unwanted PFTs
+    swapcmd="../tools/FatesPFTIndexSwapper.py --pft-indices="+pftlist+" --fin="+base_nc+" --fout="+new_nc   #+" 1>/dev/null"
+    os.system(swapcmd)
+
+    # We open the new parameter file. We only use this
+    # to do some dimension checking. 
+    fp_nc  = netcdf.netcdf_file(base_nc, 'r')
+
+    # On subsequent parameters, overwrite the file
+    for param in paramlist:
+
+        change_str = parse_syscall_str(new_nc,new_nc,param.name,param.values)
+        os.system(change_str)
+
+    # Sort the new file
+    newer_nc = os.popen('mktemp').read().rstrip('\n')
+    os.system("../tools/ncvarsort.py --fin "+new_nc+" --fout "+newer_nc+" --overwrite")
+
+    # Dump the new file to the cdl
+    os.system("ncdump "+newer_nc+" > "+new_cdl)
+
+    fp_nc.close()
+
+    print("\nBatch parameter transfer complete\n")
+
+
+# This is the actual call to main
+
+if __name__ == "__main__":
+    main()

tools/FatesPFTIndexSwapper.py

@@ -25,7 +25,9 @@
 
 pft_dim_name = 'fates_pft'
 prt_dim_name = 'fates_prt_organs'
-
+hydro_dim_name = 'fates_hydr_organs'
+litt_dim_name = 'fates_litterclass'
+string_dim_name = 'fates_string_length'
 
 class timetype:
 
@@ -165,22 +167,31 @@ def main(argv):
         # Idenfity if this variable has pft dimension
         pft_dim_found = -1
         prt_dim_found = -1
+        hydro_dim_found = -1
+        litt_dim_found  = -1
+        string_dim_found = -1
         pft_dim_len   = len(fp_in.variables.get(key).dimensions)
 
         for idim, name in enumerate(fp_in.variables.get(key).dimensions):
+
             # Manipulate data
             if(name==pft_dim_name):
                 pft_dim_found = idim
             if(name==prt_dim_name):
                 prt_dim_found = idim
-
+            if(name==litt_dim_name):
+                litt_dim_found = idim
+            if(name==hydro_dim_name):
+                hydro_dim_found = idim
+            if(name==string_dim_name):
+                string_dim_found = idim
 
         # Copy over the input data
         # Tedious, but I have to permute through all combinations of dimension position
         if( pft_dim_len == 0 ):
             out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions))
             out_var.assignValue(float(fp_in.variables.get(key).data))
-        elif( (pft_dim_found==-1) & (prt_dim_found==-1) ):
+        elif( (pft_dim_found==-1) & (prt_dim_found==-1) & (litt_dim_found==-1) & (hydro_dim_found==-1)  ):
             out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions))
             out_var[:] = in_var[:]
         elif( (pft_dim_found==0) & (pft_dim_len==1) ):           # 1D fates_pft
@@ -208,14 +219,28 @@ def main(argv):
             for id,ipft in enumerate(donor_pft_indices):
                 out_var[id] = fp_in.variables.get(key).data[ipft-1]
 
-        elif( (prt_dim_found==0) & (pft_dim_len==2) ):          # fates_prt_organs - string_length
+        elif( (prt_dim_found==0) & (pft_dim_len==2) ):         
+            out_var = fp_out.createVariable(key,'c',(fp_in.variables.get(key).dimensions))
+            out_var[:] = in_var[:]
+
+        elif( (hydro_dim_found==0) & (string_dim_found>=0) ):      
             out_var = fp_out.createVariable(key,'c',(fp_in.variables.get(key).dimensions))
             out_var[:] = in_var[:]
+
+        elif( (litt_dim_found==0) & (string_dim_found>=0) ):       
+            out_var = fp_out.createVariable(key,'c',(fp_in.variables.get(key).dimensions))
+            out_var[:] = in_var[:]   
+
+        elif( prt_dim_found==0 ): # fates_prt_organs - indices
+            out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions))
+            out_var[:] = in_var[:]
 
-        elif( prt_dim_found==0 ):
+        elif( litt_dim_found==0 ):
+            out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions))
+            out_var[:] = in_var[:]
+        elif( hydro_dim_found==0):
             out_var = fp_out.createVariable(key,'d',(fp_in.variables.get(key).dimensions))
             out_var[:] = in_var[:]
-
         else:
             print('This variable has a dimensioning that we have not considered yet.')
             print('Please add this condition to the logic above this statement.')