New boolean in analysis_compare to run report maker only

README.md

@@ -248,6 +248,7 @@ These values are:
  - `do_analysis_timeseries`:
    -  A Boolean value to run or skip the single model timeseries.
    -  Set to False if the single model analysis has already completed.
+   -  This value is overwritten by the `analysis_compare -s` which skips new timeseries analyses.
  - `do_mass_download`:
    - A boolean value to run the mass download.
    - This is not currently possible as we can only download mass file from mass-cli1 on jasmin.
@@ -293,6 +294,15 @@ These values are:
 A sample yaml exists in `input_yml/comparison_analysis_template.yml`,
 which can be adapted to additional analyses.
 
+In order to only run the report making part of the comparison analysis 
+(skip the `analysis_timeseries part`),
+either set the `do_analysis_timeseries` key to `False` in the `input_yml` file,
+or run `analysis_compare` with the command line argument: `-s` or --skip-timeseries.
+To force the analysis timeseries command to run use `--no-skip-timeseries`.
+Though without without either of these command line arguments, 
+bgcval2 will default to the value in your `input_yml` file.
+Also, the command line argument overwrites the value in `input_yml`.
+
 Once the comparison suite has been run, members of the esmeval group workspace on JASMIN
 can copy the html report to a web-visible directory, using the script:
 

bgcval2/analysis_compare.py

@@ -336,15 +336,13 @@ def timeseries_compare(jobs,
                 if strictFileCheck:
                     raise FileError('Model Files are not found jobID: %s, name: %s', jobID, name)
 
-
             if 'dataFile' in av[name] and not os.path.exists(av[name]['dataFile']):
                 print(
                     "analysis-Timeseries.py:\tWARNING:\tdata file is not found:",
                     av[name]['dataFile'])
                 if strictFileCheck:
                     raise FileError('Data Files are not found jobID: %s, name: %s', jobID, name)
 
-
             #####
             # time series and traffic lights.
             tsa = timeseriesAnalysis(
@@ -368,6 +366,8 @@ def timeseries_compare(jobs,
                 clean=False,
                 noNewFiles=True,
             )
+
+
             #dataD[(jobID,name )] = tsa.dataD
             modeldataD[(jobID, name)] = tsa.modeldataD
 
@@ -378,12 +378,6 @@ def timeseries_compare(jobs,
 
     ####
     for name in av.keys():
-#   for name in [
-#           'Temperature', 'Salinity', 'MLD', 'FreshwaterFlux',
-#           'AirSeaFluxCO2', 'AirSeaFlux', 'Chlorophyll', 'Nitrate',
-#           'Alkalinity', 'pH'
-#   ]:
-#       if name not in list(av.keys()): continue
         regions = av[name]['regions']
         layers = av[name]['layers']
         metrics = av[name]['metrics']
@@ -397,14 +391,11 @@ def timeseries_compare(jobs,
                     continue
                 title = titleify([region, layer, metric, name])
 
-                    #timesD[jobID] 	= sorted(mdata.keys())
-                    #arrD[jobID]	= [mdata[t] for t in timesD[jobID]]
-
                 times, datas = apply_shifttimes(mdata, jobID, shifttimes)
                 print('post apply_shifttimes:', len(times), len(datas))
                 times, datas = apply_timerange(times, datas, jobID, timeranges)
-                timesD[jobID] = times  #mdata.keys())
-                arrD[jobID] = datas  #t] for t in timesD[jobID]]
+                timesD[jobID] = times 
+                arrD[jobID] = datas  
                 print(jobID, region, layer, metric, len(times), len(datas))
             timesD, arrD = build_ensemble(timesD, arrD, ensembles)
 
@@ -413,7 +404,7 @@ def timeseries_compare(jobs,
             units = av[name]['modeldetails']['units']
 
             ts = 'Together'
-            for ls in ['DataOnly', ]: #  'movingav30years']
+            for ls in ['DataOnly', ]:
                 tsp.multitimeseries(
                     timesD,  # model times (in floats)
                     arrD,  # model time series
@@ -444,7 +435,7 @@ def timeseries_compare(jobs,
     if ensembles != {}:
         jobs = list(ensembles.keys())
 
-    # Senmd everything to the comparison maker:
+    # Send everything to the comparison maker:
     comparehtml5Maker(
         jobIDs=jobs,
         reportdir=bvt.folder('CompareReports2/' + analysisname),
@@ -455,6 +446,7 @@ def timeseries_compare(jobs,
         jobColours=colours,
         paths=paths,
     )
+    print('End of timeseries_compare')
 
 
 def flatten(lats, lons, dataA, dataB):
@@ -546,12 +538,10 @@ def load_comparison_yml(master_compare_yml_fn):
     details['timeranges'] = timeranges
     details['suites'] = suites
     details['auto_download'] = auto_download_dict
-#    print(details)
-#    assert 0
     return details
 
 
-def load_yml_and_run(compare_yml, config_user):
+def load_yml_and_run(compare_yml, config_user, skip_timeseries):
     """
     Loads the comparison yaml file and run compare_yml.
 
@@ -565,6 +555,11 @@ def load_yml_and_run(compare_yml, config_user):
     do_mass_download = details['do_mass_download']
     master_suites = details['master_suites']
 
+    if skip_timeseries == None:
+        pass
+    else:
+        do_analysis_timeseries = not skip_timeseries
+
     colours = details['colours']
     thicknesses = details['thicknesses']
     linestyles = details['linestyles']
@@ -649,6 +644,13 @@ def get_args():
                         help='User configuration file (for paths).',
                         required=False)
 
+    parser.add_argument('--skip_timeseries',
+                        '-s',
+                        default=None, 
+                        help='When True: skip the new timeseries analyses and make the html report. Overwrites the do_analysis_timeseries flag in input_yml.',
+                        action=argparse.BooleanOptionalAction,
+                        required=False)
+
     args = parser.parse_args()
 
     return args
@@ -670,8 +672,8 @@ def main():
         if not os.path.isfile(compare_yml):
             print(f"analysis_timeseries: Could not find comparison config file {compare_yml}")
             sys.exit(1)
-
-        load_yml_and_run(compare_yml, config_user)
+        skip_timeseries = args.skip_timeseries  
+        load_yml_and_run(compare_yml, config_user, skip_timeseries)
 
     print("Finished... ")
 

input_yml/TerraFIRMA_overshoot_historical.yml

@@ -0,0 +1,69 @@
+---
+# GC5 N96 ORCA1 spinup analysis
+name: TerraFIRMA_overshoot_historical
+
+# Run the single job analysis
+do_analysis_timeseries: True 
+
+# Download from mass:
+do_mass_download: False
+
+# master analysis suite
+master_suites: physics bgc #alkalinity physics kmf1
+
+clean: True
+
+jobs:
+    u-ca306:
+       description: 'Reference '
+       colour: 'black'
+       thickness: 0.6
+       linestyle: '-'
+       shifttime: 0.
+       timerange: [1800, 1900]
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cy623:
+       description: 'interactive ice, started from picontrol yr 2277'
+       colour: red  
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cy690: 
+       description: 'static ice sheets, started from picontrol yr 2277'
+       colour: purple
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cy691:
+       description: 'static ice sheets, started from picontrol yr 2197'
+       colour: blue
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics
+
+    u-cy692:
+       description: ' static ice sheets, started from picontrol yr 2237'
+       colour: green
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics kmf
+
+    u-cy693:
+       description: ' static ice sheets, started from picontrol yr 2317'
+       colour: goldenrod
+       thickness: 1.7
+       linestyle: '-'
+       shifttime: 0.
+       suite: kmf physics bgc #alkalinity physics kmf
+
+
+
+
+

key_files/alkalinity.yml

@@ -19,4 +19,4 @@ model_convert   : NoChange
 #    function: convertmeqm3TOumolkg
 
 layers          : Surface #50m #;100m 200m 500m 1000m 2000m
-regions         : Global #ignoreInlandSeas #;SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific

key_files/atmospco2.yml

@@ -8,6 +8,7 @@ modelFiles      : $BASEDIR_MODEL/$JOBID/medusa*$JOBIDo_1y_*_diad-T.nc
 model_vars      : ATM_PCO2
 model_convert   : NoChange
 layers          : layerless
-regions         : Global #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+
 
 

key_files/chlorophyll.yml

@@ -14,7 +14,7 @@ model_vars      : CHD CHN
 model_convert   : sum
 
 layers          : Surface
-regions         : Global #ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
 
 
 

key_files/dic.yml

@@ -15,4 +15,5 @@ model_convert   : NoChange
 
 
 layers          : Surface #50m #;100m 200m 500m 1000m 2000m
-regions         : Global #ignoreInlandSeas #;SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+

key_files/dust.yml

@@ -9,6 +9,7 @@ model_vars      : AEOLIAN
 model_convert:
     function: NoChange
 layers          : layerless
-regions         : Global 
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+
 
 

key_files/iron.yml

@@ -25,7 +25,8 @@ model_convert   :
 #data_convert_factor : 1000
 
 layers          : Surface #Transect ;CanRusTransect PTransect SOTransect Equator ArcTransect AntTransect ArcTransect AntTransect
-regions         : Global
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+
 
 
 

key_files/mld.yml

@@ -26,4 +26,5 @@ data_convert:
     maskname : mask
     areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc
 layers          : Surface
-regions         : Global  SouthernOcean
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+

key_files/nitrate.yml

@@ -21,6 +21,7 @@ data_convert    : NoChange
 data_tdict      : ZeroToZero
 
 layers          : Surface 50m #;100m 200m 500m 1000m 2000m
-regions         : Global ignoreInlandSeas #;SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+
 
 

key_files/oxygen.yml

@@ -21,5 +21,5 @@ data_convert:
     factor : 44.661
 
 layers          : Surface 100m 500m
-regions         : Global #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global SouthernOcean Equator10 #gnoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
 

key_files/salinity.yml

@@ -20,4 +20,4 @@ data_convert    : NoChange
 data_tdict      : ZeroToZero
 
 layers          : Surface
-regions         : Global #ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific

key_files/silicate.yml

@@ -19,6 +19,7 @@ data_convert    : NoChange
 data_tdict      : ZeroToZero
 
 layers          : Surface
-regions         : Global  SouthernOcean # ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific ignoreInlandSeas
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf
+
 
 

key_files/temperature.yml

@@ -21,7 +21,7 @@ data_convert    : NoChange
 data_tdict      : ZeroToZero
 
 layers          : Surface
-regions         : Global #  OnShelf OffShelf #ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific
+regions         : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific # OnShelf OffShelf