Add restart output for aerosol cycling

Updates the restart_interval setting to make sure it always includes the
STEP_GFS time if GOCART is on. This is done using the advanced functionality
of restart_interval, which allows for an explicit list of restart times
instead of a single interval.

If STEP_GFS is a multiple of restart_interval, it is left unchanged.
If STEP_GFS is not a multiple of restart_interval, restart_interval is
converted to an explicit sequence of restart times, then STEP_GFS is added.
If restart_interval is already an explicit list, STEP_GFS is just added to it.
This may add a redundant restart time, but that is ignored by FV3.

In order to complete this update, the CROW type for restart_interval had to be
changed from int to string to accomodate lists of numbers. This has the added
benefit of allowing explicit definition of restart times in other applications
as well.

As part of this update, the values for restart_interval in the forecast config
were updated to use the appropriate settings rather than hardcoded values.
The default value for the gfs restart_interval was also changed from 6 to 0 to
maintain the previous behavior when the setting was ignored.

Refs: NOAA-EMC#314

workflow/config/fcst.yaml

@@ -39,6 +39,24 @@ config_fcst:
       do: !calc doc.partition_common.resources.run_coupled_fcst
     - otherwise: !calc doc.partition_common.resources.run_gfsfcst
 
+  restart_interval_gfs: !FirstTrue
+    - when: !calc doc.settings.cplgocart == ".false."
+      do: !calc doc.fv3_gfs_settings.restart_interval
+    - otherwise: !FirstTrue
+      - when: !calc doc.fv3_gfs_settings.restart_interval == "0"
+        # Add restart file at forecast cadence in addition to end of forecast
+        do: !calc tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0] + " " + str(doc.output_settings.FHMAX_GFS)
+      - when: !calc len(doc.fv3_gfs_settings.restart_interval.split(' ')) == 1
+        # Make sure forecast cadence is included in restart interval
+        do: !FirstTrue
+          # STEP_GFS is already a multiple of the restart interval, don't change
+          - when: !calc int(tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0])%int(doc.fv3_gfs_settings.restart_interval) == 0
+            do: !calc doc.fv3_gfs_settings.restart_interval
+          # STEP_GFS is not a multiple of the restart interval, convert to explicit list and add
+          - otherwise: !calc tools.join(tools.seq(0,doc.output_settings.FHMAX_GFS,int(doc.fv3_gfs_settings.restart_interval)) + [tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]], ' ')
+      # restart_interval is already an explicit list, make sure STEP_GFS is in the list
+      - otherwise: !calc doc.fv3_gfs_settings.restart_interval + " " + tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]
+
   content: !expand |
     #!/bin/ksh -x
     
@@ -207,7 +225,7 @@ config_fcst:
         fi
     
         # Write restart files at next assimilation time
-        export restart_interval=6
+        export restart_interval={doc.fv3_enkf_settings.restart_interval}
         export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks}
         export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS}
 
@@ -219,10 +237,9 @@ config_fcst:
         else
             export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_orig"
         fi
+        export restart_interval={restart_interval_gfs}
         export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks}
         export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS}
-
-
     fi
 
     if [ $cplgocart = .true. ]; then # temporary settings for aerosol coupling
@@ -231,6 +248,7 @@ config_fcst:
         export CHM_CFGDIR="{doc.chem_settings.CFGDIR}"
         export CHM_INPDIR="{doc.chem_settings.INPDIR}"
         export dnats="$(( $dnats + {doc.chem_settings.ntdiag} ))"
+        export KMP_INIT_AT_FORK=FALSE # Temporary setting
     elif [[ $cpl == .true. ]] ; then # coupled model
         export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_cpl"
     fi

workflow/defaults/fv3_enkf.yaml

@@ -120,7 +120,7 @@ fv3_enkf_defaults: &fv3_enkf_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "6"
   imp_physics: "11"
 #     if_present: !FirstTrue
 #      - when: !calc ( imp_physics==99 )

workflow/defaults/fv3_gdas.yaml

@@ -93,7 +93,7 @@ fv3_gdas_defaults: &fv3_gdas_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "6"
   imp_physics: "11"
   phy_dependent_var: !Select
      select: !calc imp_physics

workflow/defaults/fv3_gfs.yaml

@@ -98,7 +98,7 @@ fv3_gfs_defaults: &fv3_gfs_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "0"
   imp_physics: "11"
   phy_dependent_var: !Select
      select: !calc imp_physics

workflow/schema/fv3.yaml

@@ -53,7 +53,7 @@ fv3_settings_template: !Template &fv3_settings_template
   ICO2:               { type: int }
   warm_start:         { type: bool }
   read_increment:     { type: bool }
-  restart_interval:   { type: int }
+  restart_interval:   { type: string }
 #  FIELD_TABLE:        { type: string }
 
   LEVS: