Merge pull request #1849 from GEOS-ESM/feature/bmauer/fixes-#1846

Fixes #1846

CHANGELOG.md

@@ -9,10 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added new test case for ExtData2G to test multiple input files
+
 ### Changed
 
 ### Fixed
 
+- Implemented workaround for gfortran runtime issue with YafYaml when parsing multiple ExtData yaml files for ExtData2G
+
 ### Removed
 
 ### Deprecated

Tests/ExtData_Testing_Framework/test_cases/case17/AGCM1.rc

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case17/AGCM2.rc

@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000

Tests/ExtData_Testing_Framework/test_cases/case17/CAP.rc

@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::

Tests/ExtData_Testing_Framework/test_cases/case17/CAP1.rc

@@ -0,0 +1,26 @@
+USE_EXTDATA2G: .true.
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case17/CAP2.rc

@@ -0,0 +1,16 @@
+USE_EXTDATA2G: .true.
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+

Tests/ExtData_Testing_Framework/test_cases/case17/ExtData.rc

@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%

Tests/ExtData_Testing_Framework/test_cases/case17/HISTORY1.rc

@@ -0,0 +1,20 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: vars_2d
+             vars_3d
+::
+
+  vars_2d.template:  '%y4.nc4',
+  vars_2d.format:    'CFIO',
+  vars_2d.frequency:  010000,
+  vars_2d.duration: 000000
+  vars_2d.fields: 'VAR2D', 'Root',
+                          ::
+
+  vars_3d.template:  '%y4.nc4',
+  vars_3d.format:    'CFIO',
+  vars_3d.frequency:  010000,
+  vars_3d.duration: 000000
+  vars_3d.fields: 'VAR3D', 'Root',
+                          ::

Tests/ExtData_Testing_Framework/test_cases/case17/HISTORY2.rc

@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::

Tests/ExtData_Testing_Framework/test_cases/case17/README

@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology

Tests/ExtData_Testing_Framework/test_cases/case17/extdata.yaml

@@ -0,0 +1,3 @@
+subconfigs:
+ - extdata_2d.yaml
+ - extdata_3d.yaml

Tests/ExtData_Testing_Framework/test_cases/case17/extdata_2d.yaml

@@ -0,0 +1,4 @@
+Collections:
+   twod_fstream: {template: vars_2d.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: twod_fstream}

Tests/ExtData_Testing_Framework/test_cases/case17/extdata_3d.yaml

@@ -0,0 +1,4 @@
+Collections:
+   threed_fstream: {template: vars_3d.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR3D: {variable: VAR3D, collection: threed_fstream}

Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt

@@ -14,6 +14,7 @@ case13
 case14
 case15
 case16
+case17
 case18
 case19
 case20

Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md

@@ -21,7 +21,7 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin --
 14. Testing that we can take a climatology for a non-leap year and interpolate to a leap year. Daily files each with 1 value for 2007 (non-leap year). Interpolate to 02/29/2008 (leap year)
 15. Testing that we can take a climatology for a non-leap year and interpolate to a non-leap year. Daily files each with 1 value for 2007 (non-leap year). Interpolate to 03/29/2006 (leap year)
 16. Testing that we can take a climatology for a leap year and interpolate to a non-leap year. Daily files each with 1 value for 2008 (leap year). Interpolate to 03/29/2006 15z (leap year)
-17. Not used
+17. Test ability of ExtData (2G only) to allow for subconfigs, i.e. split input yaml files into multiple files
 18. Test vector regridding
 19. Test set file to /dev/null
 20. Make daily files for 2016. Then start on February 29th in 2020 and allow extrapolation outside of dataset a climatology

gridcomps/ExtData2G/CMakeLists.txt

@@ -20,6 +20,8 @@ set (srcs
      ExtData_IOBundleMod.F90
      ExtData_IOBundleVectorMod.F90
      ExtDataMasking.F90
+     ExtDataYamlNodeStack.F90
+     ExtDataYamlNodeWrapper.F90
     )
 
 

gridcomps/ExtData2G/ExtDataConfig.F90

@@ -17,6 +17,8 @@ module MAPL_ExtDataConfig
    use MAPL_ExtDataTimeSampleMap
    use MAPL_TimeStringConversion
    use MAPL_ExtDataMask
+   use MAPL_ExtDataYamlNodeStack
+   use MAPL_ExtDataYamlNodeWrapper
    implicit none
    private
 
@@ -49,7 +51,11 @@ recursive subroutine new_ExtDataConfig_from_yaml(ext_config,config_file,current_
       integer, optional, intent(out) :: rc
 
       type(Parser)              :: p
+#ifndef __GFORTRAN__
       class(YAML_Node), allocatable :: config
+#else
+      integer :: my_stack
+#endif
       class(YAML_Node), pointer :: subcfg, ds_config, rule_config, derived_config, sample_config, subconfigs, rule_map
       class(NodeIterator), allocatable :: iter
       character(len=:), pointer :: key
@@ -74,6 +80,12 @@ recursive subroutine new_ExtDataConfig_from_yaml(ext_config,config_file,current_
       inquire(file=trim(config_file),exist=file_found)
       _ASSERT(file_found,"could not find: "//trim(config_file))
 
+#ifdef __GFORTRAN__
+      stack_depth = stack_depth + 1
+      _ASSERT(stack_depth <= MAX_FILE_DEPTH,"yaml config stack too small")
+      my_stack = stack_depth
+      associate(config => yaml_node_stack(my_stack)%a_yaml_node)   
+#endif
       p = Parser('core')
       config = p%load(config_file,rc=status)
       if (status/=_SUCCESS) then
@@ -88,7 +100,12 @@ recursive subroutine new_ExtDataConfig_from_yaml(ext_config,config_file,current_
            call new_ExtDataConfig_from_yaml(ext_config,sub_file,current_time,rc=status)
            _VERIFY(status)
          end do
+#ifdef __GFORTRAN__
+         deallocate(config)
+         config = p%load(config_file,rc=status)
+#endif
       end if
+
 
       if (config%has("Samplings")) then
          sample_config => config%of("Samplings")
@@ -164,6 +181,9 @@ recursive subroutine new_ExtDataConfig_from_yaml(ext_config,config_file,current_
          _VERIFY(status)
       end if
 
+#ifdef __GFORTRAN__
+      end associate
+#endif
       _RETURN(_SUCCESS)
    end subroutine new_ExtDataConfig_from_yaml
 

gridcomps/ExtData2G/ExtDataYamlNodeStack.F90

@@ -0,0 +1,14 @@
+module MAPL_ExtDataYamlNodeStack
+   use yaFyaml
+   use MAPL_ExtDataYamlNodeWrapper
+   implicit none
+
+   public :: max_file_depth 
+   public :: stack_depth
+   public :: yaml_node_stack
+
+   integer, save :: stack_depth = 0
+   integer, parameter :: MAX_FILE_DEPTH = 20
+   type(YamlNodeWrapper), save :: yaml_node_stack(MAX_FILE_DEPTH)
+
+end module MAPL_ExtDataYamlNodeStack

gridcomps/ExtData2G/ExtDataYamlNodeWrapper.F90

@@ -0,0 +1,11 @@
+module MAPL_ExtDataYamlNodeWrapper
+   use yaFyaml
+   implicit none
+
+   public YamlNodeWrapper
+
+   type :: YamlNodeWrapper
+      class(YAML_Node), allocatable :: a_yaml_node
+   end type
+
+end module MAPL_ExtDataYamlNodeWrapper