From 7974c0dae827150bdb6b991114531ea242cba10c Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 25 Sep 2024 08:33:53 -0400 Subject: [PATCH 01/10] Fixes #1618. Add zstandard support --- Apps/Regrid_Util.F90 | 7 ++++- CHANGELOG.md | 4 +++ .../ExtData_Testing_Framework/CMakeLists.txt | 10 +++++++ .../test_cases/case35/AGCM1.rc | 24 +++++++++++++++ .../test_cases/case35/AGCM2.rc | 29 +++++++++++++++++++ .../test_cases/case35/CAP.rc | 4 +++ .../test_cases/case35/CAP1.rc | 25 ++++++++++++++++ .../test_cases/case35/CAP2.rc | 15 ++++++++++ .../test_cases/case35/ExtData.rc | 13 +++++++++ .../test_cases/case35/HISTORY1.rc | 14 +++++++++ .../test_cases/case35/HISTORY2.rc | 5 ++++ .../test_cases/case35/README | 1 + .../test_cases/case35/extdata.yaml | 5 ++++ .../test_cases/extdata_2g_cases.txt | 1 + .../test_cases/test_case_descriptions.md | 1 + gridcomps/History/MAPL_HistoryCollection.F90 | 1 + gridcomps/History/MAPL_HistoryGridComp.F90 | 12 ++++++++ .../History/Sampler/MAPL_EpochSwathMod.F90 | 5 +++- griddedio/FieldBundleWrite.F90 | 10 ++++--- griddedio/GriddedIO.F90 | 7 +++-- pfio/CMakeLists.txt | 21 ++++++++++++++ pfio/NetCDF4_FileFormatter.F90 | 13 +++++++++ pfio/Variable.F90 | 21 +++++++++++++- 23 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/README create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml diff --git a/Apps/Regrid_Util.F90 b/Apps/Regrid_Util.F90 index 072ab7373b96..9906beed1586 100644 --- a/Apps/Regrid_Util.F90 +++ b/Apps/Regrid_Util.F90 @@ -26,6 +26,7 @@ module regrid_util_support_mod integer :: deflate, shave integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level logical :: use_weights contains procedure :: create_grid @@ -98,6 +99,7 @@ subroutine process_command_line(this,rc) this%deflate=0 this%quantize_algorithm=0 this%quantize_level=0 + this%zstandard_level=0 this%use_weights = .false. nargs = command_argument_count() do i=1,nargs @@ -161,6 +163,9 @@ subroutine process_command_line(this,rc) case('-quantize_level') call get_command_argument(i+1,astr) read(astr,*)this%quantize_level + case('-zstandard_level') + call get_command_argument(i+1,astr) + read(astr,*)this%zstandard_level case('-file_weights') this%use_weights = .true. case('--help') @@ -432,7 +437,7 @@ subroutine main() call ESMF_ClockSet(clock,currtime=time,_RC) if (.not. writer_created) then - call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,_RC) + call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,zstandard_level=support%zstandard_level,_RC) writer_created=.true. end if diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5c4fdae965..41c02e838ab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added zstandard compression support + - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this + and enabling zstandard output in History will fail if the library does not support it + ### Changed ### Fixed diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index afcb2b5be93f..203d1f2e728a 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -29,6 +29,11 @@ set(QUANTIZE_TESTS "case34" ) +# We have one test that requires netcdf zstandard support +set(ZSTD_TESTS + "case35" +) + foreach(TEST_CASE ${TEST_CASES_1G}) if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) @@ -64,6 +69,11 @@ foreach(TEST_CASE ${TEST_CASES_2G}) continue() endif() + # Skip tests that require ZSTD support if we don't have it + if (NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST ZSTD_TESTS) + continue() + endif() + if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc new file mode 100644 index 000000000000..632008de4f12 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc @@ -0,0 +1,14 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/README b/Tests/ExtData_Testing_Framework/test_cases/case35/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt index 76be9d4d54f8..241249cdf680 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt @@ -32,3 +32,4 @@ case31 case32 case33 case34 +case35 diff --git a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md index f3700c12e5ee..c06106734fd6 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md +++ b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md @@ -39,3 +39,4 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin -- 32. Case1 with deflate compression and NetCDF bitgroom quantization (only enabled if netcdf built with quantization support) 33. Case1 with deflate compression and NetCDF bitround quantization (only enabled if netcdf built with quantization support) 34. Case1 with deflate compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization support) +35. Case1 with zstandard compression (only enabled if netcdf built with zstandard support) diff --git a/gridcomps/History/MAPL_HistoryCollection.F90 b/gridcomps/History/MAPL_HistoryCollection.F90 index 39f3c0b723d1..336773e0400d 100644 --- a/gridcomps/History/MAPL_HistoryCollection.F90 +++ b/gridcomps/History/MAPL_HistoryCollection.F90 @@ -83,6 +83,7 @@ module MAPL_HistoryCollectionMod character(len=ESMF_MAXSTR) :: quantize_algorithm_string integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level integer :: slices integer :: Root integer :: Psize diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 7aa0afff2a1c..b589af60b188 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -836,6 +836,18 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%deflate, default=0, & label=trim(string) // 'deflate:' ,_RC ) + ! We only allow deflate level to be between 0 and 9 + _ASSERT( .not. (list(n)%deflate < 0 .or. list(n)%deflate > 9), 'deflate level must be between 0 and 9') + + call ESMF_ConfigGetAttribute ( cfg, list(n)%zstandard_level, default=0, & + label=trim(string) // 'zstandard_level:' ,_RC ) + + ! We only allow zstandard level to be between 0 and 22 + _ASSERT( .not. (list(n)%zstandard_level < 0 .or. list(n)%zstandard_level > 22), 'zstandard level must be between 0 and 22') + + ! We only allow either deflate or zstandard compression to be used, not both + _ASSERT( .not. (list(n)%deflate > 0 .and. list(n)%zstandard_level > 0), 'deflate and zstandard_level cannot be used together') + call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_algorithm_string, default='NONE', & label=trim(string) // 'quantize_algorithm:' ,_RC ) diff --git a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 index 0fb038bff536..cf051b2b66a8 100644 --- a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 +++ b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 @@ -90,6 +90,7 @@ module MAPL_EpochSwathMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = 1 integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -536,11 +537,12 @@ subroutine Create_bundle_RH(this,items,bundle,tunit,timeInfo,vdata,ogrid,rc) end subroutine Create_Bundle_RH - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) class (sampler), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -555,6 +557,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,_STAT) end if diff --git a/griddedio/FieldBundleWrite.F90 b/griddedio/FieldBundleWrite.F90 index 1fb4e134e304..36e7d57f9b59 100644 --- a/griddedio/FieldBundleWrite.F90 +++ b/griddedio/FieldBundleWrite.F90 @@ -29,7 +29,7 @@ module MAPL_ESMFFieldBundleWrite contains - subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock character(len=*), intent(in) :: output_file @@ -37,20 +37,21 @@ subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,defla integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer :: status type(FieldBundleWriter) :: newWriter - call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,rc=status) + call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level,rc=status) _VERIFY(status) call newWriter%write_to_file(rc=status) _VERIFY(status) _RETURN(_SUCCESS) end subroutine write_bundle_single_time - subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) class(FieldBundleWRiter), intent(inout) :: this type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock @@ -62,6 +63,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc type(TimeData) :: time_info @@ -85,7 +87,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step time_interval_=0 end if - call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level) + call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level) time_info = TimeData(clock,file_steps,time_interval_,offset) call ESMF_FieldBundleGet(bundle, fieldCount=num_fields,rc=status) _VERIFY(status) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 7eb3cf53c7f7..06ec0754460e 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -54,6 +54,7 @@ module MAPL_GriddedIOMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = MAPL_NOQUANTIZE integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -250,11 +251,12 @@ subroutine destroy(this, rc) end subroutine destroy - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) class (MAPL_GriddedIO), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -270,6 +272,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,stat=status) _VERIFY(status) @@ -409,7 +412,7 @@ subroutine CreateVariable(this,itemName,rc) _FAIL( 'Unsupported field rank') end if end if - v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel) + v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel,zstandard_level=this%zstandardLevel) call v%add_attribute('units',trim(units)) call v%add_attribute('long_name',trim(longName)) call v%add_attribute('standard_name',trim(longName)) diff --git a/pfio/CMakeLists.txt b/pfio/CMakeLists.txt index 3c64d0598926..3c6b826eb4f3 100644 --- a/pfio/CMakeLists.txt +++ b/pfio/CMakeLists.txt @@ -114,6 +114,27 @@ else () message(STATUS "netCDF does not have quantize capability") endif () +################################################################ +# Check to see if zstandard capability is present in netcdf-c. # +################################################################ + +check_c_source_compiles(" + #include + #if !NC_HAS_ZSTD + choke me + #endif + int main(void) { + return 0; + } + " + NETCDF_HAS_ZSTD) +if (NETCDF_HAS_ZSTD) + message(STATUS "netCDF has zstandard capability") + add_definitions(-DNF_HAS_ZSTD) +else () + message(STATUS "netCDF does not have zstandard capability") +endif () + if (BUILD_WITH_PFLOGGER) find_package (PFLOGGER REQUIRED) endif () diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index 7ffc315bbbba..1527d8eb5524 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -709,6 +709,7 @@ subroutine def_variables(this, cf, unusable, varname, rc) integer :: deflation integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level character(len=:), pointer :: var_name character(len=:), pointer :: dim_name class (Variable), pointer :: var @@ -783,6 +784,18 @@ subroutine def_variables(this, cf, unusable, varname, rc) #endif end if + zstandard_level = var%get_zstandard_level() + if (zstandard_level /= 0) then +#ifdef NF_HAS_ZSTD + !$omp critical + status = nf90_def_var_zstandard(this%ncid, varid, zstandard_level) + !$omp end critical + _VERIFY(status) +#else + _FAIL("netcdf was not built with zstandard support") +#endif + end if + call this%put_var_attributes(var, varid, rc=status) _VERIFY(status) diff --git a/pfio/Variable.F90 b/pfio/Variable.F90 index 624954bfd162..84958a172945 100644 --- a/pfio/Variable.F90 +++ b/pfio/Variable.F90 @@ -30,6 +30,7 @@ module pFIO_VariableMod integer :: deflation = 0 ! default no compression integer :: quantize_algorithm = 0 ! default no quantization integer :: quantize_level = 0 ! default no quantize_level + integer :: zstandard_level = 0 ! default no zstandard integer, allocatable :: chunksizes(:) contains procedure :: get_type @@ -51,6 +52,7 @@ module pFIO_VariableMod procedure :: set_deflation procedure :: get_quantize_algorithm procedure :: get_quantize_level + procedure :: get_zstandard_level procedure :: is_attribute_present generic :: operator(==) => equal generic :: operator(/=) => not_equal @@ -69,7 +71,7 @@ module pFIO_VariableMod contains - function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, rc) result(var) + function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, zstandard_level, rc) result(var) type (Variable) :: var integer, optional, intent(in) :: type class (KeywordEnforcer), optional, intent(in) :: unusable @@ -79,6 +81,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer:: empty(0) @@ -87,6 +90,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%deflation = 0 var%quantize_algorithm = 0 var%quantize_level = 0 + var%zstandard_level = 0 var%chunksizes = empty var%dimensions = StringVector() var%attributes = StringAttributeMap() @@ -120,6 +124,10 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%quantize_level = quantize_level endif + if (present(zstandard_level)) then + var%zstandard_level = zstandard_level + endif + _RETURN(_SUCCESS) _UNUSED_DUMMY(unusable) contains @@ -313,6 +321,13 @@ function get_quantize_level(this) result(quantizeLevel) quantizeLevel=this%quantize_level end function get_quantize_level + function get_zstandard_level(this) result(zstandardLevel) + class (Variable), target, intent(In) :: this + integer :: zstandardLevel + + zstandardLevel=this%zstandard_level + end function get_zstandard_level + logical function equal(a, b) class (Variable), target, intent(in) :: a type (Variable), target, intent(in) :: b @@ -388,6 +403,7 @@ subroutine serialize(this, buffer, rc) buffer = [buffer, serialize_intrinsic(this%deflation)] buffer = [buffer, serialize_intrinsic(this%quantize_algorithm)] buffer = [buffer, serialize_intrinsic(this%quantize_level)] + buffer = [buffer, serialize_intrinsic(this%zstandard_level)] if( .not. allocated(this%chunksizes)) then buffer =[buffer,[1]] @@ -453,6 +469,9 @@ subroutine deserialize(this, buffer, rc) call deserialize_intrinsic(buffer(n:),this%quantize_level) length = serialize_buffer_length(this%quantize_level) n = n + length + call deserialize_intrinsic(buffer(n:),this%zstandard_level) + length = serialize_buffer_length(this%zstandard_level) + n = n + length call deserialize_intrinsic(buffer(n:),this%chunksizes) _RETURN(_SUCCESS) end subroutine deserialize From fd1e4e0ef0d637bad2e1ed6be129c399d1e72845 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 25 Sep 2024 08:42:17 -0400 Subject: [PATCH 02/10] Add test of both quantize and zstd --- .../ExtData_Testing_Framework/CMakeLists.txt | 10 +++++++ .../test_cases/case36/AGCM1.rc | 24 +++++++++++++++ .../test_cases/case36/AGCM2.rc | 29 +++++++++++++++++++ .../test_cases/case36/CAP.rc | 4 +++ .../test_cases/case36/CAP1.rc | 25 ++++++++++++++++ .../test_cases/case36/CAP2.rc | 15 ++++++++++ .../test_cases/case36/ExtData.rc | 13 +++++++++ .../test_cases/case36/HISTORY1.rc | 16 ++++++++++ .../test_cases/case36/HISTORY2.rc | 5 ++++ .../test_cases/case36/README | 1 + .../test_cases/case36/extdata.yaml | 5 ++++ .../test_cases/extdata_2g_cases.txt | 1 + .../test_cases/test_case_descriptions.md | 2 ++ 13 files changed, 150 insertions(+) create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/README create mode 100644 Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index 203d1f2e728a..ab0c0017c483 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -34,6 +34,11 @@ set(ZSTD_TESTS "case35" ) +# We have one test which requires *both* Quantize and ZSTD support +set(QUANTIZE_AND_ZSTD_TESTS + "case36" +) + foreach(TEST_CASE ${TEST_CASES_1G}) if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) @@ -74,6 +79,11 @@ foreach(TEST_CASE ${TEST_CASES_2G}) continue() endif() + # Skip tests that require both Quantize and ZSTD support if we don't have it + if (NOT NETCDF_HAS_QUANTIZE AND NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS) + continue() + endif() + if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc new file mode 100644 index 000000000000..014b2e83e177 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc @@ -0,0 +1,16 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.quantization_algorithm: 'granular_bitround', + case1.quantization_level: 5, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/README b/Tests/ExtData_Testing_Framework/test_cases/case36/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt index 241249cdf680..b79e4a5c9d29 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt @@ -33,3 +33,4 @@ case32 case33 case34 case35 +case36 diff --git a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md index c06106734fd6..65802345b833 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md +++ b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md @@ -40,3 +40,5 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin -- 33. Case1 with deflate compression and NetCDF bitround quantization (only enabled if netcdf built with quantization support) 34. Case1 with deflate compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization support) 35. Case1 with zstandard compression (only enabled if netcdf built with zstandard support) +36. Case1 with zstandard compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization + support and zstandard support) From 38761bdb86e1b5b7bf7944a0d9b2438811fd5356 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 25 Sep 2024 11:00:28 -0400 Subject: [PATCH 03/10] add missing set_param --- gridcomps/History/MAPL_HistoryGridComp.F90 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index b589af60b188..adc88d215dbb 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -2417,6 +2417,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%xsampler%set_param(deflation=list(n)%deflate,_RC) call list(n)%xsampler%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%xsampler%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%xsampler%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%xsampler%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%xsampler%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%xsampler%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2427,6 +2428,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%mGriddedIO%set_param(deflation=list(n)%deflate,_RC) call list(n)%mGriddedIO%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%mGriddedIO%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%mGriddedIO%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%mGriddedIO%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%mGriddedIO%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%mGriddedIO%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2505,6 +2507,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'Quantize Alg: ', trim(list(n)%quantize_algorithm_string) print *, 'Quantize Lvl: ', list(n)%quantize_level end if + if (list(n)%zstandard_level > 0) then + print *, 'Zstandard Lvl: ', list(n)%zstandard_level + end if if (associated(list(n)%chunksize)) then print *, ' ChunkSize: ', list(n)%chunksize end if From e957e2f507c09deed1bed5564eb96b50d83a8ce2 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 25 Sep 2024 11:28:36 -0400 Subject: [PATCH 04/10] Fix test logic --- Tests/ExtData_Testing_Framework/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index ab0c0017c483..7edc3a303d41 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -80,7 +80,7 @@ foreach(TEST_CASE ${TEST_CASES_2G}) endif() # Skip tests that require both Quantize and ZSTD support if we don't have it - if (NOT NETCDF_HAS_QUANTIZE AND NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS) + if (NOT (NETCDF_HAS_QUANTIZE AND NETCDF_HAS_ZSTD) AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS) continue() endif() From dd28a17cdaca2b4fc0941e2d59e5ebc414313927 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Mon, 30 Sep 2024 12:01:51 -0400 Subject: [PATCH 05/10] candidate for v2.46.4 remove assert that there is no mask --- base/MAPL_EsmfRegridder.F90 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/base/MAPL_EsmfRegridder.F90 b/base/MAPL_EsmfRegridder.F90 index 91df059e6623..97587cf6421d 100644 --- a/base/MAPL_EsmfRegridder.F90 +++ b/base/MAPL_EsmfRegridder.F90 @@ -1530,9 +1530,9 @@ subroutine create_route_handle(this, kind, rc) _VERIFY(status) case (REGRID_METHOD_PATCH) - _ASSERT(.not.has_mask, "destination masking with this regrid type is unsupported") call ESMF_FieldRegridStore(src_field, dst_field, & & regridmethod=ESMF_REGRIDMETHOD_PATCH, & + & dstMaskValues = dstMaskValues, & & linetype=ESMF_LINETYPE_GREAT_CIRCLE, & ! closer to SJ Lin interpolation weights? & srcTermProcessing = srcTermProcessing, & & factorList=factorList, factorIndexList=factorIndexList, & @@ -1540,26 +1540,28 @@ subroutine create_route_handle(this, kind, rc) _VERIFY(status) case (REGRID_METHOD_CONSERVE_2ND) - _ASSERT(.not.has_mask, "destination masking with this regrid type is unsupported") call ESMF_FieldRegridStore(src_field, dst_field, & & regridmethod=ESMF_REGRIDMETHOD_CONSERVE_2ND, & + & dstMaskValues = dstMaskValues, & & linetype=ESMF_LINETYPE_GREAT_CIRCLE, & ! closer to SJ Lin interpolation weights? & srcTermProcessing = srcTermProcessing, & & factorList=factorList, factorIndexList=factorIndexList, & & routehandle=route_handle, unmappedaction=unmappedaction, rc=status) _VERIFY(status) case (REGRID_METHOD_CONSERVE, REGRID_METHOD_CONSERVE_MONOTONIC, REGRID_METHOD_VOTE, REGRID_METHOD_FRACTION) - _ASSERT(.not.has_mask, "destination masking with this regrid type is unsupported") + call ESMF_FieldRegridStore(src_field, dst_field, & & regridmethod=ESMF_REGRIDMETHOD_CONSERVE, & + & dstMaskValues = dstMaskValues, & & srcTermProcessing = srcTermProcessing, & & factorList=factorList, factorIndexList=factorIndexList, & & routehandle=route_handle, unmappedaction=unmappedaction, rc=status) _VERIFY(status) case (REGRID_METHOD_NEAREST_STOD) - _ASSERT(.not.has_mask, "destination masking with this regrid type is unsupported") + call ESMF_FieldRegridStore(src_field, dst_field, & & regridmethod=ESMF_REGRIDMETHOD_NEAREST_STOD, & + & dstMaskValues = dstMaskValues, & & factorList=factorList, factorIndexList=factorIndexList, & & routehandle=route_handle, unmappedaction=unmappedaction, rc=status) _VERIFY(status) From 442ad21f952c266c62874290068bebcf10aaa88e Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 3 Oct 2024 12:00:20 -0400 Subject: [PATCH 06/10] Update to ecbuild geos/v1.4.0 --- CHANGELOG.md | 3 +++ components.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df53e03b29fb..fddc034bb4bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - ExtDataDriver.x now uses ExtData2G by default +- Update `components.yaml` + - ecbuild geos/v1.4.0 + - Fixes bug between GCC, macOS, and the `-pipe` flag ### Fixed diff --git a/components.yaml b/components.yaml index 1796c1de157f..4360ab32a56f 100644 --- a/components.yaml +++ b/components.yaml @@ -17,4 +17,4 @@ ESMA_cmake: ecbuild: local: ./ESMA_cmake/ecbuild remote: ../ecbuild.git - tag: geos/v1.3.0 + tag: geos/v1.4.0 From aa80a662d02423adf2678526624f190ffbe47caa Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 3 Oct 2024 15:58:58 -0400 Subject: [PATCH 07/10] Fixes for NVHPC 24.9 --- CHANGELOG.md | 2 ++ base/Base/Base_Base_implementation.F90 | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fddc034bb4bd..a23fbc0536a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Workaround for NVHPC 24.9 involving `use` statement in `block` construct + ### Removed ### Deprecated diff --git a/base/Base/Base_Base_implementation.F90 b/base/Base/Base_Base_implementation.F90 index 20b27e36deea..a070bee5cba3 100644 --- a/base/Base/Base_Base_implementation.F90 +++ b/base/Base/Base_Base_implementation.F90 @@ -1954,6 +1954,7 @@ module function MAPL_LatLonGridCreate (Name, vm, & block use MAPL_Constants, only: MAPL_DEGREES_TO_RADIANS_R8 + use iso_fortran_env, only: REAL64 real(kind=REAL64), allocatable :: lons(:) real(kind=REAL64), allocatable :: lats(:) @@ -2599,7 +2600,7 @@ module subroutine MAPL_GetHorzIJIndex(npts,II,JJ,lon,lat,lonR8,latR8,Grid, rc) type(ESMF_CoordSys_Flag) :: coordSys character(len=ESMF_MAXSTR) :: grid_type - _RETURN_IF(npts == 0 ) + _RETURN_IF(npts == 0 ) ! if the grid is present then we can just get the prestored edges and the dimensions of the grid ! this also means we are running on a distributed grid ! if grid not present then the we just be running outside of ESMF and the user must From 5c246f4e7d633fde7ebd71e35bb0b45be3589b5d Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Thu, 3 Oct 2024 16:09:00 -0400 Subject: [PATCH 08/10] Update changelog and cmake for 2.49 --- CHANGELOG.md | 14 ++++++++++---- CMakeLists.txt | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a23fbc0536a6..f308d711d89d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +### Fixed + +### Removed + +### Deprecated + +## [2.49.0] - 2024-10-04 + +### Changed + - ExtDataDriver.x now uses ExtData2G by default - Update `components.yaml` - ecbuild geos/v1.4.0 @@ -20,10 +30,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Workaround for NVHPC 24.9 involving `use` statement in `block` construct -### Removed - -### Deprecated - ## [2.48.0] - 2024-09-24 ### Added diff --git a/CMakeLists.txt b/CMakeLists.txt index 212ffb6e0d23..0f06772e49dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ endif () project ( MAPL - VERSION 2.48.0 + VERSION 2.49.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF # Set the possible values of build type for cmake-gui From 5a670389768d0876646e5684d1c6c2cb07d06ae9 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 3 Oct 2024 16:39:42 -0400 Subject: [PATCH 09/10] Update CHANGELOG.md --- CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad19717987d..6e5c62c38ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added zstandard compression support - - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this - and enabling zstandard output in History will fail if the library does not support it - ### Changed ### Fixed @@ -23,6 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.49.0] - 2024-10-04 +### Added + +- Added zstandard compression support + - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this + and enabling zstandard output in History will fail if the library does not support it + ### Changed - ExtDataDriver.x now uses ExtData2G by default From 3295ceebe7e870aaca2018c12ea1ac82d0628fa5 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Mon, 7 Oct 2024 12:58:06 -0400 Subject: [PATCH 10/10] Update CMakelists and changelog for 2.46.4 release --- CHANGELOG.md | 6 ++++++ CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfea0235e181..c432373df80f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +## [2.46.4] - 2024-10-07 + +### Fixed + +- Removed erroneous asserts that blocked some use cases in creating route handles + ## [2.46.3] - 2024-08-16 ### Fixed diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a0ecb3b109a..1f6fc9356cec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ endif () project ( MAPL - VERSION 2.46.3 + VERSION 2.46.4 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF # Set the possible values of build type for cmake-gui