From af6281f7f872b5fff87b37876bd8fc578ac7db8a Mon Sep 17 00:00:00 2001 From: Trevor Ray Hillebrand Date: Fri, 21 May 2021 08:40:20 -0600 Subject: [PATCH 001/301] Fix floating point exception in effectiveViscosity calculation If meanFlowParamA == 0.0, but eEff != 0.0, 0.0 was raised to a negative power, causing a floating point exception. --- .../mpas-albany-landice/src/mode_forward/mpas_li_velocity.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_velocity.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_velocity.F index 5bf5e0b03f55..79df9363fcd0 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_velocity.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_velocity.F @@ -980,7 +980,7 @@ subroutine calculate_strain_rates_and_stresses(meshPool, geometryPool, thermalPo ! calculate effective viscosity from strain rate and flow param. do iCell = 1, nCells eEff = sqrt(exx(iCell)**2 + eyy(iCell)**2 + exx(iCell)*eyy(iCell) + exy(iCell)**2) ! effective strain rate - if (eEff == 0.0_RKIND) then + if ( (eEff == 0.0_RKIND) .or. (meanFlowParamA(iCell) == 0.0_RKIND) ) then effectiveViscosity(iCell) = 0.0_RKIND else effectiveViscosity(iCell) = & From 7d0fb11775aa84cb0ce558b9b19c6e5b7a353cf5 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 1 Jun 2021 17:07:45 -0400 Subject: [PATCH 002/301] add F20TR-MMFXX-CMIP6 compset new file: components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml modified: components/eam/cime_config/config_component.xml modified: components/eam/cime_config/config_compsets.xml --- .../use_cases/20TR_MMF-1mom_CMIP6.xml | 98 +++++++++++++++++++ .../eam/cime_config/config_component.xml | 9 +- .../eam/cime_config/config_compsets.xml | 8 ++ 3 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml diff --git a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml new file mode 100644 index 000000000000..681cb2d7d86f --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml @@ -0,0 +1,98 @@ + + + + +.true. + + + + + + + +.false. + + + + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + +atm/cam/ggas/GHG_CMIP-1-2-0_Annual_Global_0000-2014_c20180105.nc +RAMPED + + +atm/cam/volc +CMIP_DOE-ACME_radiation_1850-2014_v3_c20171205.nc +VOLC_CMIP6 +SERIAL + + +.true. +.true. +.true. + + +atm/cam/chem/trop_mam/aero +mam4_0.9x1.2_L72_2000clim_c170323.nc +.false. +CYCLICAL +01 + + + + + + + + + --> + + + + + + + + + + + + --> + + + + + + + + + + + + + + + + + --> + + + + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +SERIAL + + + + + +'H2O2', 'H2SO4', 'SO2' + + +1850-2000 + + diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 920597d3bffc..8e4d85b2cb78 100644 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -79,8 +79,8 @@ -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -use_ECPP -crm_nx 64 -crm_nx_rad 16 -crm_ny 1 -crm_ny_rad 1 - -aquaplanet - -aquaplanet -rce + -aquaplanet + -aquaplanet -rce -offline_dyn @@ -164,8 +164,9 @@ 2000_cam5_av1c-04p2-MMF-2mom 2000_cam5_av1c-04p2-MMF-1mom 2000_cam5_av1c-04p2-MMF-2mom - RCEMIP_EAMv1 - aquaplanet_MMF-1mom + RCEMIP_EAMv1 + aquaplanet_MMF-1mom + 20TR_MMF-1mom_CMIP6 run_component_cam diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index c354e55d9235..581b3fce21b1 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -272,6 +272,14 @@ 2000_EAM%MMF2-ECPP_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV + + + + F20TR-MMFXX-CMIP6 + + 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + From 75850d1c5395589bda42445b9565c8b59e16879d Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 1 Jun 2021 18:53:24 -0400 Subject: [PATCH 003/301] Fix SSTICE file for MMF-AMIP compset --- components/data_comps/docn/cime_config/config_component.xml | 4 ++++ components/eam/cime_config/config_compsets.xml | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index eb2645d57274..4167fde02493 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -188,6 +188,7 @@ $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_2010_clim_c20190821.nc $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc + $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc @@ -246,6 +247,7 @@ 1850 1869 1869 + 1869 run_component_cam_sstice env_run.xml @@ -271,6 +273,7 @@ 1850 1869 1869 + 1869 run_component_cam_sstice env_run.xml @@ -290,6 +293,7 @@ 2012 2016 2016 + 2016 run_component_cam_sstice env_run.xml diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 581b3fce21b1..a5dd7629b0e9 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -276,7 +276,6 @@ F20TR-MMFXX-CMIP6 - 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV From 1b445b3429d85a73aaca998f129ecf11d19506e1 Mon Sep 17 00:00:00 2001 From: Mauro Perego Date: Tue, 8 Jun 2021 11:30:25 -0600 Subject: [PATCH 004/301] MALI: adding muFriction to restart stream of the Registry --- components/mpas-albany-landice/src/Registry.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index b60bb7981414..2f7d76da2e81 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -750,9 +750,10 @@ - + + From bae4bb60003d0045ce53918c4448d353235e81cc Mon Sep 17 00:00:00 2001 From: Mauro Perego Date: Tue, 8 Jun 2021 11:34:47 -0600 Subject: [PATCH 005/301] MALI: avoid modyfing the effective pressure in the velocity interface Previously the effective pressure was zeroed out on floating ice. This should be done either in a pre-processing step or on the Albany side --- .../src/mode_forward/Interface_velocity_solver.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/Interface_velocity_solver.cpp b/components/mpas-albany-landice/src/mode_forward/Interface_velocity_solver.cpp index 3971076ba939..55033d119328 100644 --- a/components/mpas-albany-landice/src/mode_forward/Interface_velocity_solver.cpp +++ b/components/mpas-albany-landice/src/mode_forward/Interface_velocity_solver.cpp @@ -1251,9 +1251,6 @@ void importFields(std::vector >& marineBdyExtensionMap, dou elevationData[iv] = elev; } - bool floating = rho_ice * thicknessData[iv] + rho_ocean * bedTopographyData[iv] < 0; - if (floating && (effecPress_F != 0)) - effecPressData[iv] = 0.0; } } From 6c8f41c17ec43fdb543605a5c578dd7616b4c92f Mon Sep 17 00:00:00 2001 From: Trevor Ray Hillebrand Date: Tue, 30 Mar 2021 08:53:42 -0600 Subject: [PATCH 006/301] Seperate flood-fill into its own subroutine This initial commit compiles, but has not been tested. --- mode_forward/mpas_li_calving.F | 190 ++++++++++++++++++++++++--------- 1 file changed, 141 insertions(+), 49 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 51ade4220d70..13ae1b1cc666 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -3272,14 +3272,14 @@ subroutine remove_icebergs(domain) call mpas_timer_start("iceberg detection") call mpas_log_write("Iceberg-detection flood-fill begin.") - ! Allocate needed scratch fields + ! Allocate needed scratch fields block => domain % blocklist call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) + call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) + call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) call mpas_log_write("Iceberg-detection flood-fill: allocated.") @@ -3296,7 +3296,141 @@ subroutine remove_icebergs(domain) call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) - ! make sure masks are up to date. May not be necessary, but safer to do anyway. + ! make sure masks are up to date. May not be necessary, but safer to + ! do anyway. + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + + call mpas_log_write("Iceberg-detection flood-fill: updated masks.") + contiguousDynamicIce(:) = 0 ! initialize + newMaskCountLocal = 0 + do iCell = 1, nCellsSolve + if (li_mask_is_grounded_ice(cellMask(iCell)) .and. li_mask_is_dynamic_ice(cellMask(iCell))) then + contiguousDynamicIce(iCell) = 1 + newMaskCountLocal = newMaskCountLocal + 1 + endif + enddo + call mpas_log_write("Initialized $i cells to local mask", intArgs=(/newMaskCountLocal/)) + + block => block % next + end do + + call mpas_log_write("Iceberg-detection flood-fill initialization complete.") + + call flood_fill(domain) + + ! Now remove any ice that was not flood-filled - these are icebergs + block => domain % blocklist + do while (associated(block)) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'calvingThickness', calvingThickness) + call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) + call mpas_pool_get_array(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOld) + call mpas_pool_get_dimension(geometryPool, 'nCells', nCells) + call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) + + contiguousDynamicIceOld(:) = 0 + do iCell = 1, nCellsSolve + if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_dynamic_ice(cellMask(iCell))) then + calvingThickness(iCell) = calvingThickness(iCell) + thickness(iCell) ! remove any remaining ice here + thickness(iCell) = 0.0_RKIND + contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed + endif + enddo + localIcebergCellCount = sum(contiguousDynamicIceOld) + +! where(contiguousDynamicIce == 0 .and. li_mask_is_dynamic_ice(cellMask)) +! calvingThickness = calvingThickness + thickness ! remove any remaining ice here +! thickness = 0.0_RKIND +! end where + + block => block % next + end do + + ! Calculate total iceberg cells removed (not needed but useful for log - note requires extra reduce!) + call mpas_dmpar_sum_int(domain % dminfo, localIcebergCellCount, globalIcebergCellCount) + + ! update halo - probably not needed? + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'thickness') + call mpas_timer_stop("halo updates") + + ! clean up + call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) + call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) + + call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) + call mpas_timer_stop("iceberg detection") + end subroutine remove_icebergs + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! routine flood_fill +! +!> \brief Flood fill routine to differentiate ice sheet from icebergs +!> \author Trevor Hillebrand and Matthew Hoffman +!> \date March 2021 +!> \details +!----------------------------------------------------------------------- + subroutine flood_fill(domain) + !----------------------------------------------------------------- + ! input/output variables + !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object + + ! Local variables + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: meshPool + type (mpas_pool_type), pointer :: geometryPool + type (mpas_pool_type), pointer :: velocityPool + + type (field1dInteger), pointer :: contiguousDynamicIceField + type (field1dInteger), pointer :: contiguousDynamicIceOldField + + real (kind=RKIND), dimension(:), pointer :: calvingThickness !thickness of ice that calves (computed in this subroutine) + real (kind=RKIND), dimension(:), pointer :: thickness + integer, dimension(:), pointer :: cellMask + integer, dimension(:,:), pointer :: cellsOnCell ! list of cells that neighbor each cell + integer, dimension(:), pointer :: nEdgesOnCell ! number of cells that border each cell + + integer, dimension(:), pointer :: contiguousDynamicIce, contiguousDynamicIceOld + integer, pointer :: nCells, nCellsSolve + integer, dimension(:), pointer :: nCellsArray + integer :: iCell, jCell, n + integer :: newMaskCountLocal, newMaskCountLocalAccum, newMaskCountGlobal + integer :: err_tmp, err + integer :: globalLoopCount, localLoopCount + integer :: localIcebergCellCount, globalIcebergCellCount + + err = 0 + + + block => domain % blocklist + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) + call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) + + call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) + call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) + + call mpas_log_write("Flood-fill: allocated.") + ! First mark grounded ice to initialize flood fill mask + block => domain % blocklist + do while (associated(block)) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + + ! make sure masks are up to date. May not be necessary, but safer to + ! do anyway. call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) err = ior(err, err_tmp) @@ -3376,7 +3510,8 @@ subroutine remove_icebergs(domain) endif ! if not already marked enddo ! loop over cells - ! Accumulate cells added locally until we do the next global reduce + ! Accumulate cells added locally until we do the next global + ! reduce newMaskCountLocalAccum = newMaskCountLocalAccum + newMaskCountLocal call mpas_log_write(" Added $i new cells to local mask", intArgs=(/newMaskCountLocal/)) enddo ! local mask loop @@ -3393,53 +3528,10 @@ subroutine remove_icebergs(domain) endif end do ! global loop - - ! Now remove any ice that was not flood-filled - these are icebergs - block => domain % blocklist - do while (associated(block)) - call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) - call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) - call mpas_pool_get_array(geometryPool, 'thickness', thickness) - call mpas_pool_get_array(geometryPool, 'calvingThickness', calvingThickness) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOld) - call mpas_pool_get_dimension(geometryPool, 'nCells', nCells) - call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) - - contiguousDynamicIceOld(:) = 0 - do iCell = 1, nCellsSolve - if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_dynamic_ice(cellMask(iCell))) then - calvingThickness(iCell) = calvingThickness(iCell) + thickness(iCell) ! remove any remaining ice here - thickness(iCell) = 0.0_RKIND - contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed - endif - enddo - localIcebergCellCount = sum(contiguousDynamicIceOld) - -! where(contiguousDynamicIce == 0 .and. li_mask_is_dynamic_ice(cellMask)) -! calvingThickness = calvingThickness + thickness ! remove any remaining ice here -! thickness = 0.0_RKIND -! end where - - block => block % next - end do - - ! Calculate total iceberg cells removed (not needed but useful for log - note requires extra reduce!) - call mpas_dmpar_sum_int(domain % dminfo, localIcebergCellCount, globalIcebergCellCount) - - ! update halo - probably not needed? - call mpas_timer_start("halo updates") - call mpas_dmpar_field_halo_exch(domain, 'thickness') - call mpas_timer_stop("halo updates") - - ! clean up call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) - call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) - call mpas_timer_stop("iceberg detection") - end subroutine remove_icebergs - + end subroutine flood_fill end module li_calving From a9abcc0d34937fb66b6efad499100b7e91e9f414 Mon Sep 17 00:00:00 2001 From: Trevor Ray Hillebrand Date: Tue, 30 Mar 2021 11:53:45 -0600 Subject: [PATCH 007/301] Define seed/grow masks in calling routine and pass to flood-fill This enables each calling routine to define its own seed and grow mask criteria. The seedMask is then filled in the flood-fill routine and passed back to the calling routine. --- mode_forward/mpas_li_calving.F | 50 ++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 13ae1b1cc666..aea164ce9ca4 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -3247,6 +3247,7 @@ subroutine remove_icebergs(domain) type (field1dInteger), pointer :: contiguousDynamicIceField type (field1dInteger), pointer :: contiguousDynamicIceOldField + integer, dimension(:), allocatable :: seedMask, growMask !masks to pass to flood-fill routine real (kind=RKIND), dimension(:), pointer :: calvingThickness ! thickness of ice that calves (computed in this subroutine) real (kind=RKIND), dimension(:), pointer :: thickness @@ -3265,6 +3266,7 @@ subroutine remove_icebergs(domain) err = 0 + call mpas_pool_get_config(liConfigs, 'config_remove_icebergs', config_remove_icebergs) if (.not. config_remove_icebergs) then return ! skip this entire routine if disabled @@ -3276,11 +3278,17 @@ subroutine remove_icebergs(domain) block => domain % blocklist call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + allocate(seedMask(nCells+1)) + allocate(growMask(nCells+1)) + seedMask(:) = 0 + growMask(:) = 0 call mpas_log_write("Iceberg-detection flood-fill: allocated.") ! First mark grounded ice to initialize flood fill mask @@ -3316,9 +3324,14 @@ subroutine remove_icebergs(domain) end do call mpas_log_write("Iceberg-detection flood-fill initialization complete.") - - call flood_fill(domain) - + + seedMask(:) = contiguousDynamicIce(:) + where ( seedMask == 0 .and. li_mask_is_dynamic_ice(cellMask(:)) ) + growMask = 1 + endwhere + call flood_fill(seedMask, growMask, domain) + + contiguousDynamicIce(:) = seedMask(:) ! Now remove any ice that was not flood-filled - these are icebergs block => domain % blocklist do while (associated(block)) @@ -3360,7 +3373,8 @@ subroutine remove_icebergs(domain) ! clean up call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) - + deallocate(seedMask) + deallocate(growMask) call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) call mpas_timer_stop("iceberg detection") end subroutine remove_icebergs @@ -3374,12 +3388,13 @@ end subroutine remove_icebergs !> \date March 2021 !> \details !----------------------------------------------------------------------- - subroutine flood_fill(domain) + subroutine flood_fill(seedMask, growMask, domain) !----------------------------------------------------------------- ! input/output variables !----------------------------------------------------------------- type (domain_type), intent(inout) :: domain !< Input/Output: domain object - + integer, dimension(:), intent(inout) :: seedMask + integer, dimension(:), intent(in) :: growMask ! Local variables type (block_type), pointer :: block type (mpas_pool_type), pointer :: meshPool @@ -3394,6 +3409,7 @@ subroutine flood_fill(domain) integer, dimension(:), pointer :: cellMask integer, dimension(:,:), pointer :: cellsOnCell ! list of cells that neighbor each cell integer, dimension(:), pointer :: nEdgesOnCell ! number of cells that border each cell + integer, dimension(:), allocatable :: seedMaskOld integer, dimension(:), pointer :: contiguousDynamicIce, contiguousDynamicIceOld integer, pointer :: nCells, nCellsSolve @@ -3406,9 +3422,12 @@ subroutine flood_fill(domain) err = 0 - block => domain % blocklist call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + allocate(seedMaskOld(nCells+1)) + seedMaskOld(:) = 0 call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) @@ -3416,6 +3435,8 @@ subroutine flood_fill(domain) call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) call mpas_log_write("Flood-fill: allocated.") + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + ! First mark grounded ice to initialize flood fill mask block => domain % blocklist do while (associated(block)) @@ -3438,7 +3459,7 @@ subroutine flood_fill(domain) contiguousDynamicIce(:) = 0 ! initialize newMaskCountLocal = 0 do iCell = 1, nCellsSolve - if (li_mask_is_grounded_ice(cellMask(iCell)) .and. li_mask_is_dynamic_ice(cellMask(iCell))) then + if ( seedMask(iCell) > 0 ) then contiguousDynamicIce(iCell) = 1 newMaskCountLocal = newMaskCountLocal + 1 endif @@ -3483,7 +3504,7 @@ subroutine flood_fill(domain) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - + !allocate(seedMaskOld(nCells+1)) ! initialize local loop localLoopCount = 0 newMaskCountLocal = 1 ! need to make sure we enter the loop @@ -3493,16 +3514,15 @@ subroutine flood_fill(domain) ! initialize newMaskCountLocal = 0 - contiguousDynamicIceOld(:) = contiguousDynamicIce(:) + seedMaskOld(:) = seedMask(:) do iCell = 1, nCellsArray(1) ! this gives owned cells only - if (contiguousDynamicIceOld(iCell) == 0 .and. & ! this cell not yet marked - li_mask_is_dynamic_ice(cellMask(iCell))) then ! and is dynamic + if ( growMask(iCell)>0 ) then ! If it has a marked neighbor, then add it to the mask do n = 1, nEdgesOnCell(iCell) jCell = cellsOnCell(n, iCell) - if (contiguousDynamicIceOld(jCell) == 1) then - contiguousDynamicIce(iCell) = 1 + if ( (seedMaskOld(jCell) == 1) .and. (seedMask(iCell) .ne. 1) ) then + seedMask(iCell) = 1 newMaskCountLocal = newMaskCountLocal + 1 exit ! skip the rest of this do-loop - no need to check additional neighbors endif @@ -3527,7 +3547,7 @@ subroutine flood_fill(domain) call mpas_log_write("Too many global loops!", MPAS_LOG_ERR) endif end do ! global loop - + deallocate(seedMaskOld) call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) From 2a2560eede37cd6df4804ff34e34c5544187f426 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Wed, 28 Apr 2021 15:30:40 -0700 Subject: [PATCH 008/301] Use MPAS scratch arrays instead of allocatables --- Registry.xml | 11 ++++++++++- mode_forward/mpas_li_calving.F | 26 +++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Registry.xml b/Registry.xml index b60bb7981414..5686bcc476f1 100644 --- a/Registry.xml +++ b/Registry.xml @@ -1521,7 +1521,16 @@ is the value of that variable from the *previous* time level! + /> + + + diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index aea164ce9ca4..08741928ccc6 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -3244,10 +3244,13 @@ subroutine remove_icebergs(domain) type (mpas_pool_type), pointer :: meshPool type (mpas_pool_type), pointer :: geometryPool type (mpas_pool_type), pointer :: velocityPool + type (mpas_pool_type), pointer :: scratchPool type (field1dInteger), pointer :: contiguousDynamicIceField type (field1dInteger), pointer :: contiguousDynamicIceOldField - integer, dimension(:), allocatable :: seedMask, growMask !masks to pass to flood-fill routine + type (field1dInteger), pointer :: seedMaskField + type (field1dInteger), pointer :: growMaskField + integer, dimension(:), pointer :: seedMask, growMask !masks to pass to flood-fill routine real (kind=RKIND), dimension(:), pointer :: calvingThickness ! thickness of ice that calves (computed in this subroutine) real (kind=RKIND), dimension(:), pointer :: thickness @@ -3279,14 +3282,23 @@ subroutine remove_icebergs(domain) block => domain % blocklist call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'scratch', scratchPool) + call mpas_pool_get_field(scratchPool, 'seedMask', seedMaskField) + call mpas_allocate_scratch_field(seedMaskField, single_block_in = .false.) + seedMask => seedMaskField % array + + call mpas_pool_get_field(scratchPool, 'growMask', growMaskField) + call mpas_allocate_scratch_field(growMaskField, single_block_in = .false.) + growMask => growMaskField % array + call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) - allocate(seedMask(nCells+1)) - allocate(growMask(nCells+1)) + !allocate(seedMask(nCells+1)) + !allocate(growMask(nCells+1)) seedMask(:) = 0 growMask(:) = 0 @@ -3373,8 +3385,10 @@ subroutine remove_icebergs(domain) ! clean up call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) - deallocate(seedMask) - deallocate(growMask) + call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.false.) + call mpas_deallocate_scratch_field(growMaskField, single_block_in=.false.) + !deallocate(seedMask) + !deallocate(growMask) call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) call mpas_timer_stop("iceberg detection") end subroutine remove_icebergs @@ -3486,6 +3500,8 @@ subroutine flood_fill(seedMask, growMask, domain) ! First Update halos call mpas_timer_start("halo updates") call mpas_dmpar_field_halo_exch(domain, 'contiguousDynamicIce') + call mpas_dmpar_field_halo_exch(domain, 'growMask') + call mpas_dmpar_field_halo_exch(domain, 'seedMask') call mpas_timer_stop("halo updates") ! initialize counter of cells locally updated during this outer loop From af86c180e425ac3f66bda5064a6be9e8f5c96416 Mon Sep 17 00:00:00 2001 From: Trevor Ray Hillebrand Date: Tue, 30 Mar 2021 13:06:18 -0600 Subject: [PATCH 009/301] Fix bug in defining grow mask in remove_icebergs Use floating ice instead of grounded ice to define grow mask for remove_icebergs. Previous treatment would have removed non-dynamic margin cells. --- mode_forward/mpas_li_calving.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 08741928ccc6..297dfe133b78 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -3338,7 +3338,7 @@ subroutine remove_icebergs(domain) call mpas_log_write("Iceberg-detection flood-fill initialization complete.") seedMask(:) = contiguousDynamicIce(:) - where ( seedMask == 0 .and. li_mask_is_dynamic_ice(cellMask(:)) ) + where ( seedMask == 0 .and. li_mask_is_floating_ice(cellMask(:)) ) growMask = 1 endwhere call flood_fill(seedMask, growMask, domain) @@ -3358,7 +3358,7 @@ subroutine remove_icebergs(domain) contiguousDynamicIceOld(:) = 0 do iCell = 1, nCellsSolve - if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_dynamic_ice(cellMask(iCell))) then + if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_floating_ice(cellMask(iCell))) then calvingThickness(iCell) = calvingThickness(iCell) + thickness(iCell) ! remove any remaining ice here thickness(iCell) = 0.0_RKIND contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed From 0f59d089a20acb2a5a5228ca05fe7b09cd23c08c Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 29 Apr 2021 13:20:06 -0700 Subject: [PATCH 010/301] Remove groundedMarineMarginMask calculation from apply_calving_damage_threshold The field groundedMarineMarginMask was being calculated better never used. --- mode_forward/mpas_li_calving.F | 41 ++-------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 297dfe133b78..37dd76b60fc5 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -2965,66 +2965,29 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, e ! local variables !----------------------------------------------------------------- real (kind=RKIND), dimension(:), pointer :: calvingThickness !< Output: the applied calving rate as a thickness - real (kind=RKIND), dimension(:), pointer :: thickness, damage, bedTopography - integer, dimension(:), pointer :: groundedMarineMarginMask + real (kind=RKIND), dimension(:), pointer :: thickness, damage real(kind=RKIND), pointer :: config_damage_calving_threshold integer, dimension(:), pointer :: nEdgesOnCell ! number of cells that border each cell integer, dimension(:,:), pointer :: cellsOnCell ! list of cells that neighbor each cell integer, dimension(:), pointer :: cellMask integer, pointer :: nCells - integer :: iCell, iNeighbor, jCell, iEdge - integer :: nEmptyNeighbors - real (kind=RKIND), pointer :: config_sea_level + integer :: iCell, iNeighbor, jCell err = 0 call mpas_pool_get_config(liConfigs, 'config_damage_calving_threshold', config_damage_calving_threshold) ! get fields - call mpas_pool_get_config(liConfigs, 'config_sea_level', config_sea_level) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'thickness', thickness) - call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) call mpas_pool_get_array(geometryPool, 'calvingThickness', calvingThickness) call mpas_pool_get_array(geometryPool, 'damage', damage) - call mpas_pool_get_array(geometryPool, 'groundedMarineMarginMask', groundedMarineMarginMask) call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) calvingThickness(:) = 0.0_RKIND - groundedMarineMarginMask(:) = 0 - do iCell = 1, nCells - ! Define marine marginal cells as those that are (1) at the ice - ! margin, (2) have at least one neighboring cell without ice, (3) contain - ! grounded ice, and (4) have bed topography below sea level. - ! OR is adjacent to an inactive floating margin cell - - ! Check if neighboring cells contain ice and have bed topo below sea level - nEmptyNeighbors = 0 - do iEdge = 1, nEdgesOnCell(iCell) - iNeighbor = cellsOnCell(iEdge, iCell) - if ( (((thickness(iNeighbor) == 0.0_RKIND) & - .and. bedTopography(iNeighbor) < config_sea_level)) & - .or. ((li_mask_is_floating_ice(cellMask(iNeighbor)) & - .and. li_mask_is_margin(cellMask(iNeighbor)) & - .and. (.not. li_mask_is_dynamic_ice(cellMask(iNeighbor)))))) then - nEmptyNeighbors = nEmptyNeighbors + 1 - endif - enddo - - if ( nEmptyNeighbors > 0 & - .and. li_mask_is_grounded_ice(cellMask(iCell)) & - .and. bedTopography(iCell) < config_sea_level & - .and. li_mask_is_dynamic_ice(cellMask(iCell)) ) then - - groundedMarineMarginMask(iCell) = 1 - else - groundedMarineMarginMask(iCell) = 0 - endif - enddo - ! The calving volume needs to be distributed in three ways: ! 1. We need to first remove any "thin" ice in front of this cell ! 2. Then we remove ice from this cell if damage > threshold From 6ea1e7958320fac25606cfd19b9d68d8eb05617a Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 29 Apr 2021 13:49:34 -0700 Subject: [PATCH 011/301] Add call to flood-fill routine in apply_calving_damage_threshold This routine now uses a seed mask that is defined by the dynamic margin where damage >= threshold. The grow mask is where damage > threshold. It then removes all ice in the flood-filled seed mask, and any neighboring non-dynamic floating cells. --- mode_forward/mpas_li_calving.F | 41 ++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 37dd76b60fc5..3a316c828ef6 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -2426,7 +2426,7 @@ subroutine damage_calving(domain, err) domain=domain, err=err_tmp) err = ior(err, err_tmp) elseif (trim(config_damage_calving_method) == 'threshold') then - call apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, err_tmp) + call apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, domain, err_tmp) err = ior(err, err_tmp) else call mpas_log_write("Unknown value for config_damage_calving_method was specified!", MPAS_LOG_ERR) @@ -2937,14 +2937,13 @@ end subroutine li_finalize_damage_after_advection ! routine apply_calving_damage_threshold ! !> \brief Calve any ice that is damaged beyond a specified threshold -!> \author Tong Zhang, Matt Hoffman -!> \date Nov. 2020, March 2021 +!> \author Tong Zhang, Matt Hoffman, Trevor Hillebrand +!> \date Nov. 2020, March 2021, April 2021 !> \details This routine specified floating ice to be calved wherever the damage !> value exceeds a specified threshold, assuming the ice is connected to the calving !> front. !----------------------------------------------------------------------- - subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, err) - + subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, domain, err) !----------------------------------------------------------------- ! input variables !----------------------------------------------------------------- @@ -2954,6 +2953,7 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, e !----------------------------------------------------------------- ! input/output variables !----------------------------------------------------------------- + type (domain_type), intent(inout) :: domain !< Input/Output: domain object type (mpas_pool_type), pointer, intent(inout) :: geometryPool !< Input: geometry pool !----------------------------------------------------------------- @@ -2964,6 +2964,9 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, e !----------------------------------------------------------------- ! local variables !----------------------------------------------------------------- + type (field1dInteger), pointer :: seedMaskField + type (field1dInteger), pointer :: growMaskField + integer, dimension(:), pointer :: seedMask, growMask !masks to pass to flood-fill routine real (kind=RKIND), dimension(:), pointer :: calvingThickness !< Output: the applied calving rate as a thickness real (kind=RKIND), dimension(:), pointer :: thickness, damage real(kind=RKIND), pointer :: config_damage_calving_threshold @@ -2985,17 +2988,34 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, e call mpas_pool_get_array(geometryPool, 'damage', damage) call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) + ! Allocate scratch fields for flood-fill + call mpas_pool_get_field(scratchPool, 'seedMask', seedMaskField) + call mpas_allocate_scratch_field(seedMaskField, single_block_in = .true.) + seedMask => seedMaskField % array + seedMask(:) = 0 + + call mpas_pool_get_field(scratchPool, 'growMask', growMaskField) + call mpas_allocate_scratch_field(growMaskField, single_block_in = .true.) + growMask => growMaskField % array + growMask(:) = 0 calvingThickness(:) = 0.0_RKIND + ! define seed and grow masks for flood fill. + where ( li_mask_is_dynamic_margin(cellMask) .and. (damage .ge. config_damage_calving_threshold) ) + seedMask = 1 + end where + + where ( seedMask == 0 .and. (damage .ge. config_damage_calving_threshold) ) + growMask = 1 + end where + + call flood_fill(seedMask, growMask, domain) ! The calving volume needs to be distributed in three ways: ! 1. We need to first remove any "thin" ice in front of this cell ! 2. Then we remove ice from this cell if damage > threshold do iCell = 1, nCells - if ( (li_mask_is_floating_ice(cellMask(iCell)) .and. li_mask_is_dynamic_ice(cellMask(iCell)) ) & ! floating dynamic - .and. li_mask_is_margin(cellMask(iCell)) & - .and. (damage(iCell) .ge. config_damage_calving_threshold)) then - + if ( seedMask(iCell) == 1 ) then ! First remove ice from "thin" neighbors do iNeighbor = 1, nEdgesOnCell(iCell) jCell = cellsOnCell(iNeighbor, iCell) @@ -3027,6 +3047,9 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, e endif ! if cell is calving margin enddo ! cell loop + ! deallocate scratch fields used for flood fill + call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.true.) + call mpas_deallocate_scratch_field(growMaskField, single_block_in=.true.) end subroutine apply_calving_damage_threshold From 16814f212e1b31c336d47246f4f232afe95f0249 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 29 Apr 2021 13:59:54 -0700 Subject: [PATCH 012/301] Add new comments and remove outdated ones --- mode_forward/mpas_li_calving.F | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index 3a316c828ef6..dbc572c33c37 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -3011,9 +3011,8 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, d end where call flood_fill(seedMask, growMask, domain) - ! The calving volume needs to be distributed in three ways: - ! 1. We need to first remove any "thin" ice in front of this cell - ! 2. Then we remove ice from this cell if damage > threshold + + ! Remove ice from flood-filled mask, and any neighboring non-dynamic ice do iCell = 1, nCells if ( seedMask(iCell) == 1 ) then ! First remove ice from "thin" neighbors @@ -3026,24 +3025,6 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, d enddo calvingThickness(iCell) = thickness(iCell) - ! < apply to the field that will be used in thickness units - - ! Now calved away the neighboring cells if the damage is also > the threshold value - ! I choose to disable this section (i.e., no recursive calving) -- TZ - !TODO: a flood fill algorithm might be needed for this threshold method in the future - !do iNeighbor = 1, nEdgesOnCell(iCell) - ! jCell = cellsOnCell(iNeighbor, iCell) - ! if (li_mask_is_floating_ice(cellMask(jCell)) .and. li_mask_is_dynamic_ice(cellMask(jCell)) & - ! .and. (.not. li_mask_is_dynamic_margin(jCell)) .and. (damage(jCell) .gt. config_damage_calving_threshold)) then - ! removeVolumeHere = cellVolume(jCell) - ! < how much we want to remove here - ! calvingThickness(jCell) = removeVolumeHere / areaCell(jCell) - ! < apply to the field that will be used in thickness units - ! cellVolume(jCell) = cellVolume(jCell) - removeVolumeHere - ! update accounting on cell volume - ! endif - !enddo - endif ! if cell is calving margin enddo ! cell loop From 704793f0619fe1b14ed3910cfbda6fa3b4b270c4 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 29 Apr 2021 17:29:31 -0700 Subject: [PATCH 013/301] Throw error for damage calving if >1 block per processor --- mode_forward/mpas_li_calving.F | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mode_forward/mpas_li_calving.F b/mode_forward/mpas_li_calving.F index dbc572c33c37..36616f89d5af 100644 --- a/mode_forward/mpas_li_calving.F +++ b/mode_forward/mpas_li_calving.F @@ -110,6 +110,7 @@ subroutine li_calve_ice(domain, err) real(kind=RKIND), pointer :: config_calving_timescale integer, pointer :: nCells + integer, pointer :: config_number_of_blocks real (kind=RKIND), pointer :: deltat !< time step (s) @@ -142,7 +143,7 @@ subroutine li_calve_ice(domain, err) call mpas_pool_get_config(liConfigs, 'config_calving_timescale', config_calving_timescale) call mpas_pool_get_config(liConfigs, 'config_print_calving_info', config_print_calving_info) call mpas_pool_get_config(liConfigs, 'config_data_calving', config_data_calving) - + call mpas_pool_get_config(liConfigs, 'config_number_of_blocks', config_number_of_blocks) if (trim(config_calving) == 'none') then return ! do nothing endif @@ -213,7 +214,12 @@ subroutine li_calve_ice(domain, err) err = ior(err, err_tmp) elseif (trim(config_calving) == 'damagecalving') then - + ! Does not currently support multiple blocks per processor. + if (config_number_of_blocks /= 0) then + call mpas_log_write("External velocity solvers require that config_number_of_blocks=0", MPAS_LOG_ERR) + err_tmp = 1 + endif + err = ior(err,err_tmp) call damage_calving(domain, err_tmp) err = ior(err, err_tmp) From 3ad0eddc8437440384f04c1d6c41fc8283212845 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 16 Apr 2021 12:51:09 -0600 Subject: [PATCH 014/301] Correct the hydro dt in the log file It had been showing the dt from the previous subcycle instead of the final one. Now it displays both. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index 4fd30f26c8a3..9751c8177d49 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -1137,12 +1137,12 @@ subroutine check_timestep(domain, timeLeft, err) ! Write out dt info on the final subcycle if (proposedDt >= timeLeft) then if (config_SGH_chnl_active) then - call mpas_log_write("deltatSGH: used=$r, advec=$r, diffu=$r, pressure=$r, advecChannel=$r, diffuChannel=$r", & - realArgs=(/deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure, deltatSGHadvecChannel, & + call mpas_log_write("deltatSGH: used=$r, prev=$r, advec=$r, diffu=$r, pressure=$r, advecChannel=$r, diffuChannel=$r",& + realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure, deltatSGHadvecChannel, & deltatSGHdiffuChannel/)) else - call mpas_log_write("deltatSGH: used=$r, advec=$r, diffu=$r, pressure=$r", & - realArgs=(/deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure/)) + call mpas_log_write("deltatSGH: used=$r, previous=$r, advec=$r, diffu=$r, pressure=$r", & + realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure/)) endif endif From ae981817616596be19c52fad5a9089161d9c8d63 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 5 Mar 2021 08:44:31 -0700 Subject: [PATCH 015/301] Ignore the water thickness head in the channel model For calculations of hydropotential gradient in the channel model, this commit swtiches from using the full hydropotential to hydropotentialBase (which ignores the hydropotential contribution from the thickness of the water). This seems to prevent channel blowups around lakes. It also might not be modeling channel interactiosn with lakes properly. This requires further investigation. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index 9751c8177d49..a15bd0127514 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -1501,7 +1501,7 @@ subroutine update_channel(block, err) real (kind=RKIND), dimension(:), pointer :: channelVelocity real (kind=RKIND), dimension(:), pointer :: gradMagPhiEdge real (kind=RKIND), dimension(:), pointer :: waterFlux - real (kind=RKIND), dimension(:), pointer :: hydropotentialSlopeNormal + real (kind=RKIND), dimension(:), pointer :: hydropotentialBaseSlopeNormal real (kind=RKIND), dimension(:), pointer :: waterPressureSlopeNormal real (kind=RKIND), dimension(:), pointer :: channelOpeningRate real (kind=RKIND), dimension(:), pointer :: channelClosingRate @@ -1546,7 +1546,7 @@ subroutine update_channel(block, err) call mpas_pool_get_array(hydroPool, 'channelDischarge', channelDischarge) call mpas_pool_get_array(hydroPool, 'channelVelocity', channelVelocity) call mpas_pool_get_array(hydroPool, 'gradMagPhiEdge', gradMagPhiEdge) - call mpas_pool_get_array(hydroPool, 'hydropotentialSlopeNormal', hydropotentialSlopeNormal) + call mpas_pool_get_array(hydroPool, 'hydropotentialBaseSlopeNormal', hydropotentialBaseSlopeNormal) call mpas_pool_get_array(hydroPool, 'waterPressureSlopeNormal', waterPressureSlopeNormal) call mpas_pool_get_array(hydroPool, 'waterFlux', waterFlux) call mpas_pool_get_array(hydroPool, 'channelOpeningRate', channelOpeningRate) @@ -1567,7 +1567,7 @@ subroutine update_channel(block, err) channelDischarge(:) = 0.0_RKIND elsewhere channelDischarge = -1.0_RKIND * Kc * channelArea**alpha_c * gradMagPhiEdge**(beta_c - 2.0_RKIND) * & - hydropotentialSlopeNormal + hydropotentialBaseSlopeNormal end where where (waterFluxMask == 2) @@ -1599,8 +1599,8 @@ subroutine update_channel(block, err) Kc * channelArea**(alpha_c - 1.0_RKIND) * gradMagPhiEdge**(beta_c - 2.0_RKIND)) end where - channelMelt = (abs(channelDischarge * hydropotentialSlopeNormal) & ! channel dissipation - + abs(waterFlux * hydropotentialSlopeNormal * config_SGH_incipient_channel_width) & !some sheet dissipation + channelMelt = (abs(channelDischarge * hydropotentialBaseSlopeNormal) & ! channel dissipation + + abs(waterFlux * hydropotentialBaseSlopeNormal * config_SGH_incipient_channel_width) & !some sheet dissipation ) / latent_heat_ice channelPressureFreeze = -1.0_RKIND * iceMeltingPointPressureDependence * cp_freshwater * rho_water * & (channelDischarge + waterFlux * config_SGH_incipient_channel_width) & From 2bad940dfab4ed9dbaf8f2fe124b51f8effa9840 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 31 Mar 2021 13:15:19 -0600 Subject: [PATCH 016/301] Make model err if SGH dt < 1 millisecond --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index a15bd0127514..9e77113421e9 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -327,7 +327,7 @@ subroutine li_SGH_solve(domain, err) ! ============= ! ============= ! ============= - do while (timeLeft > 0.0_RKIND) + timecycling: do while (timeLeft > 0.0_RKIND) numSubCycles = numSubCycles + 1 @@ -468,6 +468,9 @@ subroutine li_SGH_solve(domain, err) ! ============= call check_timestep(domain, timeLeft, err_tmp) err = ior(err, err_tmp) + if (err > 0) then + exit timecycling + endif ! ============= @@ -580,7 +583,7 @@ subroutine li_SGH_solve(domain, err) ! ============= ! ============= ! ============= - end do ! while timeLeft>0 + end do timecycling ! while timeLeft>0 ! ============= ! ============= ! ============= @@ -1146,6 +1149,10 @@ subroutine check_timestep(domain, timeLeft, err) endif endif + if (proposedDt < 1.0E-3_RKIND) then + call mpas_log_write("CFL conditions are limiting subglacial hydrology timestep to <1 millisecond.", MPAS_LOG_ERR) + err = ior(err, 1) + endif ! Don't exceed the maximum allowed hydro time step proposedDt = min(proposedDt, maxDt) @@ -1191,6 +1198,7 @@ subroutine check_timestep(domain, timeLeft, err) realArgs=(/deltatSGH, deltatSGHdiffuChannel/)) endif + !-------------------------------------------------------------------- end subroutine check_timestep From 5c1a0107045f2bcb3d9b6a4ae54ca24563bba86f Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 11 Jun 2021 12:08:26 -0600 Subject: [PATCH 017/301] Have hydro timestepper print a lot more info while subcycling --- .../mode_forward/mpas_li_subglacial_hydro.F | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index 9e77113421e9..c74c040ada2c 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -466,7 +466,7 @@ subroutine li_SGH_solve(domain, err) ! ============= ! Calculate adaptive time step ! ============= - call check_timestep(domain, timeLeft, err_tmp) + call check_timestep(domain, timeLeft, numSubCycles, err_tmp) err = ior(err, err_tmp) if (err > 0) then exit timecycling @@ -971,11 +971,12 @@ end subroutine calc_edge_quantities !> This routine calculates the three timesteps associated with the !> SGH solver and compares them to the current model timestep. !----------------------------------------------------------------------- - subroutine check_timestep(domain, timeLeft, err) + subroutine check_timestep(domain, timeLeft, numSubCycles, err) !----------------------------------------------------------------- ! input variables !----------------------------------------------------------------- + integer, intent(in) :: numSubCycles !< Input: number of subcycles taken so far !----------------------------------------------------------------- ! input/output variables @@ -1024,11 +1025,21 @@ subroutine check_timestep(domain, timeLeft, err) real(kind=RKIND) :: proposedDt real(kind=RKIND) :: masterDt real(kind=RKIND), dimension(5) :: localMinValues, reducedMinValues + integer :: iEdge + real(kind=RKIND), save :: minDtSoFar + integer, save :: numDtLess1 + logical :: printDtInfo err = 0 err_tmp = 0 + printDtInfo = .false. + if (numSubCycles == 1) then + minDtSoFar = 1.0e36_RKIND + numDtLess1 = 0 + endif + call mpas_pool_get_config(liConfigs, 'config_SGH_englacial_porosity', porosity) call mpas_pool_get_config(liConfigs, 'config_SGH_chnl_active', config_SGH_chnl_active) @@ -1136,9 +1147,36 @@ subroutine check_timestep(domain, timeLeft, err) endif proposedDt = proposedDt * CFLfraction + if (proposedDt < 1.0_RKIND) then + numDtLess1 = numDtLess1 + 1 + endif + minDtSoFar = min(minDtSoFar, proposedDt) + + ! Write out dt stats every now and then + if (mod(numSubCycles, 2000)==0) then + call mpas_log_write("SGH subcycle #$i. Time left=$r s.", intArgs=(/numSubCycles/), realArgs=(/timeLeft/)) + printDtInfo=.true. + endif ! Write out dt info on the final subcycle if (proposedDt >= timeLeft) then + call mpas_log_write("SGH completed subcycling with $i timesteps.", intArgs=(/numSubCycles/)) + printDtInfo=.true. + endif + + if (proposedDt < 1.0E-2_RKIND) then + call mpas_log_write("CFL conditions are limiting subglacial hydrology timestep to <1e-2s ($r) on subcycle $i.", & + MPAS_LOG_WARN, realArgs=(/proposedDt/), intArgs=(/numSubCycles/)) + printDtInfo=.true. + endif + + if (proposedDt < 1.0E-4_RKIND) then + call mpas_log_write("CFL conditions are limiting subglacial hydrology timestep to <1e-4s.", MPAS_LOG_ERR) + printDtInfo=.true. + err = ior(err, 1) + endif + + if (printDtInfo) then if (config_SGH_chnl_active) then call mpas_log_write("deltatSGH: used=$r, prev=$r, advec=$r, diffu=$r, pressure=$r, advecChannel=$r, diffuChannel=$r",& realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure, deltatSGHadvecChannel, & @@ -1147,11 +1185,8 @@ subroutine check_timestep(domain, timeLeft, err) call mpas_log_write("deltatSGH: used=$r, previous=$r, advec=$r, diffu=$r, pressure=$r", & realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure/)) endif - endif - - if (proposedDt < 1.0E-3_RKIND) then - call mpas_log_write("CFL conditions are limiting subglacial hydrology timestep to <1 millisecond.", MPAS_LOG_ERR) - err = ior(err, 1) + call mpas_log_write(" min hydro dt used=$r; There were $i subcyles with dt<1s.", & + realArgs=(/minDtSoFar/), intArgs=(/numDtLess1/)) endif ! Don't exceed the maximum allowed hydro time step From 19ffdb4af977924b86612ad6862fd1dde5e8a26e Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 17 Jun 2021 12:42:59 -0600 Subject: [PATCH 018/301] Add location of CFL limitations to log file --- .../mode_forward/mpas_li_subglacial_hydro.F | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index c74c040ada2c..37c05aee4aef 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -1000,6 +1000,7 @@ subroutine check_timestep(domain, timeLeft, numSubCycles, err) real (kind=RKIND), dimension(:), pointer :: diffusivity real (kind=RKIND), dimension(:), pointer :: channelDiffusivity real (kind=RKIND), dimension(:), pointer :: dcEdge + integer, dimension(:), pointer :: indexToEdgeID real (kind=RKIND), pointer :: deltatSGH real (kind=RKIND), pointer :: deltatSGHadvec real (kind=RKIND), pointer :: deltatSGHdiffu @@ -1025,6 +1026,7 @@ subroutine check_timestep(domain, timeLeft, numSubCycles, err) real(kind=RKIND) :: proposedDt real(kind=RKIND) :: masterDt real(kind=RKIND), dimension(5) :: localMinValues, reducedMinValues + integer, dimension(5) :: localLocValues, reducedLocValues integer :: iEdge real(kind=RKIND), save :: minDtSoFar integer, save :: numDtLess1 @@ -1072,6 +1074,7 @@ subroutine check_timestep(domain, timeLeft, numSubCycles, err) call mpas_pool_get_array(hydroPool, 'diffusivity', diffusivity) call mpas_pool_get_array(hydroPool, 'channelDiffusivity', channelDiffusivity) call mpas_pool_get_array(meshPool, 'dcEdge', dcEdge) + call mpas_pool_get_array(meshPool, 'indexToEdgeID', indexToEdgeID) ! Calculate advective CFL-limited time step dtSGHadvecBlock = 0.5_RKIND * minval(dcEdge(1:nEdgesSolve) / (abs(waterVelocity(1:nEdgesSolve)) & @@ -1177,13 +1180,66 @@ subroutine check_timestep(domain, timeLeft, numSubCycles, err) endif if (printDtInfo) then + ! Determine location of limiting Dt + ! NOTE: ignoring multiple blocks here. + if (localMinValues(1) == reducedMinValues(1)) then ! limiting edge is on this proc + localLocValues(1) = indexToEdgeID(minloc(dcEdge(1:nEdgesSolve) / (abs(waterVelocity(1:nEdgesSolve)) & + + 1.0e-12_RKIND), dim=1)) + else + localLocValues(1) = -1 + endif + + if (localMinValues(2) == reducedMinValues(2)) then ! limiting edge is on this proc + localLocValues(2) = indexToEdgeID(minloc(dcEdge(1:nEdgesSolve)**2 / (diffusivity(1:nEdgesSolve) + 1.0e-12_RKIND), & + dim=1)) + else + localLocValues(2) = -1 + endif + + if (localMinValues(3) == reducedMinValues(3)) then ! limiting edge is on this proc + localLocValues(3) = indexToEdgeID(minloc(porosity * dcEdge(1:nEdgesSolve)**2 & + / (2.0_RKIND * diffusivity(1:nEdgesSolve) + 1.0e-12_RKIND), dim=1)) + else + localLocValues(3) = -1 + endif + + if (config_SGH_chnl_active) then + if (localMinValues(4) == reducedMinValues(4)) then ! limiting edge is on this proc + localLocValues(4) = indexToEdgeID(minloc(dcEdge(1:nEdgesSolve) / (abs(channelVelocity(1:nEdgesSolve)) & + + 1.0e-12_RKIND), dim=1)) + else + localLocValues(4) = -1 + endif + + if (localMinValues(5) == reducedMinValues(5)) then ! limiting edge is on this proc + localLocValues(5) = indexToEdgeID(minloc(dcEdge(1:nEdgesSolve)**2 / (channelDiffusivity(1:nEdgesSolve) + & + 1.0e-12_RKIND), dim=1)) + else + localLocValues(5) = -1 + endif + + ! global reduce of locations + call mpas_timer_start("global reduce") + call mpas_dmpar_max_int_array(domain % dminfo, 5, localLocValues, reducedLocValues) + call mpas_timer_stop("global reduce") + else + call mpas_timer_start("global reduce") + call mpas_dmpar_max_int_array(domain % dminfo, 3, localLocValues, reducedLocValues) + call mpas_timer_stop("global reduce") + endif + + if (config_SGH_chnl_active) then call mpas_log_write("deltatSGH: used=$r, prev=$r, advec=$r, diffu=$r, pressure=$r, advecChannel=$r, diffuChannel=$r",& realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure, deltatSGHadvecChannel, & deltatSGHdiffuChannel/)) + call mpas_log_write("Limiting edge IDs: advec=$i, diffu=$i, pressure=$i, advecChannel=$i, diffuChannel=$i", & + intArgs=reducedLocValues(1:5)) else call mpas_log_write("deltatSGH: used=$r, previous=$r, advec=$r, diffu=$r, pressure=$r", & realArgs=(/proposedDt, deltatSGH, deltatSGHadvec, deltatSGHdiffu, deltatSGHpressure/)) + call mpas_log_write("Limiting edge IDs: advec=$i, diffu=$i, pressure=$i", & + intArgs=reducedLocValues(1:3)) endif call mpas_log_write(" min hydro dt used=$r; There were $i subcyles with dt<1s.", & realArgs=(/minDtSoFar/), intArgs=(/numDtLess1/)) From a547adb713dee2a42b237a86b882e724bab5216d Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 22 Jun 2021 14:54:38 -0700 Subject: [PATCH 019/301] Do not double-count ablation at grounding line If using face melting, zero sub-shelf melt for non-dynamic cells adjacent to grounded ice. If using ice shelf melting, do not use face melt on dynamic floating ice. This avoid redundantly applying melt to floating cells if both ice shelf melt and face melt are being used. Sub-shelf melt is still applied to all floating dynamic cells and those non-dynamic cells with only floating neighbors. --- .../src/mode_forward/mpas_li_iceshelf_melt.F | 313 +++++++++++------- 1 file changed, 185 insertions(+), 128 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F index 130a043bf0a8..f5afd1672acc 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F @@ -100,22 +100,64 @@ subroutine li_face_melt_grounded_ice(domain, err) type (block_type), pointer :: block - type (mpas_pool_type), pointer :: geometryPool + type (mpas_pool_type), pointer :: geometryPool, velocityPool, meshPool character(len=StrKIND), pointer :: & - config_front_mass_bal_grounded ! option for submarine mass balance - ! at grounded glacier front - real (kind=RKIND), dimension(:), pointer :: faceMeltSpeed, faceMeltingThickness + config_front_mass_bal_grounded, & ! option for submarine mass balance at glacier front + config_basal_mass_bal_float ! option for mass balance under ice shelf + real (kind=RKIND), dimension(:), pointer :: & + faceMeltSpeed, & + faceMeltingThickness, & + thickness + integer, dimension(:), pointer :: cellMask integer :: err_tmp err = 0 err_tmp = 0 call mpas_pool_get_config(liConfigs, 'config_front_mass_bal_grounded', config_front_mass_bal_grounded) + call mpas_pool_get_config(liConfigs, 'config_basal_mass_bal_float', config_basal_mass_bal_float) if ( trim(config_front_mass_bal_grounded) == 'ismip6' & .or. trim(config_front_mass_bal_grounded) == 'uniform' ) then call grounded_face_melt_ismip6(domain, err_tmp) err = ior(err, err_tmp) + + block => domain % blocklist + do while (associated(block)) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_array(geometryPool, 'faceMeltSpeed', faceMeltSpeed) + call mpas_pool_get_array(geometryPool, 'faceMeltingThickness', faceMeltingThickness) + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + + ! If ice shelf melting is active, only apply front melting to + ! the grounded ice margin and adjacent non-dynamic cells + if (trim(config_basal_mass_bal_float) .ne. 'none') then + where ( li_mask_is_dynamic_ice(cellMask(:)) .and. li_mask_is_floating_ice(cellMask(:)) ) + faceMeltSpeed(:) = 0.0_RKIND + faceMeltingThickness(:) = 0.0_RKIND + end where + endif + ! Update halos on calvingThickness or faceMeltingThickness before applying it. + ! Testing seemed to indicate this is not necessary, but I don't understand + ! why not, so leaving it. + ! NOTE: THIS WILL NOT WORK ON MULTIPLE BLOCKS PER PROCESSOR + call mpas_timer_start("halo updates") + call mpas_dmpar_field_halo_exch(domain, 'faceMeltingThickness') + call mpas_timer_stop("halo updates") + + ! Apply facemelt: open the Ark + thickness(:) = thickness(:) - faceMeltingThickness(:) + + ! update mask + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + + block => block % next + enddo ! associated(block) + elseif ( trim(config_front_mass_bal_grounded) == 'none' ) then ! Zero entire field @@ -206,7 +248,11 @@ subroutine li_basal_melt_floating_ice(domain, err) real (kind=RKIND), pointer :: config_ice_density !< ice density integer, dimension(:), pointer :: & - cellMask ! bit mask describing whether ice is floating, dynamically active, etc. + cellMask, & ! bit mask describing whether ice is floating, dynamically active, etc. + nEdgesOnCell, & + edgeMask + + integer, dimension(:,:), pointer :: edgesOnCell type (field1dInteger), pointer :: thermalCellMaskField @@ -225,7 +271,7 @@ subroutine li_basal_melt_floating_ice(domain, err) real(kind=RKIND), pointer :: daysSinceStart - integer :: iCell, err_tmp + integer :: iCell, iEdge, err_tmp ! Local variables for some melt methods @@ -306,153 +352,179 @@ subroutine li_basal_melt_floating_ice(domain, err) if (trim(config_basal_mass_bal_float) == 'ismip6') then call iceshelf_melt_ismip6(domain, err_tmp) err = ior(err, err_tmp) + else + ! block loop + block => domain % blocklist + do while (associated(block)) - return ! do not enter block loop below - we are done here - endif + ! get pools + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) + call mpas_pool_get_subpool(block % structs, 'scratch', scratchPool) - ! block loop - block => domain % blocklist - do while (associated(block)) + ! get dimensions + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) - ! get pools - call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) - call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) - call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) - call mpas_pool_get_subpool(block % structs, 'scratch', scratchPool) + ! get fields from the mesh pool + call mpas_pool_get_array(meshPool, 'xCell', xCell) + call mpas_pool_get_array(meshPool, 'daysSinceStart',daysSinceStart) - ! get dimensions - call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + ! get fields from the geometry pool + call mpas_pool_get_array(geometryPool, 'thickness', thickness) + call mpas_pool_get_array(geometryPool, 'lowerSurface', lowerSurface) + call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'floatingBasalMassBal', floatingBasalMassBal) - ! get fields from the mesh pool - call mpas_pool_get_array(meshPool, 'xCell', xCell) - call mpas_pool_get_array(meshPool, 'daysSinceStart',daysSinceStart) + ! get fields from the scratch pool + call mpas_pool_get_field(scratchPool, 'iceCellMask', thermalCellMaskField) + call mpas_allocate_scratch_field(thermalCellMaskField, .true.) + thermalCellMask => thermalCellMaskField % array - ! get fields from the geometry pool - call mpas_pool_get_array(geometryPool, 'thickness', thickness) - call mpas_pool_get_array(geometryPool, 'lowerSurface', lowerSurface) - call mpas_pool_get_array(geometryPool, 'bedTopography', bedTopography) - call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) - call mpas_pool_get_array(geometryPool, 'floatingBasalMassBal', floatingBasalMassBal) + ! calculate masks - so we know where the ice is floating + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + + ! calculate a mask to identify ice that is thick enough to be thermally active + do iCell = 1, nCellsSolve + if (thickness(iCell) > config_thermal_thickness) then + thermalCellMask(iCell) = 1 + else + thermalCellMask(iCell) = 0 + endif + enddo - ! get fields from the scratch pool - call mpas_pool_get_field(scratchPool, 'iceCellMask', thermalCellMaskField) - call mpas_allocate_scratch_field(thermalCellMaskField, .true.) - thermalCellMask => thermalCellMaskField % array + ! ----------------- + ! Compute basal melting for floating ice. + ! ----------------- - ! calculate masks - so we know where the ice is floating - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) + ! initialize to zero melt + floatingBasalMassBal(:) = 0.0_RKIND + if (trim(config_basal_mass_bal_float) == 'none') then + ! Do nothing, handled above - ! calculate a mask to identify ice that is thick enough to be thermally active - do iCell = 1, nCellsSolve - if (thickness(iCell) > config_thermal_thickness) then - thermalCellMask(iCell) = 1 - else - thermalCellMask(iCell) = 0 - endif - enddo + elseif (trim(config_basal_mass_bal_float) == 'file') then + ! Do nothing, handled above - ! ----------------- - ! Compute basal melting for floating ice. - ! ----------------- + elseif (trim(config_basal_mass_bal_float) == 'constant') then - ! initialize to zero melt - floatingBasalMassBal(:) = 0.0_RKIND - if (trim(config_basal_mass_bal_float) == 'none') then - ! Do nothing, handled above + ! set melt rate to a constant value for floating ice + ! allow basal melt in ice-free ocean cells, in case ice is advected to those cells by the transport scheme - elseif (trim(config_basal_mass_bal_float) == 'file') then - ! Do nothing, handled above + bmlt_float_rate = config_bmlt_float_flux / (config_ice_density*latent_heat_ice) ! convert W/m^2 to m/s - elseif (trim(config_basal_mass_bal_float) == 'constant') then + floatingBasalMassBal(:) = 0.0_RKIND - ! set melt rate to a constant value for floating ice - ! allow basal melt in ice-free ocean cells, in case ice is advected to those cells by the transport scheme + do iCell = 1, nCellsSolve + if ( (li_mask_is_floating_ice(cellMask(iCell))) .or. & + (bedTopography(iCell) < config_sea_level .and. thermalCellMask(iCell) == 0) ) then + ! ice is present and floating, or ice-free ocean - bmlt_float_rate = config_bmlt_float_flux / (config_ice_density*latent_heat_ice) ! convert W/m^2 to m/s + ! Provided xCell > bmlt_float_xlimit, prescribe a uniform basal melt rate. + ! The default is 0.0, but for MISMIP+ the prescribed value of xlimit is 480 km. + if (abs(xCell(iCell)) >= config_bmlt_float_xlimit) then ! basal melting is allowed + floatingBasalMassBal(iCell) = -bmlt_float_rate + endif - floatingBasalMassBal(:) = 0.0_RKIND + endif ! ice is present and floating - do iCell = 1, nCellsSolve - if ( (li_mask_is_floating_ice(cellMask(iCell))) .or. & - (bedTopography(iCell) < config_sea_level .and. thermalCellMask(iCell) == 0) ) then - ! ice is present and floating, or ice-free ocean - - ! Provided xCell > bmlt_float_xlimit, prescribe a uniform basal melt rate. - ! The default is 0.0, but for MISMIP+ the prescribed value of xlimit is 480 km. - if (abs(xCell(iCell)) >= config_bmlt_float_xlimit) then ! basal melting is allowed - floatingBasalMassBal(iCell) = -bmlt_float_rate - endif + enddo ! iCell - endif ! ice is present and floating + ! change units from m/s to kg/m2/s + floatingBasalMassBal(:) = floatingBasalMassBal(:) * config_ice_density - enddo ! iCell + elseif (trim(config_basal_mass_bal_float) == 'mismip') then - ! change units from m/s to kg/m2/s - floatingBasalMassBal(:) = floatingBasalMassBal(:) * config_ice_density + ! compute melt rate (m/s) based on bed depth and cavity thickness + ! The MISMIP+ formula is as follows: + ! + ! bmlt_float = omega * tanh(H_c/H_0) * max(z_0 - z_d, 0) + ! + ! where H_c = lsrf - topg is the cavity thickness + ! z_d = lsrf - eus is the ice draft + ! omega = a time scale = 0.2 yr^{-1} by default + ! H_0 = 75 m by default + ! z_0 = -100 m by default - elseif (trim(config_basal_mass_bal_float) == 'mismip') then + ! allow basal melt in ice-free ocean cells, in case ice is advected to those cells by the transport scheme - ! compute melt rate (m/s) based on bed depth and cavity thickness - ! The MISMIP+ formula is as follows: - ! - ! bmlt_float = omega * tanh(H_c/H_0) * max(z_0 - z_d, 0) - ! - ! where H_c = lsrf - topg is the cavity thickness - ! z_d = lsrf - eus is the ice draft - ! omega = a time scale = 0.2 yr^{-1} by default - ! H_0 = 75 m by default - ! z_0 = -100 m by default + floatingBasalMassBal(:) = 0.0_RKIND - ! allow basal melt in ice-free ocean cells, in case ice is advected to those cells by the transport scheme + do iCell = 1, nCellsSolve - floatingBasalMassBal(:) = 0.0_RKIND + if ( (li_mask_is_floating_ice(cellMask(iCell))) .or. & + (bedTopography(iCell) < config_sea_level .and. thermalCellMask(iCell) == 0) ) then + ! ice is present and floating, or ice-free ocean - do iCell = 1, nCellsSolve + hCavity = lowerSurface(iCell) - bedTopography(iCell) + zDraft = lowerSurface(iCell) - config_sea_level + floatingBasalMassBal(iCell) = -bmlt_float_omega * tanh(hCavity/bmlt_float_h0) * max(bmlt_float_z0 - & + zDraft, 0.0_RKIND) - if ( (li_mask_is_floating_ice(cellMask(iCell))) .or. & - (bedTopography(iCell) < config_sea_level .and. thermalCellMask(iCell) == 0) ) then - ! ice is present and floating, or ice-free ocean + endif ! ice is present and floating + enddo ! iCell - hCavity = lowerSurface(iCell) - bedTopography(iCell) - zDraft = lowerSurface(iCell) - config_sea_level - floatingBasalMassBal(iCell) = -bmlt_float_omega * tanh(hCavity/bmlt_float_h0) * max(bmlt_float_z0 - & - zDraft, 0.0_RKIND) + ! change units from m/s to kg/m2/s + floatingBasalMassBal(:) = floatingBasalMassBal(:) * config_ice_density - endif ! ice is present and floating - enddo ! iCell + elseif (trim(config_basal_mass_bal_float) == 'seroussi') then - ! change units from m/s to kg/m2/s - floatingBasalMassBal(:) = floatingBasalMassBal(:) * config_ice_density + call basal_melt_thwaites_seroussi(floatingBasalMassBal, daysSinceStart, lowerSurface, cellMask, & + config_sea_level, config_ice_density, nCellsSolve, err_tmp) + err = ior(err, err_tmp) - elseif (trim(config_basal_mass_bal_float) == 'seroussi') then + elseif (trim(config_basal_mass_bal_float) == 'temperature_profile') then - call basal_melt_thwaites_seroussi(floatingBasalMassBal, daysSinceStart, lowerSurface, cellMask, & - config_sea_level, config_ice_density, nCellsSolve, err_tmp) - err = ior(err, err_tmp) + call iceshelf_melt_param_from_temp_profile(nCellsSolve, lowerSurface, cellMask, & + config_sea_level, config_ice_density, config_print_thermal_info, & + GLdepth, CFdepth, daysSinceStart, floatingBasalMassBal, err_tmp) + err = ior(err, err_tmp) - elseif (trim(config_basal_mass_bal_float) == 'temperature_profile') then + else - call iceshelf_melt_param_from_temp_profile(nCellsSolve, lowerSurface, cellMask, & - config_sea_level, config_ice_density, config_print_thermal_info, & - GLdepth, CFdepth, daysSinceStart, floatingBasalMassBal, err_tmp) - err = ior(err, err_tmp) + call mpas_log_write('Unknown option selected for config_basal_mass_bal_float:' // & + trim(config_basal_mass_bal_float), MPAS_LOG_ERR) + err = ior(err, 1) - else + endif ! config_basal_mass_bal_float - call mpas_log_write('Unknown option selected for config_basal_mass_bal_float:' // & - trim(config_basal_mass_bal_float), MPAS_LOG_ERR) - err = ior(err, 1) - endif ! config_basal_mass_bal_float + ! clean up + call mpas_deallocate_scratch_field(thermalCellMaskField, .true.) + block => block % next + enddo ! associated(block) + endif - ! clean up - call mpas_deallocate_scratch_field(thermalCellMaskField, .true.) + if (trim(config_front_mass_bal_grounded) .ne. 'none') then + block => domain % blocklist + do while (associated(block)) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_array(meshPool, 'edgesOnCell', edgesOnCell) + call mpas_pool_get_array(geometryPool, 'floatingBasalMassBal', floatingBasalMassBal) + call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) + call mpas_pool_get_array(geometryPool, 'edgeMask', edgeMask) - block => block % next - enddo ! associated(block) + ! If config_front_mass_bal_grounded is not none, only apply ice shelf melt to active cells + ! and stranded non-dynamic cells. + do iCell = 1, nCellsSolve + if ( li_mask_is_floating_ice(cellMask(iCell)) .and. (.not. li_mask_is_dynamic_ice(cellMask(iCell))) ) then + do iEdge = 1, nEdgesOnCell(iCell) + if ( li_mask_is_grounding_line(edgeMask(edgesOnCell(iEdge,iCell))) ) then + + floatingBasalMassBal(iCell) = 0.0_RKIND + endif + enddo + endif + enddo + block => block % next + enddo ! associated(block) + endif end subroutine li_basal_melt_floating_ice !----------------------------------------------------------------------- @@ -1364,21 +1436,6 @@ subroutine grounded_face_melt_ismip6(domain, err) applyToFloating, applyToGroundingLine, domain, err) - ! Update halos on calvingThickness or faceMeltingThickness before applying it. - ! Testing seemed to indicate this is not necessary, but I don't understand - ! why not, so leaving it. - ! NOTE: THIS WILL NOT WORK ON MULTIPLE BLOCKS PER PROCESSOR - call mpas_timer_start("halo updates") - call mpas_dmpar_field_halo_exch(domain, 'faceMeltingThickness') - call mpas_timer_stop("halo updates") - - ! Apply facemelt: open the Ark - thickness(:) = thickness(:) - faceMeltingThickness(:) - - ! update mask - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) - block => block % next enddo ! associated(block) From a063a34c2e8801cf7cfeb7e8c9c38ec0b763864c Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Wed, 23 Jun 2021 15:36:12 -0700 Subject: [PATCH 020/301] Use applyToGrounded and applyToGroundingLine flags Define applyToGrounded and applyToGroundingLine appropriately to avoid calculating faceMeltingThickness and faceMeltSpeed under dynamic floating ice and then zeroing them in the case that both face-melt and shelf-melt are active. --- .../src/mode_forward/mpas_li_iceshelf_melt.F | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F index f5afd1672acc..c6ac6a031844 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_iceshelf_melt.F @@ -110,6 +110,7 @@ subroutine li_face_melt_grounded_ice(domain, err) thickness integer, dimension(:), pointer :: cellMask integer :: err_tmp + logical :: applyToFloating, applyToGrounded, applyToGroundingLine err = 0 err_tmp = 0 @@ -117,9 +118,22 @@ subroutine li_face_melt_grounded_ice(domain, err) call mpas_pool_get_config(liConfigs, 'config_front_mass_bal_grounded', config_front_mass_bal_grounded) call mpas_pool_get_config(liConfigs, 'config_basal_mass_bal_float', config_basal_mass_bal_float) + ! Determine where to apply melt. This avoids redundant melting at the grounding line + ! if both face-melting and ice shelf melting are active. + applyToFloating = .false. ! As of 06/2021 there are no face-melt routines for floating ice + if (trim(config_basal_mass_bal_float) .eq. 'none') then + applyToGrounded = .false. + applyToGroundingLine = .true. + else + applyToGrounded = .true. + applyToGroundingLine = .false. + endif + + if ( trim(config_front_mass_bal_grounded) == 'ismip6' & .or. trim(config_front_mass_bal_grounded) == 'uniform' ) then - call grounded_face_melt_ismip6(domain, err_tmp) + call grounded_face_melt_ismip6(domain, applyToGrounded, & + applyToFloating, applyToGroundingLine, err_tmp) err = ior(err, err_tmp) block => domain % blocklist @@ -127,19 +141,9 @@ subroutine li_face_melt_grounded_ice(domain, err) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) - call mpas_pool_get_array(geometryPool, 'faceMeltSpeed', faceMeltSpeed) call mpas_pool_get_array(geometryPool, 'faceMeltingThickness', faceMeltingThickness) call mpas_pool_get_array(geometryPool, 'thickness', thickness) - call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) - ! If ice shelf melting is active, only apply front melting to - ! the grounded ice margin and adjacent non-dynamic cells - if (trim(config_basal_mass_bal_float) .ne. 'none') then - where ( li_mask_is_dynamic_ice(cellMask(:)) .and. li_mask_is_floating_ice(cellMask(:)) ) - faceMeltSpeed(:) = 0.0_RKIND - faceMeltingThickness(:) = 0.0_RKIND - end where - endif ! Update halos on calvingThickness or faceMeltingThickness before applying it. ! Testing seemed to indicate this is not necessary, but I don't understand ! why not, so leaving it. @@ -1308,7 +1312,8 @@ end subroutine iceshelf_melt_ismip6 !----------------------------------------------------------------------- - subroutine grounded_face_melt_ismip6(domain, err) + subroutine grounded_face_melt_ismip6(domain, applyToGrounded, & + applyToFloating, applyToGroundingLine, err) use li_calving !----------------------------------------------------------------- @@ -1316,7 +1321,7 @@ subroutine grounded_face_melt_ismip6(domain, err) !----------------------------------------------------------------- type (domain_type), intent(inout) :: & domain !< Input/Output: domain object - + logical, intent(in) :: applyToFloating, applyToGrounded, applyToGroundingLine !----------------------------------------------------------------- ! output variables !----------------------------------------------------------------- @@ -1353,16 +1358,10 @@ subroutine grounded_face_melt_ismip6(domain, err) !to pass to li_apply_front_ablation_velocity, !because faceMeltSpeed is only below water-line integer :: err_tmp - logical :: applyToFloating, applyToGrounded, applyToGroundingLine err = 0 call mpas_log_write('Starting face melt routine') - ! Define logicals for call to li_apply_front_ablation_velocity - applyToFloating = .false. ! if true, only apply to floating ice - applyToGrounded = .false. ! if true, only apply to grounded - applyToGroundingLine = .true. ! if true, apply at grounding line - ! Get sea level, bedTopography, ice density call mpas_pool_get_config(liConfigs, 'config_ice_density', rhoi) call mpas_pool_get_config(liConfigs, 'config_sea_level', seaLevel) From 370583d2b137da1b517e693238e796f77263a0ab Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Fri, 25 Jun 2021 10:30:02 -0700 Subject: [PATCH 021/301] Fix spacing and indentation issues --- .../src/mode_forward/mpas_li_calving.F | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index 36616f89d5af..c701fd91b262 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -144,6 +144,7 @@ subroutine li_calve_ice(domain, err) call mpas_pool_get_config(liConfigs, 'config_print_calving_info', config_print_calving_info) call mpas_pool_get_config(liConfigs, 'config_data_calving', config_data_calving) call mpas_pool_get_config(liConfigs, 'config_number_of_blocks', config_number_of_blocks) + if (trim(config_calving) == 'none') then return ! do nothing endif @@ -214,11 +215,11 @@ subroutine li_calve_ice(domain, err) err = ior(err, err_tmp) elseif (trim(config_calving) == 'damagecalving') then - ! Does not currently support multiple blocks per processor. - if (config_number_of_blocks /= 0) then - call mpas_log_write("External velocity solvers require that config_number_of_blocks=0", MPAS_LOG_ERR) - err_tmp = 1 - endif + ! Does not currently support multiple blocks per processor. + if (config_number_of_blocks /= 0) then + call mpas_log_write("External velocity solvers require that config_number_of_blocks=0", MPAS_LOG_ERR) + err_tmp = 1 + endif err = ior(err,err_tmp) call damage_calving(domain, err_tmp) err = ior(err, err_tmp) @@ -3250,8 +3251,8 @@ subroutine remove_icebergs(domain) call mpas_timer_start("iceberg detection") call mpas_log_write("Iceberg-detection flood-fill begin.") - ! Allocate needed scratch fields - + + ! Allocate needed scratch fields block => domain % blocklist call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) From d84360d873ab8d3274ce03369f81305229808865 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Jun 2021 10:33:33 -0600 Subject: [PATCH 022/301] Calc mask before first use of it in hydro init Water pressure was getting clobbered with garbage values in areas below sea level. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index 37c05aee4aef..edfed9c1e2c2 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -156,6 +156,10 @@ subroutine li_SGH_init(domain, err) ! TODO: Set time step appropriately on first subcycle of init deltatSGH = 1.0e-4_RKIND ! in seconds + ! Mask needs to be initialized for pressure calcs to be correct + call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) + err = ior(err, err_tmp) + ! remove invalid values - not necessary on restart, but shouldn't hurt call mpas_pool_get_array(hydroPool, 'waterThickness', waterThickness) waterThickness = max(0.0_RKIND, waterThickness) @@ -176,9 +180,6 @@ subroutine li_SGH_init(domain, err) waterPressure = rhoo * gravity * (config_sea_level - bedTopography) end where - ! Mask needs to be initialized for pressure calcs to be correct - call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) - err = ior(err, err_tmp) ! Initialize diagnostic pressure variables call calc_pressure_diag_vars(block, err_tmp) err = ior(err, err_tmp) From 5507a71b13b04a32cbc9cfc5977fdbaf983303e8 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Mon, 28 Jun 2021 14:47:58 -0600 Subject: [PATCH 023/301] Adjust if statement for readability It was technically correct before, but this change is a little cleaner looking. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index edfed9c1e2c2..121118920127 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -862,7 +862,7 @@ subroutine calc_edge_quantities(block, err) hydropotentialBaseSlopeNormal(iEdge) = 0.0_RKIND hydropotentialSlopeNormal(iEdge) = 0.0_RKIND endif - else ! phi<=0 + elseif (hydropotentialBaseSlopeNormal(iEdge) < 0.0_RKIND) then ! flow is from cell1 to cell2 if (.not. li_mask_is_grounded_ice(cellMask(cell1))) then hydropotentialBaseSlopeNormal(iEdge) = 0.0_RKIND From 4431586dd364f9d3026d24fc2a29a1cdc3e23a6b Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Mon, 28 Jun 2021 15:18:23 -0600 Subject: [PATCH 024/301] Allow water thickness diffusion across the GL/margin Before this commit, the water thickness diffusion term was zeroed at the grounding line or margin. However, the Humboldy domain had a lot of large water thicknesses (lakes) along the grounding line. This commit allows water thickness diffusion to occur at the GL. It assumes that the water thickness across the GL is zero, which can lead to large diffusion values, but the diffusivity itself is based on a 1-sided evaluaton of water thickness. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index 121118920127..bf74d04b90e9 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -934,13 +934,11 @@ subroutine calc_edge_quantities(block, err) waterFluxAdvec(iEdge) = waterVelocity(iEdge) * waterThicknessEdgeUpwind(iEdge) ! diffusive flux - numGroundedCells = li_mask_is_grounded_ice_int(cellMask(cell1)) + li_mask_is_grounded_ice_int(cellMask(cell2)) - if (numGroundedCells < 2) then - waterFluxDiffu(iEdge) = 0.0_RKIND - else - waterFluxDiffu(iEdge) = -1.0_RKIND * diffusivity(iEdge) * (waterThickness(cell2) - waterThickness(cell1)) & + ! Note: At the GL, the water thickness for one cell will be 0, meaning a large gradient. However, the + ! diffusivity uses a one sided water thickness. It's unclear what really happens to lakes at the GL/margin. + waterFluxDiffu(iEdge) = -1.0_RKIND * diffusivity(iEdge) * (waterThickness(cell2) - waterThickness(cell1)) & / dcEdge(iEdge) - endif + !endif end do where (waterFluxMask == 2) waterFluxAdvec = 0.0_RKIND From 12a4159ff1bc9c89b84455d771c58ca21306b97c Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Wed, 30 Jun 2021 15:35:49 -0700 Subject: [PATCH 025/301] Clean up extraneous fields and comments Remove variables and comments that are no longer used. Also add more description of flood_fill subroutine. --- .../src/mode_forward/mpas_li_calving.F | 44 ++++--------------- 1 file changed, 8 insertions(+), 36 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index c701fd91b262..82035db3b30d 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -217,7 +217,7 @@ subroutine li_calve_ice(domain, err) elseif (trim(config_calving) == 'damagecalving') then ! Does not currently support multiple blocks per processor. if (config_number_of_blocks /= 0) then - call mpas_log_write("External velocity solvers require that config_number_of_blocks=0", MPAS_LOG_ERR) + call mpas_log_write("damagecalving requires that config_number_of_blocks=0", MPAS_LOG_ERR) err_tmp = 1 endif err = ior(err,err_tmp) @@ -3265,14 +3265,8 @@ subroutine remove_icebergs(domain) call mpas_allocate_scratch_field(growMaskField, single_block_in = .false.) growMask => growMaskField % array - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) - call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) - - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) - call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) - !allocate(seedMask(nCells+1)) - !allocate(growMask(nCells+1)) + seedMask(:) = 0 growMask(:) = 0 @@ -3335,16 +3329,11 @@ subroutine remove_icebergs(domain) if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_floating_ice(cellMask(iCell))) then calvingThickness(iCell) = calvingThickness(iCell) + thickness(iCell) ! remove any remaining ice here thickness(iCell) = 0.0_RKIND - contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed + !contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed endif enddo localIcebergCellCount = sum(contiguousDynamicIceOld) -! where(contiguousDynamicIce == 0 .and. li_mask_is_dynamic_ice(cellMask)) -! calvingThickness = calvingThickness + thickness ! remove any remaining ice here -! thickness = 0.0_RKIND -! end where - block => block % next end do @@ -3357,12 +3346,8 @@ subroutine remove_icebergs(domain) call mpas_timer_stop("halo updates") ! clean up - call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) - call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.false.) call mpas_deallocate_scratch_field(growMaskField, single_block_in=.false.) - !deallocate(seedMask) - !deallocate(growMask) call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) call mpas_timer_stop("iceberg detection") end subroutine remove_icebergs @@ -3371,7 +3356,11 @@ end subroutine remove_icebergs ! ! routine flood_fill ! -!> \brief Flood fill routine to differentiate ice sheet from icebergs +!> \brief Flood fill routine to differentiate ice sheet from icebergs, critically damaged regions, etc. +!> The calling routine defines the seedMask to specify the initial masked region, and the growMask +!> to specify the potential region to be filled by the flood-fill routine. The flood-fill routine +!> then adds the contiguous areas of the growMask to the seedMask to produce the final mask, which +!> is passed back to the calling routine for application. !> \author Trevor Hillebrand and Matthew Hoffman !> \date March 2021 !> \details @@ -3389,12 +3378,6 @@ subroutine flood_fill(seedMask, growMask, domain) type (mpas_pool_type), pointer :: geometryPool type (mpas_pool_type), pointer :: velocityPool - type (field1dInteger), pointer :: contiguousDynamicIceField - type (field1dInteger), pointer :: contiguousDynamicIceOldField - - real (kind=RKIND), dimension(:), pointer :: calvingThickness !thickness of ice that calves (computed in this subroutine) - real (kind=RKIND), dimension(:), pointer :: thickness - integer, dimension(:), pointer :: cellMask integer, dimension(:,:), pointer :: cellsOnCell ! list of cells that neighbor each cell integer, dimension(:), pointer :: nEdgesOnCell ! number of cells that border each cell integer, dimension(:), allocatable :: seedMaskOld @@ -3406,7 +3389,6 @@ subroutine flood_fill(seedMask, growMask, domain) integer :: newMaskCountLocal, newMaskCountLocalAccum, newMaskCountGlobal integer :: err_tmp, err integer :: globalLoopCount, localLoopCount - integer :: localIcebergCellCount, globalIcebergCellCount err = 0 @@ -3416,11 +3398,6 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) allocate(seedMaskOld(nCells+1)) seedMaskOld(:) = 0 - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIce', contiguousDynamicIceField) - call mpas_allocate_scratch_field(contiguousDynamicIceField, single_block_in = .false.) - - call mpas_pool_get_field(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOldField) - call mpas_allocate_scratch_field(contiguousDynamicIceOldField, single_block_in = .false.) call mpas_log_write("Flood-fill: allocated.") call mpas_pool_get_dimension(meshPool, 'nCells', nCells) @@ -3431,7 +3408,6 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) - call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) @@ -3488,13 +3464,11 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) - call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) call mpas_pool_get_array(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOld) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - !allocate(seedMaskOld(nCells+1)) ! initialize local loop localLoopCount = 0 newMaskCountLocal = 1 ! need to make sure we enter the loop @@ -3538,8 +3512,6 @@ subroutine flood_fill(seedMask, growMask, domain) endif end do ! global loop deallocate(seedMaskOld) - call mpas_deallocate_scratch_field(contiguousDynamicIceField, single_block_in=.false.) - call mpas_deallocate_scratch_field(contiguousDynamicIceOldField, single_block_in=.false.) end subroutine flood_fill From 1ce0e167930707cf92c7a92f2496daa59173ea17 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 1 Jul 2021 10:34:09 -0600 Subject: [PATCH 026/301] Ensure hydropotential adjacent to ocean does not fall below sea level Because of the check that prevents ocean inflow in from the ocean to the glacier, it is possible for hydropotential to drop far below the ocean level, for example if efficient channels develop at such a location. In effect, there is no longer an enforced boundary condition once the hydropotential first drops below sea level, because such a condition does not feel the b.c. via an influx from the ocean. Thus we need to apply a Dirichlet b.c. in those situations to keep hydrpotential close (at) sea level in such locations. If we ever remove the restriction that water cannot flow in from the ocean, then this check should be reverted. --- .../src/mode_forward/mpas_li_subglacial_hydro.F | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F index bf74d04b90e9..b985159d2404 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_subglacial_hydro.F @@ -1449,11 +1449,18 @@ subroutine calc_pressure(block, err) waterPressure = max(0.0_RKIND, waterPressure) waterPressure = min(waterPressure, rhoi * gravity * thickness) - ! set pressure correctly under floating ice and open ocean + do iCell = 1, nCells if ( li_mask_is_floating_ice(cellMask(iCell)) .or. & ((.not. li_mask_is_ice(cellMask(iCell))) .and. (bedTopography(iCell) < config_sea_level) ) ) then + ! set pressure correctly under floating ice and open ocean waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) + elseif (li_mask_is_grounding_line(cellMask(iCell))) then + ! At GL, don't let water pressure fall below ocean pressure + ! TODO: Not sure if this should include the water layer thickness term. Leaving it off. + if (waterPressure(iCell) < rhoo * gravity * (config_sea_level - bedTopography(iCell))) then + waterPressure(iCell) = rhoo * gravity * (config_sea_level - bedTopography(iCell)) + endif endif enddo From d44386e1c2f344101ef1f43110f38145590b1b5e Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 1 Jul 2021 15:10:56 -0600 Subject: [PATCH 027/301] Add option to have higher sheet cond. for water thickness > bump height Is is disabled by default. --- .../src/Registry_subglacial_hydro.xml | 4 ++++ .../mode_forward/mpas_li_subglacial_hydro.F | 21 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml index 96075442a59b..6179f9af8114 100644 --- a/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml +++ b/components/mpas-albany-landice/src/Registry_subglacial_hydro.xml @@ -45,6 +45,10 @@ description="conductivity coefficient for subglacial water flux" possible_values="positive real number" /> + 0.0_RKIND) then + ! Use a thickness weighted conductivity coeff. when water thickness exceeds bump height + do iEdge = 1, nEdges + conduc_coeff_wtd = (conduc_coeff * min(waterThicknessEdge(iEdge), bedRoughMax) + & + conduc_coeff_drowned * max(waterThicknessEdge(iEdge) - bedRoughMax, 0.0_RKIND)) / & + (waterThicknessEdge(iEdge) + 1.0e-16_RKIND) ! Regularization only applies where value doesn't matter + effectiveConducEdge(iEdge) = conduc_coeff_wtd * waterThicknessEdge(iEdge)**(alpha-1.0_RKIND) * & + (gradMagPhiEdge(iEdge)+1.0e-30_RKIND)**(beta - 2.0_RKIND) ! small value used for regularization + end do + else + ! Just use a single conductivity coeff. + effectiveConducEdge(:) = conduc_coeff * waterThicknessEdge(:)**(alpha-1.0_RKIND) *& + (gradMagPhiEdge(:)+1.0e-30_RKIND)**(beta - 2.0_RKIND) ! small value used for regularization + endif ! calculate diffusivity on edges diffusivity(:) = rho_water * gravity * effectiveConducEdge(:) * waterThicknessEdge(:) From 8763f9f7c08e648bba184bd8f52122b60d6f3bfd Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 1 Jul 2021 16:12:42 -0700 Subject: [PATCH 028/301] Remove contiguousDynamicIce and contiguousDynamicIceOld These variables were superfluous and now have been replaced with seedMask and seedMaskOld. --- .../mpas-albany-landice/src/Registry.xml | 8 ---- .../src/mode_forward/mpas_li_calving.F | 47 ++++++------------- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index 468669445447..99aef942b14a 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -1167,14 +1167,6 @@ is the value of that variable from the *previous* time level! - - domain % blocklist do while (associated(block)) @@ -3319,20 +3312,20 @@ subroutine remove_icebergs(domain) call mpas_pool_get_array(geometryPool, 'cellMask', cellMask) call mpas_pool_get_array(geometryPool, 'thickness', thickness) call mpas_pool_get_array(geometryPool, 'calvingThickness', calvingThickness) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOld) call mpas_pool_get_dimension(geometryPool, 'nCells', nCells) call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) + !allocate(seedMaskOld(nCells+1)) ! debug: make this a mask of where icebergs were removed + !seedMaskOld(:) = 0 - contiguousDynamicIceOld(:) = 0 + localIcebergCellCount = 0 do iCell = 1, nCellsSolve - if (contiguousDynamicIce(iCell) == 0 .and. li_mask_is_floating_ice(cellMask(iCell))) then + if (seedMask(iCell) == 0 .and. li_mask_is_floating_ice(cellMask(iCell))) then calvingThickness(iCell) = calvingThickness(iCell) + thickness(iCell) ! remove any remaining ice here thickness(iCell) = 0.0_RKIND - !contiguousDynamicIceOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed + localIcebergCellCount = localIcebergCellCount + 1 + !seedMaskOld(iCell) = 1 ! debug: make this a mask of where icebergs were removed endif enddo - localIcebergCellCount = sum(contiguousDynamicIceOld) block => block % next end do @@ -3346,6 +3339,7 @@ subroutine remove_icebergs(domain) call mpas_timer_stop("halo updates") ! clean up + !deallocate(seedMaskOld) !un-comment for debugging call mpas_deallocate_scratch_field(seedMaskField, single_block_in=.false.) call mpas_deallocate_scratch_field(growMaskField, single_block_in=.false.) call mpas_log_write("Iceberg-detection flood-fill complete. Removed $i iceberg cells.", intArgs=(/globalIcebergCellCount/)) @@ -3359,8 +3353,9 @@ end subroutine remove_icebergs !> \brief Flood fill routine to differentiate ice sheet from icebergs, critically damaged regions, etc. !> The calling routine defines the seedMask to specify the initial masked region, and the growMask !> to specify the potential region to be filled by the flood-fill routine. The flood-fill routine -!> then adds the contiguous areas of the growMask to the seedMask to produce the final mask, which -!> is passed back to the calling routine for application. +!> then adds the contiguous areas of the growMask to the seedMask, which is passed back to the +!> calling routine for application. Any integer fields in Registry can be used for seedMask and +!> growMask, but there are existing variables with those names that can be used. !> \author Trevor Hillebrand and Matthew Hoffman !> \date March 2021 !> \details @@ -3382,9 +3377,7 @@ subroutine flood_fill(seedMask, growMask, domain) integer, dimension(:), pointer :: nEdgesOnCell ! number of cells that border each cell integer, dimension(:), allocatable :: seedMaskOld - integer, dimension(:), pointer :: contiguousDynamicIce, contiguousDynamicIceOld integer, pointer :: nCells, nCellsSolve - integer, dimension(:), pointer :: nCellsArray integer :: iCell, jCell, n integer :: newMaskCountLocal, newMaskCountLocalAccum, newMaskCountGlobal integer :: err_tmp, err @@ -3408,7 +3401,6 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) call mpas_pool_get_dimension(geometryPool, 'nCellsSolve', nCellsSolve) call mpas_pool_get_array(meshPool, 'cellsOnCell', cellsOnCell) @@ -3420,14 +3412,7 @@ subroutine flood_fill(seedMask, growMask, domain) err = ior(err, err_tmp) call mpas_log_write("Iceberg-detection flood-fill: updated masks.") - contiguousDynamicIce(:) = 0 ! initialize - newMaskCountLocal = 0 - do iCell = 1, nCellsSolve - if ( seedMask(iCell) > 0 ) then - contiguousDynamicIce(iCell) = 1 - newMaskCountLocal = newMaskCountLocal + 1 - endif - enddo + newMaskCountLocal = sum(seedMask) call mpas_log_write("Initialized $i cells to local mask", intArgs=(/newMaskCountLocal/)) block => block % next @@ -3449,7 +3434,6 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_log_write(" Starting global processor loop $i", intArgs=(/globalLoopCount/)) ! First Update halos call mpas_timer_start("halo updates") - call mpas_dmpar_field_halo_exch(domain, 'contiguousDynamicIce') call mpas_dmpar_field_halo_exch(domain, 'growMask') call mpas_dmpar_field_halo_exch(domain, 'seedMask') call mpas_timer_stop("halo updates") @@ -3464,10 +3448,7 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) call mpas_pool_get_subpool(block % structs, 'velocity', velocityPool) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIce', contiguousDynamicIce) - call mpas_pool_get_array(geometryPool, 'contiguousDynamicIceOld', contiguousDynamicIceOld) call mpas_pool_get_dimension(meshPool, 'nCells', nCells) - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) ! initialize local loop localLoopCount = 0 @@ -3480,7 +3461,7 @@ subroutine flood_fill(seedMask, growMask, domain) newMaskCountLocal = 0 seedMaskOld(:) = seedMask(:) - do iCell = 1, nCellsArray(1) ! this gives owned cells only + do iCell = 1, nCellsSolve ! this gives owned cells only if ( growMask(iCell)>0 ) then ! If it has a marked neighbor, then add it to the mask do n = 1, nEdgesOnCell(iCell) From 40a5bb710ee4860cbd6734b72ec95a5b8e3fc6d7 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 2 Jul 2021 15:45:11 -0600 Subject: [PATCH 029/301] Update/reduce flood fill log output --- .../src/mode_forward/mpas_li_calving.F | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index 8dace1a9e527..10c47c963bd7 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -3392,7 +3392,7 @@ subroutine flood_fill(seedMask, growMask, domain) allocate(seedMaskOld(nCells+1)) seedMaskOld(:) = 0 - call mpas_log_write("Flood-fill: allocated.") + !call mpas_log_write("Flood-fill: allocated.") call mpas_pool_get_dimension(meshPool, 'nCells', nCells) ! First mark grounded ice to initialize flood fill mask @@ -3411,14 +3411,14 @@ subroutine flood_fill(seedMask, growMask, domain) call li_calculate_mask(meshPool, velocityPool, geometryPool, err_tmp) err = ior(err, err_tmp) - call mpas_log_write("Iceberg-detection flood-fill: updated masks.") + !call mpas_log_write("Flood-fill: updated masks.") newMaskCountLocal = sum(seedMask) - call mpas_log_write("Initialized $i cells to local mask", intArgs=(/newMaskCountLocal/)) + !call mpas_log_write("Initialized $i cells to local mask", intArgs=(/newMaskCountLocal/)) block => block % next end do - call mpas_log_write("Iceberg-detection flood-fill initialization complete.") + !call mpas_log_write("Flood-fill initialization complete.") ! Outer loop over processors (should also have a loop over blocks) ! Inner loop over cells on that processor @@ -3427,11 +3427,11 @@ subroutine flood_fill(seedMask, growMask, domain) call mpas_dmpar_sum_int(domain % dminfo, newMaskCountLocal, newMaskCountGlobal) globalLoopCount = 0 - call mpas_log_write("Initialized $i cells to global mask", intArgs=(/newMaskCountGlobal/)) + call mpas_log_write("Flood fill initialized $i cells to global seedMask", intArgs=(/newMaskCountGlobal/)) do while (newMaskCountGlobal > 0) globalLoopCount = globalLoopCount + 1 - call mpas_log_write(" Starting global processor loop $i", intArgs=(/globalLoopCount/)) + !call mpas_log_write(" Starting global processor loop $i", intArgs=(/globalLoopCount/)) ! First Update halos call mpas_timer_start("halo updates") call mpas_dmpar_field_halo_exch(domain, 'growMask') @@ -3455,14 +3455,14 @@ subroutine flood_fill(seedMask, growMask, domain) newMaskCountLocal = 1 ! need to make sure we enter the loop do while (newMaskCountLocal > 0) localLoopCount = localLoopCount + 1 - call mpas_log_write(" Starting local cell loop $i", intArgs=(/localLoopCount/)) + !call mpas_log_write(" Starting local cell loop $i", intArgs=(/localLoopCount/)) ! initialize newMaskCountLocal = 0 seedMaskOld(:) = seedMask(:) do iCell = 1, nCellsSolve ! this gives owned cells only - if ( growMask(iCell)>0 ) then + if ( growMask(iCell)>0 ) then ! If it has a marked neighbor, then add it to the mask do n = 1, nEdgesOnCell(iCell) jCell = cellsOnCell(n, iCell) @@ -3478,7 +3478,7 @@ subroutine flood_fill(seedMask, growMask, domain) ! Accumulate cells added locally until we do the next global ! reduce newMaskCountLocalAccum = newMaskCountLocalAccum + newMaskCountLocal - call mpas_log_write(" Added $i new cells to local mask", intArgs=(/newMaskCountLocal/)) + !call mpas_log_write(" Added $i new cells to local mask", intArgs=(/newMaskCountLocal/)) enddo ! local mask loop block => block % next @@ -3494,6 +3494,8 @@ subroutine flood_fill(seedMask, growMask, domain) end do ! global loop deallocate(seedMaskOld) + call mpas_log_write("Flood fill complete.") + end subroutine flood_fill end module li_calving From a314035d0035ee5cdde387181f2df75cfd358789 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 30 Mar 2021 14:43:23 -0700 Subject: [PATCH 030/301] Add option for speed limit on calvingVelocity Currently only supported for von Mises calving, after Choi et al. (2018): Comparison of four calving laws to model Greenland outlet glaciers, who impose a 3km/yr speed limit. Default is set to 100 m/s, so essentially no speed limit. --- .../mpas-albany-landice/src/Registry.xml | 4 ++++ .../src/mode_forward/mpas_li_calving.F | 20 +++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index 99aef942b14a..0bc79d5609b8 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -184,6 +184,10 @@ description="If true, then ice from small grounded islands is removed. Specifically, this finds one- and two-cell masses of ice that are surrounded by open ocean and eliminates them by sending them to the calving flux. Grounded such ice masses would be islands, floating would be icebergs. The ice removed is added to the calvingThickness field." possible_values=".true. or .false." /> + 0.0", MPAS_LOG_ERR) @@ -1594,17 +1593,18 @@ subroutine von_Mises_calving(domain, err) ! Calculate calving velocity for grounded cells at marine margin if ( .not. li_mask_is_floating_ice(cellMask(iCell)) ) then - calvingVelocity(iCell) = sqrt(xvelmean(iCell)**2.0_RKIND + yvelmean(iCell)**2.0_RKIND) * & - vonMisesStress(iCell) / config_grounded_von_Mises_threshold_stress + calvingVelocity(iCell) = min(sqrt(xvelmean(iCell)**2.0_RKIND + yvelmean(iCell)**2.0_RKIND) * & + vonMisesStress(iCell) / config_grounded_von_Mises_threshold_stress, config_calving_speed_limit) ! If config_floating_von_Mises_threshold_stress is not 0.0, calculate ! calvingVelocity. If config_floating_von_Mises_threshold_stress is ! 0.0, remove floating ice in loop below. - elseif (li_mask_is_floating_ice(cellMask(iCell)) .and. config_floating_von_Mises_threshold_stress .ne. 0.0_RKIND) then - calvingVelocity(iCell) = sqrt(xvelmean(iCell)**2 + yvelmean(iCell)**2) * & - vonMisesStress(iCell) / config_floating_von_Mises_threshold_stress + elseif ( li_mask_is_floating_ice(cellMask(iCell)) .and. config_floating_von_Mises_threshold_stress .ne. 0.0_RKIND) then + calvingVelocity(iCell) = min(sqrt(xvelmean(iCell)**2 + yvelmean(iCell)**2) * & + vonMisesStress(iCell) / config_floating_von_Mises_threshold_stress, config_calving_speed_limit) endif enddo + call mpas_log_write("calling li_apply_front_ablation_velocity from von Mises stress calving routine") ! Convert calvingVelocity to calvingThickness call li_apply_front_ablation_velocity(meshPool, geometryPool,velocityPool, & From ed19e594592d013f3d374a01e6a2927f1ce486de Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 14 Jul 2021 15:09:15 -0600 Subject: [PATCH 031/301] Force a read of 'forcing' streams on init Previously, input files used as time-varying forcing failed to be read at the initial model time. This commit forces input streams with an input_interval set to a time duration to be forced to be read at the model initial time. They are read in the same we these "forcing" files are read during timestepping by using the most recent previous time level from the input file. --- .../src/mode_forward/mpas_li_core.F | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F index 2e19d2834405..48564598d690 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F @@ -112,7 +112,9 @@ function li_core_init(domain, startTimeStamp) result(err) real (kind=RKIND), pointer :: daysSinceStart integer, dimension(:), pointer :: cellProcID type (MPAS_Time_type) :: xtime_timeType, simulationStartTime_timeType - + type (MPAS_stream_list_type), pointer :: stream_cursor + type (MPAS_Alarm_type), pointer :: alarm_cursor + character (len=StrKIND) :: actualWhen err = 0 err_tmp = 0 @@ -152,6 +154,50 @@ function li_core_init(domain, startTimeStamp) result(err) err = ior(err, err_tmp) call mpas_timer_stop('reset_io_alarms') + ! + ! Read time-varying inputs, if present (i.e., forcing) + ! + ! Note: The stream manager will not read streams with input_interval set to a time interval on init + ! But in general, we want that behavior - we do not want to have to wait a whole time step (or input_interval) + ! before having forcing applied. Therefore, we must force those streams to be read here. + call mpas_timer_start('io_read', .false.) + stream_cursor => domain % streamManager % streams % head + do while (associated(stream_cursor)) + if (stream_cursor % direction == MPAS_STREAM_INPUT) then + ! Only consider streams of direction input (don't consider input/output streams) + if ((trim(stream_cursor % name) == 'input') .or. & + (trim(stream_cursor % name) == 'restart')) then + ! Don't attempt to interact with these two special streams + ! (even though restart is not type input, including to be safe) + cycle + endif + ! Only force a read of streams that are recurring. To find out if a stream is recurring, we need to + ! get its input alarm. To find the alarm, loop over the alarms in the stream manager's clock until we find it. + alarm_cursor => domain % streamManager % streamClock % alarmListHead + do while (associated(alarm_cursor)) + if (trim(alarm_cursor % alarmID) == (trim(stream_cursor % name)//'_input')) then + ! We found the stream we are looking for + if (alarm_cursor % isRecurring) then + ! Force a read of this stream, and use the latest before time available in the file + call mpas_stream_mgr_read(domain % streamManager, streamID = stream_cursor % name, rightNow = .true., & + whence = MPAS_STREAM_LATEST_BEFORE, actualWhen=actualWhen, ierr=err_tmp) + call mpas_log_write("Forced an initial read of input stream '" // stream_cursor%name //"' from time: " // & + trim(actualWhen)) + err = ior(err, err_tmp) + endif + exit ! skip the rest of this loop - we processed the alarm we were looking for + endif + alarm_cursor => alarm_cursor % next + end do + stream_cursor => stream_cursor % next + end if ! if stream direction is INPUT + end do + call mpas_timer_stop('io_read') + call mpas_timer_start('reset_io_alarms', .false.) + call mpas_stream_mgr_reset_alarms(domain % streamManager, direction=MPAS_STREAM_INPUT, ierr=err_tmp) + err = ior(err, err_tmp) + call mpas_timer_stop('reset_io_alarms') + ! === ! Initialize some time stuff on each block ! === From 6301c5680e4fe03200f7501967674e46c55fe5c5 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 15 Jul 2021 09:10:23 -0600 Subject: [PATCH 032/301] Fix a few bugs and add more info to log file The new functionality now works as expected for a simple test. --- .../src/mode_forward/mpas_li_core.F | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F index 48564598d690..245ccc8d8643 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_core.F @@ -129,9 +129,11 @@ function li_core_init(domain, startTimeStamp) result(err) err = ior(err, err_tmp) if (config_do_restart) then + call mpas_log_write("This is a restart: read stream 'restart'.") call mpas_stream_mgr_read(domain % streamManager, streamID='restart', ierr=err_tmp) err = ior(err, err_tmp) else + call mpas_log_write("This is not a restart: read stream 'input'.") call mpas_stream_mgr_read(domain % streamManager, streamID='input', ierr=err_tmp) err = ior(err, err_tmp) end if @@ -145,6 +147,7 @@ function li_core_init(domain, startTimeStamp) result(err) ! ! Read the remaining input streams ! + call mpas_log_write("Looking for other input streams set for 'initial_only'.") call mpas_timer_start('io_read', .false.) call mpas_stream_mgr_read(domain % streamManager, ierr=err_tmp) err = ior(err, err_tmp) @@ -153,6 +156,7 @@ function li_core_init(domain, startTimeStamp) result(err) call mpas_stream_mgr_reset_alarms(domain % streamManager, direction=MPAS_STREAM_INPUT, ierr=err_tmp) err = ior(err, err_tmp) call mpas_timer_stop('reset_io_alarms') + call mpas_log_write("Finished reading 'initial_only' input streams.") ! ! Read time-varying inputs, if present (i.e., forcing) @@ -161,6 +165,7 @@ function li_core_init(domain, startTimeStamp) result(err) ! But in general, we want that behavior - we do not want to have to wait a whole time step (or input_interval) ! before having forcing applied. Therefore, we must force those streams to be read here. call mpas_timer_start('io_read', .false.) + call mpas_log_write("Looking for recurring input streams (forcing) that should be forced to be read at the initial time.") stream_cursor => domain % streamManager % streams % head do while (associated(stream_cursor)) if (stream_cursor % direction == MPAS_STREAM_INPUT) then @@ -169,6 +174,7 @@ function li_core_init(domain, startTimeStamp) result(err) (trim(stream_cursor % name) == 'restart')) then ! Don't attempt to interact with these two special streams ! (even though restart is not type input, including to be safe) + stream_cursor => stream_cursor % next cycle endif ! Only force a read of streams that are recurring. To find out if a stream is recurring, we need to @@ -181,22 +187,23 @@ function li_core_init(domain, startTimeStamp) result(err) ! Force a read of this stream, and use the latest before time available in the file call mpas_stream_mgr_read(domain % streamManager, streamID = stream_cursor % name, rightNow = .true., & whence = MPAS_STREAM_LATEST_BEFORE, actualWhen=actualWhen, ierr=err_tmp) - call mpas_log_write("Forced an initial read of input stream '" // stream_cursor%name //"' from time: " // & - trim(actualWhen)) + call mpas_log_write(" * Forced an initial read of input stream '" // trim(stream_cursor%name) // & + "' from time: " // trim(actualWhen)) err = ior(err, err_tmp) endif exit ! skip the rest of this loop - we processed the alarm we were looking for endif alarm_cursor => alarm_cursor % next end do - stream_cursor => stream_cursor % next end if ! if stream direction is INPUT + stream_cursor => stream_cursor % next end do call mpas_timer_stop('io_read') call mpas_timer_start('reset_io_alarms', .false.) call mpas_stream_mgr_reset_alarms(domain % streamManager, direction=MPAS_STREAM_INPUT, ierr=err_tmp) err = ior(err, err_tmp) call mpas_timer_stop('reset_io_alarms') + call mpas_log_write("Finished processing recurring input streams at initial time.") ! === ! Initialize some time stuff on each block From 2d19d3dcbc48d716823ec17200896e007c5d024c Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 15:13:36 -0700 Subject: [PATCH 033/301] setting up namelist requirements for spmode --- components/elm/bld/ELMBuildNamelist.pm | 238 +++---- .../bld/namelist_files/namelist_defaults.xml | 629 +++++++++--------- .../namelist_files/namelist_definition.xml | 138 ++-- 3 files changed, 512 insertions(+), 493 deletions(-) diff --git a/components/elm/bld/ELMBuildNamelist.pm b/components/elm/bld/ELMBuildNamelist.pm index 19cd56e724fd..65b835dd6f17 100755 --- a/components/elm/bld/ELMBuildNamelist.pm +++ b/components/elm/bld/ELMBuildNamelist.pm @@ -22,7 +22,7 @@ # 2012-07-01 Kluzek Add some common CESM namelist options # 2013-12 Andre Refactor everything into subroutines # 2013-12 Muszala Add Ecosystem Demography functionality -# 2015-09-30 X.Shi Add pdep streams capability +# 2015-09-30 X.Shi Add pdep streams capability #-------------------------------------------------------------------------------------------- package ELMBuildNamelist; @@ -123,14 +123,14 @@ OPTIONS mode. The spinup state is saved to the restart file. - If the values match between the model and the restart - file it proceeds as directed. + If the values match between the model and the restart + file it proceeds as directed. If the restart file is in spinup mode and the model is in - normal mode, then it performs the exit spinup step - and proceeds in normal mode after that. + normal mode, then it performs the exit spinup step + and proceeds in normal mode after that. - If the restart file has normal mode and the model is in + If the restart file has normal mode and the model is in spinup, then it enters spinup. This is useful if you change a parameter and want to rapidly re-equilibrate without doing a cold start. @@ -298,7 +298,7 @@ sub process_commandline { envxml_dir => ".", vichydro => 0, maxpft => "default", - betr_mode => "default", + betr_mode => "default", methane => 0, nutrient => "default", nutrient_comp_pathway => "default", @@ -348,7 +348,7 @@ sub process_commandline { "maxpft=i" => \$opts{'maxpft'}, "v|verbose" => \$opts{'verbose'}, "version" => \$opts{'version'}, - "betr_mode=s" => \$opts{'betr_mode'}, + "betr_mode=s" => \$opts{'betr_mode'}, "methane" => \$opts{'methane'}, "nutrient=s" => \$opts{'nutrient'}, "nutrient_comp_pathway=s" => \$opts{'nutrient_comp_pathway'}, @@ -498,7 +498,7 @@ sub read_namelist_definition { sub read_envxml_case_files { # read the contents of the env*.xml files in the case directory my ($opts) = @_; - + my %envxml = (); if ( defined($opts->{'envxml_dir'}) ) { (-d $opts->{'envxml_dir'}) or fatal_error( "envxml_dir is not a directory" ); @@ -675,7 +675,7 @@ sub process_namelist_commandline_options { setup_cmdl_fates_mode($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_cmdl_bgc_spinup($opts, $nl_flags, $definition, $defaults, $nl, $cfg, $physv); setup_cmdl_vichydro($opts, $nl_flags, $definition, $defaults, $nl, $physv); - setup_cmdl_betr_mode($opts, $nl_flags, $definition, $defaults, $nl, $physv); + setup_cmdl_betr_mode($opts, $nl_flags, $definition, $defaults, $nl, $physv); } #------------------------------------------------------------------------------- @@ -782,9 +782,10 @@ sub setup_cmdl_fates_mode { # The following variables may be set by the user and are compatible with use_fates # no need to set defaults, covered in a different routine my @list = ( "fates_spitfire_mode", "use_vertsoilc", "use_century_decomp", - "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys", - "use_fates_inventory_init", "use_fates_fixed_biogeog", "fates_inventory_ctrl_filename","use_fates_logging", - "use_fates_parteh_mode","use_fates_cohort_age_tracking","use_snicar_ad"); + "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys", + "use_fates_inventory_init", "use_fates_fixed_biogeog", "use_fates_nocomp","use_fates_sp", + "fates_inventory_ctrl_filename","use_fates_logging", + "use_fates_parteh_mode","use_fates_cohort_age_tracking","use_snicar_ad"); foreach my $var ( @list ) { if ( defined($nl->get_value($var)) ) { $nl_flags->{$var} = $nl->get_value($var); @@ -802,7 +803,7 @@ sub setup_cmdl_fates_mode { } else { - + # we only dis-allow ed_spitfire with non-ed runs $var = "fates_spitfire_mode"; if ( defined($nl->get_value($var)) ) { @@ -943,7 +944,7 @@ sub setup_cmdl_bgc { $val = $opts->{$var}; $nl_flags->{'bgc_mode'} = $val; - + my $var = "bgc_mode"; if ( $nl_flags->{$var} eq "default" ) { $nl_flags->{$var} = $defaults->get_value($var); @@ -1012,7 +1013,7 @@ sub setup_cmdl_bgc { # # Now set use_cn # $var = "use_cn"; - + # Now set use_cn and use_fates foreach $var ( "use_cn", "use_fates" ) { $val = $nl_flags->{$var}; @@ -1062,15 +1063,15 @@ sub setup_cmdl_methane { $var = "use_lch4"; $val = ".true."; - + if ( defined($nl->get_value($var)) && $nl->get_value($var) ne $val ) { fatal_error("$var is inconsistent with the commandline setting of -methane"); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); @@ -1088,104 +1089,104 @@ sub setup_cmdl_nutrient { my $val = $opts->{$var}; if ($val ne "default"){ - + if ($nl_flags->{"bgc_mode"} eq "sp"){ fatal_error("-nutrient nutrient_option can ONLY be used with elm with -bgc cn|bgc|fates"); } else { - + if ($val eq "c"){ $var = "suplnitro"; $val = "'ALL'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + $var = "suplphos"; $val = "'ALL'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + } elsif ($val eq "cn") { $var = "suplnitro"; $val = "'NONE'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + $var = "suplphos"; $val = "'ALL'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + } elsif ($val eq "cnp") { $var = "suplnitro"; $val = "'NONE'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + $var = "suplphos"; $val = "'NONE'"; if ( defined($nl->get_value($var)) ) { $val = $nl->get_value($var); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + } else { fatal_error("-nutrient has a value ($val) that is not valid. Valid values are: [c, cn, cnp] \n"); } @@ -1203,54 +1204,54 @@ sub setup_cmdl_nutrient_comp { $nl_flags->{'nu_com'} = ""; if ($val ne "default"){ - + if ($nl_flags->{"bgc_mode"} eq "sp"){ fatal_error("-nutrient_comp_pathway option can ONLY be used with elm with -bgc cn|bgc|fates"); } else { - + if ($val eq "rd"){ $var = "nu_com"; $val = "'RD'"; - + if ( defined($nl->get_value($var)) && $nl->get_value($var) ne $val ) { fatal_error("$var is inconsistent with the commandline setting of -nutrient_comp_pathway"); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + $nl_flags->{$var} = 'RD'; - + } elsif ($val eq "eca") { $var = "nu_com"; $val = "'ECA'"; - + if ( defined($nl->get_value($var)) && $nl->get_value($var) ne $val ) { fatal_error("$var is inconsistent with the commandline setting of -nutrient_comp_pathway"); } - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + $nl_flags->{$var} = 'ECA'; - + $var = "nfix_ptase_plant"; $val = '.true.'; $nl->set_variable_value($group, $var, $val); - + } else { fatal_error("-nutrient_comp_pathway has a value ($val) that is not valid. Valid values are: [rd, eca] \n"); } @@ -1266,38 +1267,38 @@ sub setup_cmdl_soil_decomp { my $val = $opts->{$var}; if ($val ne "default"){ - + if ($nl_flags->{"bgc_mode"} eq "sp"){ fatal_error("-soil_decomp option can ONLY be used with elm with -bgc cn|bgc|fates"); } else { - + if ($val eq "ctc"){ $var = "use_century_decomp"; $val = ".false."; - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + } elsif ($val eq "century") { $var = "use_century_decomp"; $val = ".true."; - + my $group = $definition->get_group_name($var); $nl_flags->{$var} = $val; $nl->set_variable_value($group, $var, $val); - + if ( ! $definition->is_valid_value( $var, $val ) ) { my @valid_values = $definition->get_valid_values( $var ); fatal_error("$var has a value ($val) that is NOT valid. Valid values are: @valid_values\n"); } - + } else { fatal_error("-soil_decomp has a value ($val) that is not valid. Valid values are: [rd, eca] \n"); } @@ -1328,7 +1329,7 @@ sub setup_cmdl_crop { "** * by the command-line options -bgc cn\n" . "** * by a default configuration file, specified by -defaults\n"); } - + $var = "use_crop"; $val = ".false."; if ($nl_flags->{'crop'} eq 1) { @@ -1358,7 +1359,7 @@ sub setup_cmdl_maxpft { $val = $maxpatchpft{$nl_flags->{'use_crop'}}; } $nl_flags->{'maxpft'} = $val; - + if ( ($nl_flags->{'bgc_mode'} ne "sp") && ($nl_flags->{'maxpft'} != $maxpatchpft{$nl_flags->{'use_crop'}}) ) { fatal_error("** For CN or BGC mode you MUST set max patch PFT's to $maxpatchpft{$nl_flags->{'use_crop'}}\n" . "**\n" . @@ -1381,7 +1382,7 @@ sub setup_cmdl_maxpft { "NOT validated / scientifically supported.\n"); } verbose_message("Using $nl_flags->{'maxpft'} for maxpft."); - + $var = "maxpatch_pft"; $val = $nl_flags->{'maxpft'}; my $group = $definition->get_group_name($var); @@ -1590,7 +1591,7 @@ sub setup_cmdl_dynamic_vegetation { "** Set the bgc mode to 'cn' or 'bgc' by the following means from highest to lowest precedence:\n" . "** * by the command-line options -bgc cn\n"); } - + $var = "use_cndv"; $nl_flags->{$var} = ".false."; if ($nl_flags->{'dynamic_vegetation'} eq 1) { @@ -1619,11 +1620,11 @@ sub setup_cmdl_vichydro { my $var = "vichydro"; $val = $opts->{$var}; $nl_flags->{'vichydro'} = $val; - + if ($nl_flags->{'vichydro'} eq 1) { message("Using VIC hydrology for runoff calculations."); } - + $var = "use_vichydro"; $val = $nl->get_value($var); if ($nl_flags->{'vichydro'} eq 1) { @@ -1841,12 +1842,12 @@ sub process_namelist_inline_logic { setup_logic_surface_dataset($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_initial_conditions($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_dynamic_subgrid($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); - + # setup_logic_snowpack($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_fates($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_bgc_spinup($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_supplemental_nitrogen($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); - + ######################################### # namelist group: clm_humanindex_inparm # ######################################### @@ -1872,7 +1873,7 @@ sub process_namelist_inline_logic { # namelist group: ndepdyn_nml # ############################### setup_logic_nitrogen_deposition($opts->{'test'}, $nl_flags, $definition, $defaults, $nl, $physv); - + ############################### # namelist group: pdepdyn_nml # ############################### @@ -1924,7 +1925,7 @@ sub setup_logic_site_specific { my $group = $definition->get_group_name($var); $nl->set_variable_value($group, $var, $val); } - + # res check prevents polluting the namelist with an unnecessary # false variable for every run if ($nl_flags->{'res'} eq "1x1_mexicocityMEX") { @@ -2138,7 +2139,7 @@ sub setup_logic_glacier { if (lc($nl->get_value($var)) ne lc($val)) { fatal_error("glc_do_dynglacier can only be set via the env variable $clm_upvar: it can NOT be set in user_nl_clm\n"); } - + my $var = "maxpatch_glcmec"; add_default($opts->{'test'}, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'val'=>$nl_flags->{'glc_nec'} ); @@ -2482,7 +2483,7 @@ sub setup_logic_dynamic_subgrid { setup_logic_do_transient_pfts($test_files, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_do_transient_crops($test_files, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_do_harvest($test_files, $nl_flags, $definition, $defaults, $nl, $physv); - + } #------------------------------------------------------------------------------- @@ -2495,24 +2496,24 @@ sub setup_logic_do_transient_pfts { # - flanduse_timeseries # - use_cndv # - use_fates - # + # my ($test_files, $nl_flags, $definition, $defaults, $nl, $physv) = @_; my $var = 'do_transient_pfts'; - + # Start by assuming a default value of '.true.'. Then check a number of # conditions under which do_transient_pfts cannot be true. Under these # conditions: (1) set default value to '.false.'; (2) make sure that the # value is indeed false (e.g., that the user didn't try to set it to true). - + my $default_val = ".true."; - + # cannot_be_true will be set to a non-empty string in any case where # do_transient_pfts should not be true; if it turns out that # do_transient_pfts IS true in any of these cases, a fatal error will be # generated my $cannot_be_true = ""; - + if (string_is_undef_or_empty($nl->get_value('flanduse_timeseries'))) { $cannot_be_true = "$var can only be set to true when running a transient case (flanduse_timeseries non-blank)"; } @@ -2522,21 +2523,21 @@ sub setup_logic_do_transient_pfts { elsif (value_is_true($nl->get_value('use_fates'))) { $cannot_be_true = "$var cannot be combined with use_fates"; } - + if ($cannot_be_true) { $default_val = ".false."; } - + if (!$cannot_be_true) { # Note that, if the variable cannot be true, we don't call add_default # - so that we don't clutter up the namelist with variables that don't # matter for this case add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, val=>$default_val); } - + # Make sure the value is false when it needs to be false - i.e., that the # user hasn't tried to set a true value at an inappropriate time. - + if (value_is_true($nl->get_value($var)) && $cannot_be_true) { fatal_error($cannot_be_true); } @@ -2553,7 +2554,7 @@ sub setup_logic_do_transient_crops { # - flanduse_timeseries # - use_crop # - use_fates - # + # my ($test_files, $nl_flags, $definition, $defaults, $nl, $physv) = @_; my $var = 'do_transient_crops'; @@ -2562,48 +2563,48 @@ sub setup_logic_do_transient_crops { # conditions under which do_transient_crops cannot be true. Under these # conditions: (1) set default value to '.false.'; (2) make sure that the # value is indeed false (e.g., that the user didn't try to set it to true). - + my $default_val = ".true."; - + # cannot_be_true will be set to a non-empty string in any case where # do_transient_crops should not be true; if it turns out that # do_transient_crops IS true in any of these cases, a fatal error will be # generated my $cannot_be_true = ""; - + if (string_is_undef_or_empty($nl->get_value('flanduse_timeseries'))) { $cannot_be_true = "$var can only be set to true when running a transient case (flanduse_timeseries non-blank)"; } - + elsif (!value_is_true($nl->get_value("irrigate"))) { $cannot_be_true = "$var should be set to true when running with irrigate = true"; } - + elsif (value_is_true($nl->get_value('use_fates'))) { # In principle, use_fates should be compatible with # do_transient_crops. However, this hasn't been tested, so to be safe, # we are not allowing this combination for now. $cannot_be_true = "$var has not been tested with FATES, so for now these two options cannot be combined"; } - + if ($cannot_be_true) { $default_val = ".false."; } - + if (!$cannot_be_true) { # Note that, if the variable cannot be true, we don't call add_default # - so that we don't clutter up the namelist with variables that don't # matter for this case add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, val=>$default_val); } - + # Make sure the value is false when it needs to be false - i.e., that the # user hasn't tried to set a true value at an inappropriate time. - + if (value_is_true($nl->get_value($var)) && $cannot_be_true) { fatal_error($cannot_be_true); } - + } #------------------------------------------------------------------------------- @@ -2618,21 +2619,21 @@ sub setup_logic_do_harvest { # - use_fates # my ($test_files, $nl_flags, $definition, $defaults, $nl, $physv) = @_; - + my $var = 'do_harvest'; # Start by assuming a default value of '.true.'. Then check a number of # conditions under which do_harvest cannot be true. Under these # conditions: (1) set default value to '.false.'; (2) make sure that the # value is indeed false (e.g., that the user didn't try to set it to true). - + my $default_val = ".true."; - + # cannot_be_true will be set to a non-empty string in any case where # do_harvest should not be true; if it turns out that do_harvest IS true # in any of these cases, a fatal error will be generated my $cannot_be_true = ""; - + if (string_is_undef_or_empty($nl->get_value('flanduse_timeseries'))) { $cannot_be_true = "$var can only be set to true when running a transient case (flanduse_timeseries non-blank)"; } @@ -2642,25 +2643,25 @@ sub setup_logic_do_harvest { elsif (value_is_true($nl->get_value('use_fates'))) { $cannot_be_true = "$var currently doesn't work with FATES"; } - + if ($cannot_be_true) { $default_val = ".false."; } - + if (!$cannot_be_true) { # Note that, if the variable cannot be true, we don't call add_default # - so that we don't clutter up the namelist with variables that don't # matter for this case add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, val=>$default_val); } - + # Make sure the value is false when it needs to be false - i.e., that the # user hasn't tried to set a true value at an inappropriate time. - + if (value_is_true($nl->get_value($var)) && $cannot_be_true) { fatal_error($cannot_be_true); } - + } #------------------------------------------------------------------------------- @@ -2672,7 +2673,7 @@ sub setup_logic_bgc_spinup { # only set bgc_spinup state if CN is on. add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'spinup_state', 'bgc_spinup'=>$nl_flags->{'bgc_spinup'} ); } - + if ( $nl_flags->{'bgc_mode'} eq "sp" && defined($nl->get_value('override_bgc_restart_mismatch_dump'))) { fatal_error("CN must be on if override_bgc_restart_mismatch_dump is set.\n"); } @@ -2702,7 +2703,7 @@ sub setup_logic_supplemental_nitrogen { if ( $nl_flags->{'use_crop'} ne ".true." && $suplnitro =~ /PROG_CROP_ONLY/i ) { fatal_error("supplemental Nitrogen is set to run over prognostic crops, but prognostic crop is NOT active!\n"); } - + if ( $suplnitro =~ /ALL/i ) { if ( $nl_flags->{'bgc_spinup'} ne "off" ) { warning("There is no need to use a bgc_spinup mode when supplemental Nitrogen is on for all PFT's, as these modes spinup Nitrogen\n"); @@ -2942,7 +2943,7 @@ sub setup_logic_popd_streams { 'use_cn'=>$nl_flags->{'use_cn'}, 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); # Set align year, if first and last years are different - if ( $nl->get_value('stream_year_first_popdens') != + if ( $nl->get_value('stream_year_first_popdens') != $nl->get_value('stream_year_last_popdens') ) { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'model_year_align_popdens', 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); @@ -2980,7 +2981,7 @@ sub setup_logic_lightning_streams { 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); # Set align year, if first and last years are different - if ( $nl->get_value('stream_year_first_lightng') != + if ( $nl->get_value('stream_year_first_lightng') != $nl->get_value('stream_year_last_lightng') ) { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'model_year_align_lightng', 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); @@ -3024,7 +3025,7 @@ sub setup_logic_megan { if ($opts->{'megan'} ) { if ( value_is_true( $nl_flags->{'use_fates'} ) ) { - fatal_error("MEGAN can NOT be on when FATES is also on.\n" . + fatal_error("MEGAN can NOT be on when FATES is also on.\n" . " Use the '-no-megan' option when '-bgc fates' is activated"); } add_default($opts->{'test'}, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'megan_specifier'); @@ -3049,18 +3050,18 @@ sub setup_logic_lai_streams { fatal_error("turning use_lai_streams on is incompatable with use_crop set to true."); } if ( $nl_flags->{'bgc_mode'} eq "sp" ) { - + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_lai_streams'); - add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'lai_mapalgo', + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'lai_mapalgo', 'hgrid'=>$nl_flags->{'res'} ); - add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_first_lai', + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_first_lai', 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); - add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_last_lai', + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'stream_year_last_lai', 'sim_year'=>$nl_flags->{'sim_year'}, 'sim_year_range'=>$nl_flags->{'sim_year_range'}); # Set align year, if first and last years are different - if ( $nl->get_value('stream_year_first_lai') != + if ( $nl->get_value('stream_year_first_lai') != $nl->get_value('stream_year_last_lai') ) { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'model_year_align_lai', 'sim_year'=>$nl_flags->{'sim_year'}, @@ -3076,7 +3077,7 @@ sub setup_logic_lai_streams { defined($nl->get_value('stream_fldfilename_lai')) ) { fatal_error("When bgc is NOT SP none of the following can be set: stream_year_first_lai,\n" . "stream_year_last_lai, model_year_align_lai, nor\n" . - "stream_fldfilename_lai (eg. don't use this option with BGC,CN,CNDV nor BGDCV).\n"); + "stream_fldfilename_lai (eg. don't use this option with BGC,CN,CNDV nor BGDCV).\n"); } } } @@ -3101,7 +3102,7 @@ sub setup_logic_pflotran { # my ($test_files, $nl_flags, $definition, $defaults, $nl, $physv) = @_; - + if ( $nl_flags->{'use_pflotran'} eq '.true.' ) { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'pflotran_inputdir' ); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'pflotran_prefix' ); @@ -3132,7 +3133,10 @@ sub setup_logic_fates { add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_inventory_init', 'use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_inventory_ctrl_filename','use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_cohort_age_tracking','use_fates'=>$nl_flags->{'use_fates'}); + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_sp', 'use_fates'=>$nl_flags->{'use_fates'}); + add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_nocomp', 'use_fates'=>$nl_flags->{'use_fates'}); add_default($test_files, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_paramfile', 'phys'=>$nl_flags->{'phys'}); + } } @@ -3156,10 +3160,10 @@ sub write_output_files { # CLM component my @groups; { - @groups = qw(elm_inparm ndepdyn_nml pdepdyn_nml popd_streams light_streams lai_streams elm_canopyhydrology_inparm - elm_soilhydrology_inparm dynamic_subgrid finidat_consistency_checks dynpft_consistency_checks + @groups = qw(elm_inparm ndepdyn_nml pdepdyn_nml popd_streams light_streams lai_streams elm_canopyhydrology_inparm + elm_soilhydrology_inparm dynamic_subgrid finidat_consistency_checks dynpft_consistency_checks elmu_inparm elm_soilstate_inparm elm_pflotran_inparm betr_inparm); - #@groups = qw(elm_inparm elm_canopyhydrology_inparm elm_soilhydrology_inparm + #@groups = qw(elm_inparm elm_canopyhydrology_inparm elm_soilhydrology_inparm # finidat_consistency_checks dynpft_consistency_checks); # Eventually only list namelists that are actually used when CN on #if ( $nl_flags->{'bgc_mode'} eq "cn" ) { @@ -3460,7 +3464,7 @@ sub check_use_case_name { "in namelist_files/use_cases/README\n"; my $desc = "[a-zA-Z0-9]*"; my $rcp = "rcp[0-9\.]+"; - my $rcp = "(rcp|SSP)[0-9\.]+"; + my $rcp = "(rcp|SSP)[0-9\.]+"; if ( $use_case =~ /^[0-9]+-[0-9]+([a-zA-Z0-9_\.]*)_transient$/ ) { my $string = $1; if ( $string =~ /^_($rcp)_*($desc)$/ ) { diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index c18d2192c86a..19f6b1210638 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -121,7 +121,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/paramdata/fates_params_default_12pft_api16.c210527.nc -lnd/clm2/paramdata/CNP_parameters_c131108.nc +lnd/clm2/paramdata/CNP_parameters_c131108.nc lnd/clm2/paramdata/CNP_parameters_c180529.nc lnd/clm2/paramdata/CNP_parameters_c180312.nc' @@ -410,9 +410,9 @@ lnd/clm2/surfdata_map/surfdata_twpx4v1_simyr2000_c170706.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_rcp8.5_simyr1850-2100_c140610.nc -lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_simyr1850-2015_c180306.nc - DUMMY lnd/clm2/surfdata_map/surfdata.pftdyn_ne16np4_hist_simyr1850-2005_c160803.nc @@ -445,11 +445,11 @@ lnd/clm2/surfdata_map/surfdata_twpx4v1_simyr2000_c170706.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_rcp8.5_simyr1850-2100_c140610.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_360x720cru_rcp6.0_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_0.9x1.25_rcp6.0_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_1.9x2.5_rcp6.0_simyr1850-2100_c130524.nc lnd/clm2/surfdata_map/surfdata.pftdyn_10x15_rcp6.0_simyr1850-2100_c130524.nc @@ -458,11 +458,11 @@ lnd/clm2/surfdata_map/surfdata_twpx4v1_simyr2000_c170706.nc lnd/clm2/surfdata_map/surfdata.pftdyn_ne30np4_rcp6.0_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_360x720cru_rcp4.5_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_0.9x1.25_rcp4.5_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_1.9x2.5_rcp4.5_simyr1850-2100_c130524.nc lnd/clm2/surfdata_map/surfdata.pftdyn_10x15_rcp4.5_simyr1850-2100_c130524.nc @@ -471,11 +471,11 @@ lnd/clm2/surfdata_map/surfdata_twpx4v1_simyr2000_c170706.nc lnd/clm2/surfdata_map/surfdata.pftdyn_ne30np4_rcp4.5_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_360x720cru_rcp2.6_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_0.9x1.25_rcp2.6_simyr1850-2100_c130524.nc -lnd/clm2/surfdata_map/surfdata.pftdyn_1.9x2.5_rcp2.6_simyr1850-2100_c130524.nc lnd/clm2/surfdata_map/surfdata.pftdyn_10x15_rcp2.6_simyr1850-2100_c130524.nc @@ -730,361 +730,361 @@ this mask will have smb calculated over the entire global land surface - + -lnd/clm2/mappingdata/maps/0.1x0.1/map_0.1x0.1_nomask_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_0.5x0.5_AVHRR_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_0.5x0.5_MODIS_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_0.5x0.5_nomask_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_10x10min_nomask_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_3x3min_MODIS_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_5x5min_IGBP-GSDP_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_5x5min_ISRIC-WISE_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_5x5min_nomask_to_0.1x0.1_nomask_aave_da_c120406.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_ne120np4_nomask_to_0.1x0.1_nomask_aave_da_c120711.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_4x5_nomask_to_0.1x0.1_nomask_aave_da_c120706.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_1.9x2.5_nomask_to_0.1x0.1_nomask_aave_da_c120709.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_ne240np4_nomask_to_0.1x0.1_nomask_aave_da_c120711.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_0.9x1.25_GRDC_to_0.1x0.1_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_360x720_cruncep_to_0.1x0.1_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/0.1x0.1/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.1x0.1_nomask_aave_da_c130405.nc - + - + -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_0.5x0.5_AVHRR_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_0.5x0.5_MODIS_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_0.5x0.5_nomask_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_10x10min_nomask_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_3x3min_MODIS_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_3x3min_USGS_to_1x1_asphaltjungleNJ_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_3x3min_LandScan2004_to_1x1_asphaltjungleNJ_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_5x5min_IGBP-GSDP_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_5x5min_ISRIC-WISE_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_5x5min_nomask_to_1x1_asphaltjungleNJ_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_3x3min_GLOBE-Gardner_to_1x1_asphaltjungleNJ_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_asphaltjungleNJ_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_0.9x1.25_GRDC_to_1x1_asphaltjungleNJ_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_360x720_cruncep_to_1x1_asphaltjungleNJ_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_asphaltjungleNJ/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_asphaltjungleNJ_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_AVHRR_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_MODIS_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_MODIS_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_USGS_to_1x1_brazil_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_LandScan2004_to_1x1_brazil_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_IGBP-GSDP_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_ISRIC-WISE_to_1x1_brazil_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_GLOBE-Gardner_to_1x1_brazil_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_brazil_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_GRDC_to_1x1_brazil_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_360x720_cruncep_to_1x1_brazil_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_0.5x0.5_AVHRR_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_0.5x0.5_MODIS_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_0.5x0.5_nomask_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_10x10min_nomask_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_3x3min_MODIS_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_3x3min_USGS_to_1x1_camdenNJ_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_3x3min_LandScan2004_to_1x1_camdenNJ_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_5x5min_IGBP-GSDP_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_5x5min_ISRIC-WISE_to_1x1_camdenNJ_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_5x5min_nomask_to_1x1_camdenNJ_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_3x3min_GLOBE-Gardner_to_1x1_camdenNJ_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_camdenNJ_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_0.9x1.25_GRDC_to_1x1_camdenNJ_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_360x720_cruncep_to_1x1_camdenNJ_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_camdenNJ/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_camdenNJ_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_0.5x0.5_AVHRR_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_0.5x0.5_MODIS_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_0.5x0.5_nomask_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_10x10min_nomask_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_3x3min_MODIS_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_3x3min_USGS_to_1x1_mexicocityMEX_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_3x3min_LandScan2004_to_1x1_mexicocityMEX_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_5x5min_IGBP-GSDP_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_5x5min_ISRIC-WISE_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_5x5min_nomask_to_1x1_mexicocityMEX_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_3x3min_GLOBE-Gardner_to_1x1_mexicocityMEX_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_mexicocityMEX_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_0.9x1.25_GRDC_to_1x1_mexicocityMEX_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_360x720_cruncep_to_1x1_mexicocityMEX_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_mexicocityMEX/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_mexicocityMEX_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_numaIA/map_0.5x0.5_AVHRR_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_0.5x0.5_MODIS_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_0.5x0.5_nomask_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_10x10min_nomask_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_3x3min_MODIS_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_3x3min_USGS_to_1x1_numaIA_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_3x3min_LandScan2004_to_1x1_numaIA_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_5x5min_IGBP-GSDP_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_5x5min_ISRIC-WISE_to_1x1_numaIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_5x5min_nomask_to_1x1_numaIA_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_numaIA/map_3x3min_GLOBE-Gardner_to_1x1_numaIA_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_numaIA/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_numaIA_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_0.9x1.25_GRDC_to_1x1_numaIA_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_360x720_cruncep_to_1x1_numaIA_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_numaIA/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_numaIA_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_0.5x0.5_AVHRR_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_0.5x0.5_MODIS_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_0.5x0.5_nomask_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_10x10min_nomask_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_3x3min_MODIS_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_3x3min_USGS_to_1x1_smallvilleIA_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_3x3min_LandScan2004_to_1x1_smallvilleIA_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_5x5min_IGBP-GSDP_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_5x5min_ISRIC-WISE_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_5x5min_nomask_to_1x1_smallvilleIA_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_3x3min_GLOBE-Gardner_to_1x1_smallvilleIA_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_smallvilleIA_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_0.9x1.25_GRDC_to_1x1_smallvilleIA_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_360x720_cruncep_to_1x1_smallvilleIA_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_smallvilleIA/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_smallvilleIA_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_0.5x0.5_AVHRR_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_0.5x0.5_MODIS_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_0.5x0.5_nomask_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_10x10min_nomask_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_3x3min_MODIS_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_3x3min_USGS_to_1x1_tropicAtl_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_3x3min_LandScan2004_to_1x1_tropicAtl_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_5x5min_IGBP-GSDP_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_5x5min_ISRIC-WISE_to_1x1_tropicAtl_nomask_aave_da_c120718.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_5x5min_nomask_to_1x1_tropicAtl_nomask_aave_da_c120718.nc lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_3x3min_GLOBE-Gardner_to_1x1_tropicAtl_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_tropicAtl_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_0.9x1.25_GRDC_to_1x1_tropicAtl_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_360x720_cruncep_to_1x1_tropicAtl_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_tropicAtl/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_tropicAtl_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_0.5x0.5_AVHRR_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_0.5x0.5_MODIS_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_0.5x0.5_nomask_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_10x10min_nomask_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_3x3min_MODIS_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_3x3min_USGS_to_1x1_urbanc_alpha_nomask_aave_da_c120928.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_3x3min_LandScan2004_to_1x1_urbanc_alpha_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_5x5min_IGBP-GSDP_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_5x5min_ISRIC-WISE_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_5x5min_nomask_to_1x1_urbanc_alpha_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_3x3min_GLOBE-Gardner_to_1x1_urbanc_alpha_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_urbanc_alpha_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_0.9x1.25_GRDC_to_1x1_urbanc_alpha_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_360x720_cruncep_to_1x1_urbanc_alpha_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_urbanc_alpha/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_urbanc_alpha_nomask_aave_da_c130403.nc - + - + -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_0.5x0.5_AVHRR_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_0.5x0.5_MODIS_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_0.5x0.5_nomask_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_10x10min_nomask_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_3x3min_MODIS_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_3x3min_USGS_to_1x1_vancouverCAN_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_3x3min_LandScan2004_to_1x1_vancouverCAN_nomask_aave_da_c121114.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_5x5min_IGBP-GSDP_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_5x5min_ISRIC-WISE_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_5x5min_nomask_to_1x1_vancouverCAN_nomask_aave_da_c120717.nc lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_3x3min_GLOBE-Gardner_to_1x1_vancouverCAN_nomask_aave_da_c120927.nc lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_3x3min_GLOBE-Gardner-mergeGIS_to_1x1_vancouverCAN_nomask_aave_da_c120927.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_0.9x1.25_GRDC_to_1x1_vancouverCAN_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_360x720_cruncep_to_1x1_vancouverCAN_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1x1_vancouverCAN/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_vancouverCAN_nomask_aave_da_c130403.nc - + -lnd/clm2/mappingdata/maps/0.9x1.25/map_0.5x0.5_landuse_to_0.9x1.25_aave_da_110307.nc -lnd/clm2/mappingdata/maps/0.9x1.25/map_0.5x0.5_lanwat_to_0.9x1.25_aave_da_110307.nc lnd/clm2/mappingdata/maps/0.9x1.25/map_10minx10min_topo_to_0.9x1.25_aave_da_110630.nc @@ -1104,18 +1104,18 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/0.9x1.25/map_3x3min_GLOBE-Gardner_to_0.9x1.25_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/0.9x1.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.9x1.25_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/0.9x1.25/map_0.9x1.25_GRDC_to_0.9x1.25_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/0.9x1.25/map_360x720_cruncep_to_0.9x1.25_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/0.9x1.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.9x1.25_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/0.9x1.25/map_0.5x0.5_GSDTG2000_to_0.9x1.25_nomask_aave_da_c161010.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_0.5x0.5_landuse_to_1.9x2.5_aave_da_110307.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_0.5x0.5_lanwat_to_1.9x2.5_aave_da_110307.nc lnd/clm2/mappingdata/maps/1.9x2.5/map_10minx10min_topo_to_1.9x2.5_aave_da_110307.nc @@ -1135,19 +1135,19 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/1.9x2.5/map_3x3min_GLOBE-Gardner_to_1.9x2.5_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/1.9x2.5/map_3x3min_GLOBE-Gardner-mergeGIS_to_1.9x2.5_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_0.9x1.25_GRDC_to_1.9x2.5_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_360x720_cruncep_to_1.9x2.5_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1.9x2.5_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/1.9x2.5/map_0.5x0.5_GSDTG2000_to_1.9x2.5_nomask_aave_da_c161010.nc -lnd/clm2/mappingdata/maps/10x15/map_0.5x0.5_landuse_to_10x15_aave_da_110307.nc -lnd/clm2/mappingdata/maps/10x15/map_0.5x0.5_lanwat_to_10x15_aave_da_110307.nc lnd/clm2/mappingdata/maps/10x15/map_10minx10min_topo_to_10x15_aave_da_110307.nc @@ -1167,16 +1167,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/10x15/map_3x3min_GLOBE-Gardner_to_10x15_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/10x15/map_3x3min_GLOBE-Gardner-mergeGIS_to_10x15_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/10x15/map_0.9x1.25_GRDC_to_10x15_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/10x15/map_360x720_cruncep_to_10x15_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/10x15/map_1km-merge-10min_HYDRO1K-merge-nomask_to_10x15_nomask_aave_da_c130411.nc -lnd/clm2/mappingdata/maps/360x720/map_0.5x0.5_MODIS_to_360x720_nomask_aave_da_c120830.nc -lnd/clm2/mappingdata/maps/360x720/map_0.5x0.5_AVHRR_to_360x720_nomask_aave_da_c120830.nc lnd/clm2/mappingdata/maps/360x720/map_10x10min_nomask_to_360x720_nomask_aave_da_c120830.nc @@ -1196,18 +1196,18 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/360x720/map_3x3min_GLOBE-Gardner_to_360x720_nomask_aave_da_c121128.nc lnd/clm2/mappingdata/maps/360x720/map_3x3min_GLOBE-Gardner-mergeGIS_to_360x720_nomask_aave_da_c121128.nc -lnd/clm2/mappingdata/maps/360x720/map_0.9x1.25_GRDC_to_360x720_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/360x720/map_360x720_cruncep_to_360x720_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/360x720/map_1km-merge-10min_HYDRO1K-merge-nomask_to_360x720_nomask_aave_da_c130403.nc -lnd/clm2/mappingdata/maps/360x720/map_0.5x0.5_GSDTG2000_to_360x720_nomask_aave_da_c170913.nc -lnd/clm2/mappingdata/maps/512x1024/map_0.5x0.5_MODIS_to_512x1024_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/512x1024/map_0.5x0.5_AVHRR_to_512x1024_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/512x1024/map_10x10min_nomask_to_512x1024_nomask_aave_da_c110920.nc @@ -1227,17 +1227,17 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/512x1024/map_3x3min_GLOBE-Gardner_to_512x1024_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/512x1024/map_3x3min_GLOBE-Gardner-mergeGIS_to_512x1024_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/512x1024/map_0.9x1.25_GRDC_to_512x1024_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/512x1024/map_360x720_cruncep_to_512x1024_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/512x1024/map_1km-merge-10min_HYDRO1K-merge-nomask_to_512x1024_nomask_aave_da_c130403.nc -lnd/clm2/mappingdata/maps/128x256/map_0.5x0.5_MODIS_to_128x256_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/128x256/map_0.5x0.5_AVHRR_to_128x256_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/128x256/map_10x10min_nomask_to_128x256_nomask_aave_da_c110920.nc @@ -1257,17 +1257,17 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/128x256/map_3x3min_GLOBE-Gardner_to_128x256_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/128x256/map_3x3min_GLOBE-Gardner-mergeGIS_to_128x256_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/128x256/map_0.9x1.25_GRDC_to_128x256_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/128x256/map_360x720_cruncep_to_128x256_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/128x256/map_1km-merge-10min_HYDRO1K-merge-nomask_to_128x256_nomask_aave_da_c130403.nc -lnd/clm2/mappingdata/maps/64x128/map_0.5x0.5_MODIS_to_64x128_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/64x128/map_0.5x0.5_AVHRR_to_64x128_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/64x128/map_10x10min_nomask_to_64x128_nomask_aave_da_c110920.nc @@ -1287,16 +1287,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/64x128/map_3x3min_GLOBE-Gardner_to_64x128_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/64x128/map_3x3min_GLOBE-Gardner-mergeGIS_to_64x128_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/64x128/map_0.9x1.25_GRDC_to_64x128_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/64x128/map_360x720_cruncep_to_64x128_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/64x128/map_1km-merge-10min_HYDRO1K-merge-nomask_to_64x128_nomask_aave_da_c130403.nc -lnd/clm2/mappingdata/maps/48x96/map_0.5x0.5_MODIS_to_48x96_nomask_aave_da_c110822.nc -lnd/clm2/mappingdata/maps/48x96/map_0.5x0.5_AVHRR_to_48x96_nomask_aave_da_c110822.nc lnd/clm2/mappingdata/maps/48x96/map_10x10min_nomask_to_48x96_nomask_aave_da_c110822.nc @@ -1316,16 +1316,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/48x96/map_3x3min_GLOBE-Gardner_to_48x96_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/48x96/map_3x3min_GLOBE-Gardner-mergeGIS_to_48x96_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/48x96/map_0.9x1.25_GRDC_to_48x96_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/48x96/map_360x720_cruncep_to_48x96_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/48x96/map_1km-merge-10min_HYDRO1K-merge-nomask_to_48x96_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/32x64/map_0.5x0.5_MODIS_to_32x64_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/32x64/map_0.5x0.5_AVHRR_to_32x64_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/32x64/map_10x10min_nomask_to_32x64_nomask_aave_da_c110920.nc @@ -1345,16 +1345,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/32x64/map_3x3min_GLOBE-Gardner_to_32x64_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/32x64/map_3x3min_GLOBE-Gardner-mergeGIS_to_32x64_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/32x64/map_0.9x1.25_GRDC_to_32x64_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/32x64/map_360x720_cruncep_to_32x64_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/32x64/map_1km-merge-10min_HYDRO1K-merge-nomask_to_32x64_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/8x16/map_0.5x0.5_MODIS_to_8x16_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/8x16/map_0.5x0.5_AVHRR_to_8x16_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/8x16/map_10x10min_nomask_to_8x16_nomask_aave_da_c110920.nc @@ -1374,16 +1374,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/8x16/map_3x3min_GLOBE-Gardner_to_8x16_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/8x16/map_3x3min_GLOBE-Gardner-mergeGIS_to_8x16_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/8x16/map_0.9x1.25_GRDC_to_8x16_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/8x16/map_360x720_cruncep_to_8x16_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/8x16/map_1km-merge-10min_HYDRO1K-merge-nomask_to_8x16_nomask_aave_da_c130411.nc -lnd/clm2/mappingdata/maps/4x5/map_0.5x0.5_MODIS_to_4x5_nomask_aave_da_c110822.nc -lnd/clm2/mappingdata/maps/4x5/map_0.5x0.5_AVHRR_to_4x5_nomask_aave_da_c110822.nc lnd/clm2/mappingdata/maps/4x5/map_10x10min_nomask_to_4x5_nomask_aave_da_c110822.nc @@ -1403,16 +1403,16 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/4x5/map_3x3min_GLOBE-Gardner_to_4x5_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/4x5/map_3x3min_GLOBE-Gardner-mergeGIS_to_4x5_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/4x5/map_0.9x1.25_GRDC_to_4x5_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/4x5/map_360x720_cruncep_to_4x5_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/4x5/map_1km-merge-10min_HYDRO1K-merge-nomask_to_4x5_nomask_aave_da_c130411.nc -lnd/clm2/mappingdata/maps/0.23x0.31/map_0.5x0.5_MODIS_to_0.23x0.31_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/0.23x0.31/map_0.5x0.5_AVHRR_to_0.23x0.31_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/0.23x0.31/map_10x10min_nomask_to_0.23x0.31_nomask_aave_da_c110920.nc @@ -1434,17 +1434,17 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/0.23x0.31/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.23x0.31_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/0.23x0.31/map_3x3min_LandScan2004_to_0.23x0.31_nomask_aave_da_c120518.nc -lnd/clm2/mappingdata/maps/0.23x0.31/map_0.9x1.25_GRDC_to_0.23x0.31_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/0.23x0.31/map_360x720_cruncep_to_0.23x0.31_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/0.23x0.31/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.23x0.31_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/2.5x3.33/map_0.5x0.5_MODIS_to_2.5x3.33_nomask_aave_da_c110823.nc -lnd/clm2/mappingdata/maps/2.5x3.33/map_0.5x0.5_AVHRR_to_2.5x3.33_nomask_aave_da_c110823.nc lnd/clm2/mappingdata/maps/2.5x3.33/map_10x10min_nomask_to_2.5x3.33_nomask_aave_da_c110823.nc @@ -1464,66 +1464,66 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/2.5x3.33/map_3x3min_GLOBE-Gardner_to_2.5x3.33_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/2.5x3.33/map_3x3min_GLOBE-Gardner-mergeGIS_to_2.5x3.33_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/2.5x3.33/map_0.9x1.25_GRDC_to_2.5x3.33_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/2.5x3.33/map_360x720_cruncep_to_2.5x3.33_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/2.5x3.33/map_1km-merge-10min_HYDRO1K-merge-nomask_to_2.5x3.33_nomask_aave_da_c130405.nc - + -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.5x0.5_AVHRR_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.5x0.5_MODIS_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.5x0.5_GSDTG2000_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.5x0.5_nomask_to_0.5x0.5_nomask_aave_da_c111021.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_10x10min_IGBPmergeICESatGIS_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_10x10min_nomask_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_5x5min_IGBP-GSDP_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_5x5min_nomask_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_MODIS_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_5x5min_ISRIC-WISE_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_LandScan2004_to_0.5x0.5_nomask_aave_da_c190417.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_GLOBE-Gardner_to_0.5x0.5_nomask_aave_da_c190417.nc lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.1x0.1_nomask_to_0.5x0.5_nomask_aave_da_c120706.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_ne240np4_nomask_to_0.5x0.5_nomask_aave_da_c120711.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_4x5_nomask_to_0.5x0.5_nomask_aave_da_c120706.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_1.9x2.5_nomask_to_0.5x0.5_nomask_aave_da_c120709.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_ne120np4_nomask_to_0.5x0.5_nomask_aave_da_c120711.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_3x3min_USGS_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_0.9x1.25_GRDC_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_360x720cru_cruncep_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/0.5x0.5/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.5x0.5_nomask_aave_da_c190417.nc -lnd/clm2/mappingdata/maps/ne4np4/map_0.5x0.5_MODIS_to_ne4np4_nomask_aave_da_c160614.nc -lnd/clm2/mappingdata/maps/ne4np4/map_0.5x0.5_AVHRR_to_ne4np4_nomask_aave_da_c160614.nc lnd/clm2/mappingdata/maps/ne4np4/map_10x10min_nomask_to_ne4np4_nomask_aave_da_c160614.nc @@ -1545,21 +1545,21 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne4np4/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne4np4_nomask_aave_da_c160614.nc lnd/clm2/mappingdata/maps/ne4np4/map_3x3min_LandScan2004_to_ne4np4_nomask_aave_da_c160614.nc -lnd/clm2/mappingdata/maps/ne4np4/map_0.9x1.25_GRDC_to_ne4np4_nomask_aave_da_c160614.nc -lnd/clm2/mappingdata/maps/ne4np4/map_360x720_cruncep_to_ne4np4_nomask_aave_da_c160614.nc -lnd/clm2/mappingdata/maps/ne4np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne4np4_nomask_aave_da_c160614.nc lnd/clm2/mappingdata/maps/ne4np4/map_ne4np4_nomask_to_0.5x0.5_nomask_aave_da_c160614.nc -lnd/clm2/mappingdata/maps/ne4np4/map_0.5x0.5_GSDTG2000_to_ne4np4_nomask_aave_da_c210401.nc -lnd/clm2/mappingdata/maps/ne16np4/map_0.5x0.5_MODIS_to_ne16np4_nomask_aave_da_c110922.nc -lnd/clm2/mappingdata/maps/ne16np4/map_0.5x0.5_AVHRR_to_ne16np4_nomask_aave_da_c110922.nc lnd/clm2/mappingdata/maps/ne16np4/map_10x10min_nomask_to_ne16np4_nomask_aave_da_c110922.nc @@ -1579,20 +1579,20 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne16np4/map_3x3min_GLOBE-Gardner_to_ne16np4_nomask_aave_da_c120924.nc lnd/clm2/mappingdata/maps/ne16np4/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne16np4_nomask_aave_da_c120924.nc -lnd/clm2/mappingdata/maps/ne16np4/map_0.9x1.25_GRDC_to_ne16np4_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/ne16np4/map_360x720_cruncep_to_ne16np4_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/ne16np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne16np4_nomask_aave_da_c130408.nc lnd/clm2/mappingdata/maps/ne16np4/map_ne16np4_nomask_to_0.5x0.5_nomask_aave_da_c110922.nc lnd/clm2/mappingdata/maps/ne16np4/map_0.5x0.5_GSDTG2000_to_ne16np4_nomask_aave_da_c210330.nc -lnd/clm2/mappingdata/maps/ne30np4/map_0.5x0.5_landuse_to_ne30np4_aave_da_110320.nc -lnd/clm2/mappingdata/maps/ne30np4/map_0.5x0.5_lanwat_to_ne30np4_aave_da_110320.nc lnd/clm2/mappingdata/maps/ne30np4/map_10minx10min_topo_to_ne30np4_aave_da_110320.nc @@ -1612,21 +1612,21 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne30np4/map_3x3min_GLOBE-Gardner_to_ne30np4_nomask_aave_da_c120924.nc lnd/clm2/mappingdata/maps/ne30np4/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne30np4_nomask_aave_da_c120924.nc -lnd/clm2/mappingdata/maps/ne30np4/map_0.9x1.25_GRDC_to_ne30np4_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/ne30np4/map_360x720_cruncep_to_ne30np4_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/ne30np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne30np4_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/ne30np4/map_0.5x0.5_GSDTG2000_to_ne30np4_nomask_aave_da_c161010.nc lnd/clm2/mappingdata/maps/ne30np4/map_ne30np4_to_0.5x0.5rtm_aave_da_110320.nc -lnd/clm2/mappingdata/maps/ne60np4/map_0.5x0.5_MODIS_to_ne60np4_nomask_aave_da_c110922.nc -lnd/clm2/mappingdata/maps/ne60np4/map_0.5x0.5_AVHRR_to_ne60np4_nomask_aave_da_c110922.nc lnd/clm2/mappingdata/maps/ne60np4/map_10x10min_nomask_to_ne60np4_nomask_aave_da_c110922.nc @@ -1646,18 +1646,18 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne60np4/map_3x3min_GLOBE-Gardner_to_ne60np4_nomask_aave_da_c120924.nc lnd/clm2/mappingdata/maps/ne60np4/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne60np4_nomask_aave_da_c120924.nc -lnd/clm2/mappingdata/maps/ne60np4/map_0.9x1.25_GRDC_to_ne60np4_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/ne60np4/map_360x720_cruncep_to_ne60np4_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/ne60np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne60np4_nomask_aave_da_c130405.nc lnd/clm2/mappingdata/maps/ne60np4/map_ne60np4_nomask_to_0.5x0.5_nomask_aave_da_c110922.nc -lnd/clm2/mappingdata/maps/ne120np4/map_0.5x0.5_landuse_to_ne120np4_aave_da_110320.nc -lnd/clm2/mappingdata/maps/ne120np4/map_0.5x0.5_lanwat_to_ne120np4_aave_da_110320.nc lnd/clm2/mappingdata/maps/ne120np4/map_10minx10min_topo_to_ne120np4_aave_da_110320.nc @@ -1683,28 +1683,28 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne120np4/map_3x3min_USGS_to_ne120np4_nomask_aave_da_c120913.nc lnd/clm2/mappingdata/maps/ne120np4/map_5x5min_ISRIC-WISE_to_ne120np4_nomask_aave_da_c111115.nc -lnd/clm2/mappingdata/maps/ne120np4/map_0.9x1.25_GRDC_to_ne120np4_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/ne120np4/map_360x720_cruncep_to_ne120np4_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/ne120np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne120np4_nomask_aave_da_c130405.nc -lnd/clm2/mappingdata/maps/ne120np4/map_0.5x0.5_GSDTG2000_to_ne120np4_nomask_aave_da_171213.nc - + -lnd/clm2/mappingdata/maps/ne120np4/map_0.1x0.1_nomask_to_ne120np4_nomask_aave_da_c120706.nc -lnd/clm2/mappingdata/maps/ne120np4/map_3x3min_USGS_to_ne120np4_nomask_aave_da_c120913.nc - + -lnd/clm2/mappingdata/maps/5x5_amazon/map_0.5x0.5_MODIS_to_5x5_amazon_nomask_aave_da_c110920.nc -lnd/clm2/mappingdata/maps/5x5_amazon/map_0.5x0.5_AVHRR_to_5x5_amazon_nomask_aave_da_c110920.nc lnd/clm2/mappingdata/maps/5x5_amazon/map_10x10min_nomask_to_5x5_amazon_nomask_aave_da_c110920.nc @@ -1724,49 +1724,49 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/5x5_amazon/map_3x3min_GLOBE-Gardner_to_5x5_amazon_nomask_aave_da_c120923.nc lnd/clm2/mappingdata/maps/5x5_amazon/map_3x3min_GLOBE-Gardner-mergeGIS_to_5x5_amazon_nomask_aave_da_c120923.nc -lnd/clm2/mappingdata/maps/5x5_amazon/map_0.9x1.25_GRDC_to_5x5_amazon_nomask_aave_da_c130309.nc -lnd/clm2/mappingdata/maps/5x5_amazon/map_360x720_cruncep_to_5x5_amazon_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/5x5_amazon/map_1km-merge-10min_HYDRO1K-merge-nomask_to_5x5_amazon_nomask_aave_da_c130403.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_0.5x0.5_AVHRR_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_0.5x0.5_MODIS_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_0.5x0.5_nomask_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_10x10min_nomask_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_3x3min_MODIS_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_3x3min_USGS_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_3x3min_LandScan2004_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_5x5min_IGBP-GSDP_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_5x5min_ISRIC-WISE_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_5x5min_nomask_to_ne30pg2_nomask_aave_da_c201210.nc lnd/clm2/mappingdata/maps/ne30np4pg2/map_3x3min_GLOBE-Gardner_to_ne30pg2_nomask_aave_da_c201210.nc lnd/clm2/mappingdata/maps/ne30np4pg2/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_0.9x1.25_GRDC_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_360x720_cruncep_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne30np4pg2/map_0.5x0.5_GSDTG2000_to_ne30pg2_nomask_aave_da_c201210.nc -lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_MODIS_to_ne240np4_nomask_aave_da_c110922.nc -lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_AVHRR_to_ne240np4_nomask_aave_da_c110922.nc lnd/clm2/mappingdata/maps/ne240np4/map_10x10min_nomask_to_ne240np4_nomask_aave_da_c110922.nc @@ -1786,11 +1786,11 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/ne240np4/map_3x3min_GLOBE-Gardner_to_ne240np4_nomask_aave_da_c120925.nc lnd/clm2/mappingdata/maps/ne240np4/map_3x3min_GLOBE-Gardner-mergeGIS_to_ne240np4_nomask_aave_da_c120925.nc -lnd/clm2/mappingdata/maps/ne240np4/map_0.9x1.25_GRDC_to_ne240np4_nomask_aave_da_c130308.nc -lnd/clm2/mappingdata/maps/ne240np4/map_360x720_cruncep_to_ne240np4_nomask_aave_da_c130326.nc -lnd/clm2/mappingdata/maps/ne240np4/map_1km-merge-10min_HYDRO1K-merge-nomask_to_ne240np4_nomask_aave_da_c130405.nc lnd/clm2/mappingdata/maps/ne240np4/map_ne240np4_nomask_to_0.5x0.5_nomask_aave_da_c110922.nc @@ -1831,7 +1831,7 @@ this mask will have smb calculated over the entire global land surface >lnd/clm2/mappingdata/maps/northamericax4v1pg2/map_0.5x0.5_GSDTG2000_to_northamericax4v1pg2_nomask_aave_da_c210112.nc - + lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_AVHRR_to_0.125x0.125_nomask_aave_da_c190725.nc @@ -1940,12 +1940,17 @@ this mask will have smb calculated over the entire global land surface .false. .false. .false. +.false. 1 .false. "/dev/null" +.true. +.false. +.true. +.false. + .true. .true. .false. - diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 4bf4a670ea95..12d8d377c83c 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -8,51 +8,51 @@ - -If non-blank, then interpinic will be called to interpolate finidat_interp_source and +If non-blank, then interpinic will be called to interpolate finidat_interp_source and create output file specified by finidat_interp_dest. For this to be used finidat MUST BE blank. - -If finidat_interp_source is set to non-blank, then interpinic will be called +If finidat_interp_source is set to non-blank, then interpinic will be called to interpolate finidat_interp_source and create output file finidat_interp_dest. - Full pathname of initial conditions file. If blank ELM will startup from arbitrary initial conditions. - Full pathname of master restart file for a branch run. (only used if RUN_TYPE=branch) (Set with RUN_REFCASE and RUN_REFDATE) - Full pathname of land fraction data file. @@ -85,8 +85,8 @@ Type of CO2 feedback. -Supplemental Nitrogen mode and for what type of vegetation it's turned on for. -In this mode Nitrogen is unlimited rather than prognosed and in general vegetation is +Supplemental Nitrogen mode and for what type of vegetation it's turned on for. +In this mode Nitrogen is unlimited rather than prognosed and in general vegetation is over-productive. NONE = No vegetation types get supplemental Nitrogen ALL = Supplemental Nitrogen is active for all vegetation types @@ -94,8 +94,8 @@ over-productive. -Supplemental Phosphorus mode and for what type of vegetation it's turned on for. -In this mode Phosphorus is unlimited rather than prognosed and in general vegetation is +Supplemental Phosphorus mode and for what type of vegetation it's turned on for. +In this mode Phosphorus is unlimited rather than prognosed and in general vegetation is over-productive. NONE = No vegetation types get supplemental Phosphorus ALL = Supplemental Phosphorus is active for all vegetation types @@ -128,14 +128,14 @@ If TRUE, run forest N/P fertilization experiment -If TRUE, do not allow soil labile, secondary, occcluded, and +If TRUE, do not allow soil labile, secondary, occcluded, and parent material phosphorus to change, during Regular Spinup, for ECA model -If TRUE consider priority of plant to get a fraction of +If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase @@ -188,7 +188,7 @@ Interpolate the soil layers on the surface dataset to the soil layers specified If TRUE, irrigation will be active. - + @@ -267,7 +267,7 @@ Full pathname of land-ice mask data file (on lnd grid). -Full pathname of topography data file. Only required when +Full pathname of topography data file. Only required when land-ice model is active. @@ -312,6 +312,16 @@ Switch deciding which nutrient model to use in FATES. Toggle to turn on FATES fixed biogeography mode (STUB ONLY, MUST BE FALSE). + +Toggle to turn on no competition mode (only relevant if FATES is being used). + + + +Toggle to turn on FATES satellite phenology mode (only relevant if FATES is being used). + + Toggle to turn on the logging module (only relevant if FATES is being used). @@ -367,7 +377,7 @@ Full pathname datafile with fates parameters input_pathname="abs" group="elm_inparm" valid_values="" > Full pathname datafile with soil order dependent constants - + Full pathname of time varying PFT data file. This causes the land-use types of @@ -379,12 +389,12 @@ the initial surface dataset to vary over time. Full pathname of surface data file. - SNICAR (SNow, ICe, and Aerosol Radiative model) optical data file name - SNICAR (SNow, ICe, and Aerosol Radiative model) snow aging data file name @@ -486,7 +496,7 @@ Per tape series maximum number of time samples. -Per tape series history file density (i.e. output precision) +Per tape series history file density (i.e. output precision) 1=double precision 2=single precision Default: 2,2,2,2,2,2 @@ -494,7 +504,7 @@ Per tape series history file density (i.e. output precision) -Per tape series history write frequency. +Per tape series history write frequency. positive means in time steps 0=monthly negative means hours @@ -682,7 +692,7 @@ SCRIP format grid data file group="clmexp" valid_values="none,64bit_offset,netcdf4" > Flag to pass to the ESMF mapping utility, telling it what kind of large file support is needed for an output file generated with this grid as -either the source or destination ('none', '64bit_offset' or 'netcdf4'). +either the source or destination ('none', '64bit_offset' or 'netcdf4'). Filename for mksurfdata_map to remap raw data into the output surface dataset @@ -992,7 +1002,7 @@ by getco2_historical.ncl - Aerosol deposition file name (only used for aerdepregrid.ncl) @@ -1226,16 +1236,16 @@ Mapping file to go from one resolution/land-mask to another resolution/land-mask Land mask description for mksurfdata input files - + Horizontal grid resolutions for mksurfdata input files - + @@ -1254,40 +1264,40 @@ Add a note to the output namelist about the options given to build-namelist -CLM run type. +CLM run type. 'default' use the default type of clm_start type for this configuration 'cold' is a run from arbitrary initial conditions 'arb_ic' is a run using initial conditions if provided, OR arbitrary initial conditions if no files can be found - 'startup' is an initial run with initial conditions provided. + 'startup' is an initial run with initial conditions provided. 'continue' is a restart run. 'branch' is a restart run in which properties of the output history files may be changed. Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools - + Representative concentration pathway for future scenarios [radiative forcing at peak or 2100 in W/m^2] -999.9 means do NOT use a future scenario, just use historical data. - + Land mask description - + If TRUE, irrigation will be active (find surface datasets with active irrigation). - + + + @@ -1328,8 +1338,8 @@ Command line arguement for turning on CN spinup mode. Command line arguement for biogeochemistry mode for CLM4.5 sp = Satellitte Phenology cn = Carbon Nitrogen model - bgc = CLM4.5 BGC model with: - CENTURY model pools + bgc = CLM4.5 BGC model with: + CENTURY model pools Methane model Vertically resolved Carbon fates = functionally assembled terrestrial ecosystem simulator @@ -1340,20 +1350,20 @@ Command line arguement for biogeochemistry mode for CLM4.5 Where dry deposition is calculated (from land, atmosphere, or from a table) - @@ -1371,9 +1381,9 @@ Flag for overriding the crash that should occur if user tries to start the model -Flag for setting the state of the Accelerated decomposition spinup state for the model. - 0 = normal model behavior; - 1 = AD spinup. +Flag for setting the state of the Accelerated decomposition spinup state for the model. + 0 = normal model behavior; + 1 = AD spinup. Entering and exiting spinup mode occurs automatically by comparing the namelist and restart file values for this variable. @@ -1479,7 +1489,7 @@ Flag to use the atmospheric time series of C14 concentrations from bomb fallout, -Filename with time series of atmospheric Delta C14 data. variables in file are "time" and "atm_delta_c14". time variable is in format 1950.0, and time values must be monotonically increasing for interpolation, however spacing can be unequal. atm_delta_c14 variable has units of permil. +Filename with time series of atmospheric Delta C14 data. variables in file are "time" and "atm_delta_c14". time variable is in format 1950.0, and time values must be monotonically increasing for interpolation, however spacing can be unequal. atm_delta_c14 variable has units of permil. (EXPERIMENTAL and NOT tested) @@ -1513,7 +1523,7 @@ Minimum lake depth to increase non-molecular thermal diffusivities by the factor group="elm_inparm" valid_values="" > Factor to increase non-molecular thermal diffusivities for lakes deeper than deepmixing_depthcrit to account for unresolved 3D processes. -Set to 1 to +Set to 1 to -Specify what reaction module will be used within the betr framework. +Specify what reaction module will be used within the betr framework. -Allows user to tune the value of aereoxid. If set to FALSE, then use the value of aereoxid from +Allows user to tune the value of aereoxid. If set to FALSE, then use the value of aereoxid from the parameter file (set to 0.0, but may be tuned with values in the range {0.0,1.0}. If set to TRUE, then don't fix aere (see ch4Mod.F90). Default: .true. @@ -1652,7 +1662,7 @@ used in the Comp_Name variable on the file. MEGAN specifier. This is in the form of: Chem-compound = megan_compound(s) where megan_compound(s) can be the sum of megan compounds with a "+" between them. In each equation, the item to the left of the equal sign is a CAM chemistry compound, the -items to the right are compounds known to the MEGAN model (single or combinations). +items to the right are compounds known to the MEGAN model (single or combinations). For example, megan_specifier = 'ISOP = isoprene', 'C10H16 = pinene_a + carene_3 + thujene_a' @@ -1665,13 +1675,13 @@ If TRUE then use mapped MEGAN emissions factors for isoprene. List of possible MEGAN compounds to use (the list used by the simulation is on the megan_factors_file as the Comp_Name) - + From 0158658d6ce062e1bdc760e2729a403cb6e76ea8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 21:47:20 -0700 Subject: [PATCH 034/301] adding fates switches to elm_varctl --- components/elm/src/main/elm_varctl.F90 | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90 index a5f0c966c4f3..f35fc49ee021 100644 --- a/components/elm/src/main/elm_varctl.F90 +++ b/components/elm/src/main/elm_varctl.F90 @@ -219,19 +219,21 @@ module elm_varctl ! FATES switches !---------------------------------------------------------- - logical, public :: use_fates = .false. ! true => use ED - integer, public :: fates_spitfire_mode = 0 ! 0 for no fire; 1 for constant ignitions - logical, public :: use_fates_fixed_biogeog = .false. ! true => use fixed biogeography mode - logical, public :: use_fates_logging = .false. ! true => turn on logging module - logical, public :: use_fates_planthydro = .false. ! true => turn on fates hydro + logical, public :: use_fates = .false. ! true => use ED + integer, public :: fates_spitfire_mode = 0 ! 0 for no fire; 1 for constant ignitions + logical, public :: use_fates_fixed_biogeog = .false. ! true => use fixed biogeography mode + logical, public :: use_fates_logging = .false. ! true => turn on logging module + logical, public :: use_fates_planthydro = .false. ! true => turn on fates hydro logical, public :: use_fates_cohort_age_tracking = .false. ! true => turn on cohort age tracking - logical, public :: use_fates_ed_st3 = .false. ! true => static stand structure - logical, public :: use_fates_ed_prescribed_phys = .false. ! true => prescribed physiology - logical, public :: use_fates_inventory_init = .false. ! true => initialize fates from inventory - character(len=256), public :: fates_inventory_ctrl_filename = '' ! filename for inventory control - integer, public :: fates_parteh_mode = -9 ! 1 => carbon only - ! 2 => C+N+P (not enabled yet) - ! no others enabled + logical, public :: use_fates_ed_st3 = .false. ! true => static stand structure + logical, public :: use_fates_ed_prescribed_phys = .false. ! true => prescribed physiology + logical, public :: use_fates_inventory_init = .false. ! true => initialize fates from inventory + logical, public :: use_fates_nocomp = .false. ! true => use no comopetition mode + logical, public :: use_fates_sp = .false. ! true => use FATES satellite phenology mode + character(len=256), public :: fates_inventory_ctrl_filename = '' ! filename for inventory control + integer, public :: fates_parteh_mode = -9 ! 1 => carbon only + ! 2 => C+N+P (not enabled yet) + ! no others enabled !---------------------------------------------------------- From 269661530e4f2d6ae8bbc42631993d7647ad6e1d Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 21:52:30 -0700 Subject: [PATCH 035/301] adding avoidance of snow fraction adjustment of elai and esai in SatPhen --- .../src/biogeochem/SatellitePhenologyMod.F90 | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/components/elm/src/biogeochem/SatellitePhenologyMod.F90 b/components/elm/src/biogeochem/SatellitePhenologyMod.F90 index 527b9eb5e671..f334d611c869 100644 --- a/components/elm/src/biogeochem/SatellitePhenologyMod.F90 +++ b/components/elm/src/biogeochem/SatellitePhenologyMod.F90 @@ -2,8 +2,8 @@ module SatellitePhenologyMod !----------------------------------------------------------------------- ! !DESCRIPTION: - ! CLM Satelitte Phenology model (SP) ecosystem dynamics (phenology, vegetation). - ! Allow some subroutines to be used by the CLM Carbon Nitrogen model (CLMCN) + ! CLM Satelitte Phenology model (SP) ecosystem dynamics (phenology, vegetation). + ! Allow some subroutines to be used by the CLM Carbon Nitrogen model (CLMCN) ! so that DryDeposition code can get estimates of LAI differences between months. ! ! !USES: @@ -22,7 +22,7 @@ module SatellitePhenologyMod use decompMod , only : gsmap_lnd_gdc2glo use domainMod , only : ldomain use fileutils , only : getavu, relavu - use VegetationType , only : veg_pp + use VegetationType , only : veg_pp use CanopyStateType , only : canopystate_type use WaterstateType , only : waterstate_type use ColumnDataType , only : col_ws @@ -30,7 +30,7 @@ module SatellitePhenologyMod use spmdMod , only : masterproc use spmdMod , only : mpicom, comp_id use mct_mod - use ncdio_pio + use ncdio_pio ! ! !PUBLIC TYPES: implicit none @@ -60,7 +60,7 @@ module SatellitePhenologyMod !----------------------------------------------------------------------- contains - + !----------------------------------------------------------------------- ! ! lai_init @@ -90,10 +90,10 @@ subroutine lai_init(bounds) integer :: i ! index integer :: stream_year_first_lai ! first year in Lai stream to use integer :: stream_year_last_lai ! last year in Lai stream to use - integer :: model_year_align_lai ! align stream_year_first_lai with + integer :: model_year_align_lai ! align stream_year_first_lai with integer :: nu_nml ! unit for namelist file integer :: nml_error ! namelist i/o error flag - type(mct_ggrid) :: dom_elm ! domain information + type(mct_ggrid) :: dom_elm ! domain information character(len=CL) :: stream_fldFileName_lai ! lai stream filename to read character(len=CL) :: lai_mapalgo = 'bilinear' ! Mapping alogrithm @@ -143,9 +143,9 @@ subroutine lai_init(bounds) write(iulog,*) ' ' write(iulog,*) 'lai_stream settings:' - write(iulog,*) ' stream_year_first_lai = ',stream_year_first_lai - write(iulog,*) ' stream_year_last_lai = ',stream_year_last_lai - write(iulog,*) ' model_year_align_lai = ',model_year_align_lai + write(iulog,*) ' stream_year_first_lai = ',stream_year_first_lai + write(iulog,*) ' stream_year_last_lai = ',stream_year_last_lai + write(iulog,*) ' model_year_align_lai = ',model_year_align_lai write(iulog,*) ' stream_fldFileName_lai = ',trim(stream_fldFileName_lai) endif @@ -158,7 +158,7 @@ subroutine lai_init(bounds) fldList = shr_string_listCreateField( numLaiFields, laiString ) call shr_strdata_create(sdat_lai,name="laidyn", & - pio_subsystem=pio_subsystem, & + pio_subsystem=pio_subsystem, & pio_iotype=shr_pio_getiotype(inst_name), & mpicom=mpicom, compid=comp_id, & gsmap=gsmap_lnd_gdc2glo, ggrid=dom_elm, & @@ -171,7 +171,7 @@ subroutine lai_init(bounds) domFileName=trim(stream_fldFileName_lai), & domTvarName='time', & domXvarName='lon' , & - domYvarName='lat' , & + domYvarName='lat' , & domAreaName='area', & domMaskName='mask', & filePath='', & @@ -204,7 +204,7 @@ subroutine lai_interp(bounds, canopystate_vars) ! ! !ARGUMENTS: implicit none - type(bounds_type) , intent(in) :: bounds + type(bounds_type) , intent(in) :: bounds type(canopystate_type) , intent(inout) :: canopystate_vars ! ! !LOCAL VARIABLES: @@ -262,7 +262,7 @@ subroutine SatellitePhenologyInit (bounds) use shr_infnan_mod, only : nan => shr_infnan_nan, assignment(=) ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds + type(bounds_type), intent(in) :: bounds ! ! !LOCAL VARIABLES: integer :: ier ! error code @@ -302,10 +302,11 @@ subroutine SatellitePhenology(bounds, num_nolakep, filter_nolakep, & ! Calculates leaf areas (tlai, elai), stem areas (tsai, esai) and height (htop). ! ! !USES: - use pftvarcon, only : noveg, nbrdlf_dcd_brl_shrub + use pftvarcon, only : noveg, nbrdlf_dcd_brl_shrub + use elm_varctl, only : use_fates_sp ! ! !ARGUMENTS: - type(bounds_type) , intent(in) :: bounds + type(bounds_type) , intent(in) :: bounds integer , intent(in) :: num_nolakep ! number of column non-lake points in pft filter integer , intent(in) :: filter_nolakep(bounds%endp-bounds%begp+1) ! patch filter for non-lake points type(waterstate_type) , intent(in) :: waterstate_vars @@ -318,14 +319,14 @@ subroutine SatellitePhenology(bounds, num_nolakep, filter_nolakep, & !----------------------------------------------------------------------- associate( & - frac_sno => col_ws%frac_sno , & ! Input: [real(r8) (:) ] fraction of ground covered by snow (0 to 1) - snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) - tlai => canopystate_vars%tlai_patch , & ! Output: [real(r8) (:) ] one-sided leaf area index, no burying by snow + frac_sno => col_ws%frac_sno , & ! Input: [real(r8) (:) ] fraction of ground covered by snow (0 to 1) + snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) + tlai => canopystate_vars%tlai_patch , & ! Output: [real(r8) (:) ] one-sided leaf area index, no burying by snow tsai => canopystate_vars%tsai_patch , & ! Output: [real(r8) (:) ] one-sided stem area index, no burying by snow elai => canopystate_vars%elai_patch , & ! Output: [real(r8) (:) ] one-sided leaf area index with burying by snow esai => canopystate_vars%esai_patch , & ! Output: [real(r8) (:) ] one-sided stem area index with burying by snow - htop => canopystate_vars%htop_patch , & ! Output: [real(r8) (:) ] canopy top (m) - hbot => canopystate_vars%hbot_patch , & ! Output: [real(r8) (:) ] canopy bottom (m) + htop => canopystate_vars%htop_patch , & ! Output: [real(r8) (:) ] canopy top (m) + hbot => canopystate_vars%hbot_patch , & ! Output: [real(r8) (:) ] canopy bottom (m) frac_veg_nosno_alb => canopystate_vars%frac_veg_nosno_alb_patch & ! Output: [integer (:) ] fraction of vegetation not covered by snow (0 OR 1) [-] ) @@ -366,7 +367,7 @@ subroutine SatellitePhenology(bounds, num_nolakep, filter_nolakep, & ! problems associated with very small lai and sai. ! snow burial fraction for short vegetation (e.g. grasses) as in - ! Wang and Zeng, 2007. + ! Wang and Zeng, 2007. if (veg_pp%itype(p) > noveg .and. veg_pp%itype(p) <= nbrdlf_dcd_brl_shrub ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) @@ -377,18 +378,23 @@ subroutine SatellitePhenology(bounds, num_nolakep, filter_nolakep, & endif ! area weight by snow covered fraction + if (.not.use_fates_sp) then + ! Do not set these in FATES_SP mode as they turn on the 'vegsol' filter and also + ! are duplicated by the FATE variables (in the FATES IFP indexing space) - elai(p) = max(tlai(p)*(1.0_r8 - frac_sno(c)) + tlai(p)*fb*frac_sno(c), 0.0_r8) - esai(p) = max(tsai(p)*(1.0_r8 - frac_sno(c)) + tsai(p)*fb*frac_sno(c), 0.0_r8) - if (elai(p) < 0.05_r8) elai(p) = 0._r8 - if (esai(p) < 0.05_r8) esai(p) = 0._r8 + elai(p) = max(tlai(p)*(1.0_r8 - frac_sno(c)) + tlai(p)*fb*frac_sno(c), 0.0_r8) + esai(p) = max(tsai(p)*(1.0_r8 - frac_sno(c)) + tsai(p)*fb*frac_sno(c), 0.0_r8) + if (elai(p) < 0.05_r8) elai(p) = 0._r8 + if (esai(p) < 0.05_r8) esai(p) = 0._r8 - ! Fraction of vegetation free of snow + ! Fraction of vegetation free of snow + + if ((elai(p) + esai(p)) >= 0.05_r8) then + frac_veg_nosno_alb(p) = 1 + else + frac_veg_nosno_alb(p) = 0 + end if - if ((elai(p) + esai(p)) >= 0.05_r8) then - frac_veg_nosno_alb(p) = 1 - else - frac_veg_nosno_alb(p) = 0 end if end do ! end of patch loop @@ -408,7 +414,7 @@ subroutine interpMonthlyVeg (bounds, canopystate_vars) use clm_time_manager, only : get_curr_date, get_step_size, get_nstep ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds + type(bounds_type), intent(in) :: bounds type(canopystate_type), intent(inout) :: canopystate_vars ! ! !LOCAL VARIABLES: @@ -466,12 +472,12 @@ subroutine readAnnualVegetation (bounds, canopystate_vars) use shr_scam_mod, only : shr_scam_getCloseLatLon ! ! !ARGUMENTS: - type(bounds_type), intent(in) :: bounds + type(bounds_type), intent(in) :: bounds type(canopystate_type), intent(inout) :: canopystate_vars ! ! !LOCAL VARIABLES: type(file_desc_t) :: ncid ! netcdf id - real(r8), pointer :: annlai(:,:) ! 12 months of monthly lai from input data set + real(r8), pointer :: annlai(:,:) ! 12 months of monthly lai from input data set real(r8), pointer :: mlai(:,:) ! lai read from input files real(r8):: closelat,closelon ! single column vars integer :: ier ! error code @@ -488,13 +494,13 @@ subroutine readAnnualVegetation (bounds, canopystate_vars) character(len=32) :: subname = 'readAnnualVegetation' !----------------------------------------------------------------------- - annlai => canopystate_vars%annlai_patch + annlai => canopystate_vars%annlai_patch ! Determine necessary indices allocate(mlai(bounds%begg:bounds%endg,0:numpft), stat=ier) if (ier /= 0) then - write(iulog,*)subname, 'allocation error ' + write(iulog,*)subname, 'allocation error ' call endrun(msg=errMsg(__FILE__, __LINE__)) end if @@ -540,7 +546,7 @@ subroutine readAnnualVegetation (bounds, canopystate_vars) else !! non-vegetated pft annlai(k,p) = 0._r8 end if - end do ! end of loop over patches + end do ! end of loop over patches enddo ! months loop @@ -567,7 +573,7 @@ subroutine readMonthlyVegetation (bounds, & use netcdf ! ! !ARGUMENTS: - type(bounds_type) , intent(in) :: bounds + type(bounds_type) , intent(in) :: bounds character(len=*) , intent(in) :: fveg ! file with monthly vegetation data integer , intent(in) :: months(2) ! months to be interpolated (1 to 12) type(canopystate_type), intent(inout) :: canopystate_vars From b8f7669e20bf451b148373e9d251d993cc430dd8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 22:06:31 -0700 Subject: [PATCH 036/301] adding hist_patch spmode variables to CanopyState --- .../elm/src/biogeophys/CanopyStateType.F90 | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/components/elm/src/biogeophys/CanopyStateType.F90 b/components/elm/src/biogeophys/CanopyStateType.F90 index 17736cb7ac6e..6cd9b32ba198 100644 --- a/components/elm/src/biogeophys/CanopyStateType.F90 +++ b/components/elm/src/biogeophys/CanopyStateType.F90 @@ -9,7 +9,7 @@ module CanopyStateType use landunit_varcon , only : istsoil, istcrop use elm_varcon , only : spval,ispval use elm_varpar , only : nlevcan, nvegwcs - use elm_varctl , only : iulog, use_cn, use_fates, use_hydrstress + use elm_varctl , only : iulog, use_cn, use_fates, use_hydrstress, use_fates_sp use LandunitType , only : lun_pp use ColumnType , only : col_pp use VegetationType , only : veg_pp @@ -39,6 +39,9 @@ module CanopyStateType real(r8) , pointer :: tlai_patch (:) ! patch canopy one-sided leaf area index, no burying by snow real(r8) , pointer :: tsai_patch (:) ! patch canopy one-sided stem area index, no burying by snow + real(r8) , pointer :: tlai_hist_patch (:) ! patch canopy one-sided leaf area index, for SP mode + real(r8) , pointer :: tsai_hist_patch (:) ! patch canopy one-sided stem area index, for SP mode + real(r8) , pointer :: htop_hist_patch (:) ! patch canopy height, for SP mode real(r8) , pointer :: elai_patch (:) ! patch canopy one-sided leaf area index with burying by snow real(r8) , pointer :: esai_patch (:) ! patch canopy one-sided stem area index with burying by snow real(r8) , pointer :: elai240_patch (:) ! patch canopy one-sided leaf area index with burying by snow average over 10days @@ -117,6 +120,9 @@ subroutine InitAllocate(this, bounds ) allocate(this%frac_veg_nosno_alb_patch (begp:endp)) ; this%frac_veg_nosno_alb_patch (:) = 0 allocate(this%tlai_patch (begp:endp)) ; this%tlai_patch (:) = nan allocate(this%tsai_patch (begp:endp)) ; this%tsai_patch (:) = nan + allocate(this%tlai_hist_patch (begp:endp)) ; this%tlai_hist_patch (:) = nan + allocate(this%tsai_hist_patch (begp:endp)) ; this%tsai_hist_patch (:) = nan + allocate(this%htop_hist_patch (begp:endp)) ; this%htop_hist_patch (:) = nan allocate(this%elai_patch (begp:endp)) ; this%elai_patch (:) = nan allocate(this%elai240_patch (begp:endp)) ; this%elai240_patch (:) = nan allocate(this%esai_patch (begp:endp)) ; this%esai_patch (:) = nan @@ -180,15 +186,27 @@ subroutine InitHistory(this, bounds) avgflag='A', long_name='exposed one-sided stem area index', & ptr_patch=this%esai_patch) - this%tlai_patch(begp:endp) = spval - call hist_addfld1d (fname='TLAI', units='1', & - avgflag='A', long_name='total projected leaf area index', & - ptr_patch=this%tlai_patch) - - this%tsai_patch(begp:endp) = spval - call hist_addfld1d (fname='TSAI', units='1', & - avgflag='A', long_name='total projected stem area index', & - ptr_patch=this%tsai_patch) + if (use_fates_sp) then + this%tlai_hist_patch(begp:endp) = spval + call hist_addfld1d (fname='TLAI', units='m', & + avgflag='A', long_name='TLAI weights for SP mode', & + ptr_patch=this%tlai_hist_patch) + + this%tsai_hist_patch(begp:endp) = spval + call hist_addfld1d (fname='TSAI', units='m', & + avgflag='A', long_name='TSAI weights for SP mode', & + ptr_patch=this%tsai_hist_patch) + else + this%tlai_patch(begp:endp) = spval + call hist_addfld1d (fname='TLAI', units='1', & + avgflag='A', long_name='total projected leaf area index', & + ptr_patch=this%tlai_patch) + + this%tsai_patch(begp:endp) = spval + call hist_addfld1d (fname='TSAI', units='1', & + avgflag='A', long_name='total projected stem area index', & + ptr_patch=this%tsai_patch) + end if if (use_cn .or. use_fates) then this%fsun_patch(begp:endp) = spval @@ -215,10 +233,19 @@ subroutine InitHistory(this, bounds) end if if (use_cn .or. use_fates) then - this%htop_patch(begp:endp) = spval - call hist_addfld1d (fname='HTOP', units='m', & - avgflag='A', long_name='canopy top', & - ptr_patch=this%htop_patch) + + if (use_fates_sp) then + this%htop_hist_patch(begp:endp) = spval + call hist_addfld1d (fname='HTOP', units='m', & + avgflag='A', long_name='HTOP weights for SP mode', & + ptr_patch=this%htop_hist_patch) + else + this%htop_patch(begp:endp) = spval + call hist_addfld1d (fname='HTOP', units='m', & + avgflag='A', long_name='canopy top', & + ptr_patch=this%htop_patch) + end if + end if if (use_cn .or. use_fates) then @@ -459,6 +486,10 @@ subroutine InitCold(this, bounds) this%laisha_patch(p) = 0._r8 end if + this%tlai_hist_patch(p) = 0._r8 + this%tsai_hist_patch(p) = 0._r8 + this%htop_hist_patch(p) = 0._r8 + ! needs to be initialized to spval to avoid problems when averaging for the accum ! field this%fsun_patch(p) = spval From 83300582cd854ed7d512588192bd57bf6025ecfb Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 22:13:03 -0700 Subject: [PATCH 037/301] adding fates switches to SatellitePhenolgy and related calls --- components/elm/src/main/elm_driver.F90 | 40 +++++++++++++------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/components/elm/src/main/elm_driver.F90 b/components/elm/src/main/elm_driver.F90 index 9f8565e20d29..90937a712899 100644 --- a/components/elm/src/main/elm_driver.F90 +++ b/components/elm/src/main/elm_driver.F90 @@ -14,7 +14,7 @@ module elm_driver use elm_varpar , only : nlevtrc_soil, nlevsoi use elm_varctl , only : wrtdia, iulog, create_glacier_mec_landunit, use_fates, use_betr, use_extrasnowlayers use elm_varctl , only : use_cn, use_lch4, use_voc, use_noio, use_c13, use_c14 - use elm_varctl , only : use_erosion + use elm_varctl , only : use_erosion, use_fates_sp use clm_time_manager , only : get_step_size, get_curr_date, get_ref_date, get_nstep, is_beg_curr_day, get_curr_time_string use clm_time_manager , only : get_curr_calday, get_days_per_year use elm_varpar , only : nlevsno, nlevgrnd, crop_prog @@ -137,9 +137,9 @@ module elm_driver use GridcellDataType , only : grc_cf, c13_grc_cf, c14_grc_cf use GridcellDataType , only : grc_ns, grc_nf use GridcellDataType , only : grc_ps, grc_pf - use TopounitDataType , only : top_as, top_af - use LandunitType , only : lun_pp - use ColumnType , only : col_pp + use TopounitDataType , only : top_as, top_af + use LandunitType , only : lun_pp + use ColumnType , only : col_pp use ColumnDataType , only : col_es, col_ef, col_ws, col_wf use ColumnDataType , only : col_cs, c13_col_cs, c14_col_cs use ColumnDataType , only : col_cf, c13_col_cf, c14_col_cf @@ -247,7 +247,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) call get_curr_date(year_curr,mon_curr, day_curr,secs_curr) dayspyr_mod = get_days_per_year() jday_mod = get_curr_calday() - + if (do_budgets) then call WaterBudget_Reset() @@ -277,7 +277,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) ! vegetation top [mhvt1,mhvt2] and vegetation bottom [mhvb1,mhvb2]. The ! weights obtained here are used in subroutine SatellitePhenology to obtain time ! interpolated values. - if (doalb .or. ( n_drydep > 0 .and. drydep_method == DD_XLND )) then + if (doalb .or. ( n_drydep > 0 .and. drydep_method == DD_XLND ) .or. use_fates_sp) then call t_startf('interpMonthlyVeg') call interpMonthlyVeg(bounds_proc, canopystate_vars) call t_stopf('interpMonthlyVeg') @@ -406,11 +406,11 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) call col_ps%Summary(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc) - + call BeginGridCBalance(bounds_clump, col_cs, grc_cs) call BeginGridNBalance(bounds_clump, col_ns, grc_ns) call BeginGridPBalance(bounds_clump, col_ps, grc_ps) - + end if call t_stopf('cnpinit') @@ -482,15 +482,15 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) call EndGridCBalanceAfterDynSubgridDriver(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_cs, grc_cs, grc_cf) - + call EndGridNBalanceAfterDynSubgridDriver(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_ns, grc_ns, grc_nf) - + call EndGridPBalanceAfterDynSubgridDriver(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_ps, grc_ps, grc_pf) - + end do !$OMP END PARALLEL DO call t_stopf('cnbalchk_at_grid') @@ -567,7 +567,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) call BeginColPBalance(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_ps) - + call t_stopf('begcnpbalwf') end if @@ -1099,7 +1099,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) end if else ! not use_cn - if (doalb) then + if (doalb.or.(use_fates_sp.and.doalb)) then ! Prescribed biogeography - prescribed canopy structure, some prognostic carbon fluxes call SatellitePhenology(bounds_clump, & @@ -1283,21 +1283,21 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) endif if (use_cn .or. use_fates) then - + call t_startf('cnbalchk') - + call ColCBalanceCheck(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_cs, col_cf) - + call ColNBalanceCheck(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_ns, col_nf) - + call ColPBalanceCheck(bounds_clump, & filter(nc)%num_soilc, filter(nc)%soilc, & col_ps, col_pf) - + call GridCBalanceCheck(bounds_clump, col_cs, col_cf, grc_cs, grc_cf) call t_stopf('cnbalchk') @@ -1459,7 +1459,7 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate) soilstate_vars%sucsat_col(bounds_proc%begc:bounds_proc%endc, 1:), & soilstate_vars%bsw_col(bounds_proc%begc:bounds_proc%endc, 1:), & soilstate_vars%hksat_col(bounds_proc%begc:bounds_proc%endc, 1:)) - + call t_stopf('elm_drv_io_htapes') ! Write restart/initial files if appropriate if (rstwr) then @@ -1732,7 +1732,7 @@ subroutine elm_drv_patch2col (bounds, num_nolakec, filter_nolakec, & call p2c (bounds, num_nolakec, filter_nolakec, & qflx_snow_grnd_patch(bounds%begp:bounds%endp), & qflx_snow_grnd_col (bounds%begc:bounds%endc)) - + if (.not. use_extrasnowlayers) then call p2c (bounds, num_allc, filter_allc, & veg_wf%qflx_snwcp_liq(bounds%begp:bounds%endp), & From 53b5164134cf6afccda7f6ffef9e314b574fb10a Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 22:24:28 -0700 Subject: [PATCH 038/301] add SatellitePhen calls for initialization --- components/elm/src/main/elm_initializeMod.F90 | 160 ++++++++++-------- 1 file changed, 88 insertions(+), 72 deletions(-) diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index 42ec8ac31c65..0df5d0d890fb 100644 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -12,27 +12,27 @@ module elm_initializeMod use elm_varctl , only : nsrest, nsrStartup, nsrContinue, nsrBranch use elm_varctl , only : create_glacier_mec_landunit, iulog use elm_varctl , only : use_lch4, use_cn, use_voc, use_c13, use_c14 - use elm_varctl , only : use_fates, use_betr + use elm_varctl , only : use_fates, use_betr, use_fates_sp use elm_varsur , only : wt_lunit, urban_valid, wt_nat_patch, wt_cft, wt_glc_mec, topo_glc_mec use elm_varsur , only : fert_cft use perf_mod , only : t_startf, t_stopf !use readParamsMod , only : readParameters use readParamsMod , only : readSharedParameters, readPrivateParameters use ncdio_pio , only : file_desc_t - + use BeTRSimulationALM, only : create_betr_simulation_alm - ! + ! !----------------------------------------- ! Definition of component types !----------------------------------------- use GridcellType , only : grc_pp use TopounitType , only : top_pp use TopounitDataType , only : top_as, top_af, top_es - use LandunitType , only : lun_pp + use LandunitType , only : lun_pp use ColumnType , only : col_pp - use ColumnDataType , only : col_es + use ColumnDataType , only : col_es use VegetationType , only : veg_pp - use VegetationDataType , only : veg_es + use VegetationDataType , only : veg_es use elm_instMod use WaterBudgetMod , only : WaterBudget_Reset @@ -51,14 +51,14 @@ module elm_initializeMod subroutine initialize1( ) ! ! !DESCRIPTION: - ! CLM initialization first phase + ! CLM initialization first phase ! ! !USES: use elm_varpar , only: elm_varpar_init, natpft_lb, natpft_ub, cft_lb, cft_ub, maxpatch_glcmec use elm_varcon , only: elm_varcon_init use landunit_varcon , only: landunit_varcon_init, max_lunit, istice_mec use column_varcon , only: col_itype_to_icemec_class - use elm_varctl , only: fsurdat, fatmlndfrc, flndtopo, fglcmask, noland, version + use elm_varctl , only: fsurdat, fatmlndfrc, flndtopo, fglcmask, noland, version use pftvarcon , only: pftconrd use soilorder_varcon , only: soilorder_conrd use decompInitMod , only: decompInit_lnd, decompInit_clumps, decompInit_gtlcp @@ -88,7 +88,7 @@ subroutine initialize1( ) integer :: ns, ni, nj ! global grid sizes integer :: begg, endg ! processor bounds integer :: icemec_class ! current icemec class (1..maxpatch_glcmec) - type(bounds_type) :: bounds_proc + type(bounds_type) :: bounds_proc type(bounds_type) :: bounds_clump ! clump bounds integer ,pointer :: amask(:) ! global land mask integer ,pointer :: cellsOnCell(:,:) ! grid cell level connectivity @@ -187,7 +187,7 @@ subroutine initialize1( ) endif ! *** Get JUST gridcell processor bounds *** - ! Remaining bounds (landunits, columns, patches) will be determined + ! Remaining bounds (landunits, columns, patches) will be determined ! after the call to decompInit_glcp - so get_proc_bounds is called ! twice and the gridcell information is just filled in twice @@ -218,7 +218,7 @@ subroutine initialize1( ) write(iulog,*) 'Attempting to read atm topo from ',trim(flndtopo) call shr_sys_flush(iulog) endif - call surfrd_get_topo(ldomain, flndtopo) + call surfrd_get_topo(ldomain, flndtopo) endif ! Initialize urban model input (initialize urbinp data structure) @@ -261,14 +261,14 @@ subroutine initialize1( ) ! ------------------------------------------------------------------------ ! Ask Fates to evaluate its own dimensioning needs. - ! + ! ! (Note: fates_maxELementsPerSite is the critical variable used by CLM ! to allocate space, determined in this routine) ! ------------------------------------------------------------------------ call ELMFatesGlobals() - + ! ------------------------------------------------------------------------ ! Determine decomposition of subgrid scale landunits, topounits, columns, patches ! ------------------------------------------------------------------------ @@ -284,7 +284,7 @@ subroutine initialize1( ) ! *** Get ALL processor bounds - for gridcells, landunit, columns and patches *** call get_proc_bounds(bounds_proc) - + ! Allocate memory for subgrid data structures ! This is needed here BEFORE the following call to initGridcells ! Note that the assumption is made that none of the subgrid initialization @@ -292,22 +292,22 @@ subroutine initialize1( ) ! Initialize the gridcell data types call grc_pp%Init (bounds_proc%begg_all, bounds_proc%endg_all) - + ! Initialize the topographic unit data types call top_pp%Init (bounds_proc%begt_all, bounds_proc%endt_all) ! topology and physical properties call top_as%Init (bounds_proc%begt_all, bounds_proc%endt_all) ! atmospheric state variables (forcings) call top_af%Init (bounds_proc%begt_all, bounds_proc%endt_all) ! atmospheric flux variables (forcings) call top_es%Init (bounds_proc%begt_all, bounds_proc%endt_all) ! energy state - + ! Initialize the landunit data types call lun_pp%Init (bounds_proc%begl_all, bounds_proc%endl_all) - + ! Initialize the column data types call col_pp%Init (bounds_proc%begc_all, bounds_proc%endc_all) - + ! Initialize the vegetation (PFT) data types call veg_pp%Init (bounds_proc%begp_all, bounds_proc%endp_all) - + ! Initialize the cohort data types (nothing here yet) ! ...to be added later... @@ -347,7 +347,7 @@ subroutine initialize1( ) ! ------------------------------------------------------------------------ ! Set CH4 Model Parameters from namelist. - ! Need to do before initTimeConst so that it knows whether to + ! Need to do before initTimeConst so that it knows whether to ! look for several optional parameters on surfdata file. if (use_lch4) then @@ -401,7 +401,7 @@ subroutine initialize2( ) use elm_varctl , only : use_century_decomp, single_column, scmlat, scmlon, use_cn, use_fates use elm_varorb , only : eccen, mvelpp, lambm0, obliqr use clm_time_manager , only : get_step_size, get_curr_calday - use clm_time_manager , only : get_curr_date, get_nstep, advance_timestep + use clm_time_manager , only : get_curr_date, get_nstep, advance_timestep use clm_time_manager , only : timemgr_init, timemgr_restart_io, timemgr_restart use controlMod , only : nlfilename use decompMod , only : get_proc_clumps, get_proc_bounds, get_clump_bounds, bounds_type @@ -416,31 +416,32 @@ subroutine initialize2( ) use histFileMod , only : hist_htapes_build, htapes_fieldlist use histFileMod , only : hist_addfld1d, hist_addfld2d, no_snow_normal use restFileMod , only : restFile_getfile, restFile_open, restFile_close - use restFileMod , only : restFile_read, restFile_write - use accumulMod , only : print_accum_fields + use restFileMod , only : restFile_read, restFile_write + use accumulMod , only : print_accum_fields use ndepStreamMod , only : ndep_init, ndep_interp use EcosystemDynMod , only : EcosystemDynInit use pdepStreamMod , only : pdep_init, pdep_interp use DecompCascadeBGCMod , only : init_decompcascade_bgc use DecompCascadeCNMod , only : init_decompcascade_cn use CNDecompCascadeContype, only : init_decomp_cascade_constants - use VegetationPropertiesType , only : veg_vp - use SoilorderConType , only : soilorderconInit - use LakeCon , only : LakeConInit - use SatellitePhenologyMod , only : SatellitePhenologyInit, readAnnualVegetation, interpMonthlyVeg + use VegetationPropertiesType , only : veg_vp + use SoilorderConType , only : soilorderconInit + use LakeCon , only : LakeConInit + use SatellitePhenologyMod , only : SatellitePhenologyInit, readAnnualVegetation + use SatellitePhenologyMod , only : interpMonthlyVeg, SatellitePhenology use SnowSnicarMod , only : SnowAge_init, SnowOptics_init use initVerticalMod , only : initVertical use lnd2atmMod , only : lnd2atm_minimal use glc2lndMod , only : glc2lnd_type - use lnd2glcMod , only : lnd2glc_type + use lnd2glcMod , only : lnd2glc_type use SoilWaterRetentionCurveFactoryMod , only : create_soil_water_retention_curve use elm_varctl , only : use_elm_interface, use_pflotran use elm_interface_pflotranMod , only : elm_pf_interface_init !, elm_pf_set_restart_stamp - use tracer_varcon , only : is_active_betr_bgc + use tracer_varcon , only : is_active_betr_bgc use clm_time_manager , only : is_restart use ALMbetrNLMod , only : betr_namelist_buffer ! - ! !ARGUMENTS + ! !ARGUMENTS implicit none ! ! !LOCAL VARIABLES: @@ -451,7 +452,7 @@ subroutine initialize2( ) integer :: ncsec ! current time of day [seconds] integer :: nc ! clump index integer :: nclumps ! number of clumps on this processor - character(len=256) :: fnamer ! name of netcdf restart file + character(len=256) :: fnamer ! name of netcdf restart file character(len=256) :: pnamer ! full pathname of netcdf restart file character(len=256) :: locfn ! local file name type(file_desc_t) :: ncid ! netcdf id @@ -474,7 +475,7 @@ subroutine initialize2( ) integer :: begc, endc integer :: begl, endl real(r8), pointer :: data2dptr(:,:) ! temp. pointers for slicing larger arrays - character(len=32) :: subname = 'initialize2' + character(len=32) :: subname = 'initialize2' !---------------------------------------------------------------------- call t_startf('elm_init2') @@ -498,7 +499,7 @@ subroutine initialize2( ) ! Initialize time manager ! ------------------------------------------------------------------------ - if (nsrest == nsrStartup) then + if (nsrest == nsrStartup) then call timemgr_init() else call restFile_getfile(file=fnamer, path=pnamer) @@ -522,11 +523,11 @@ subroutine initialize2( ) call shr_orb_decl( caldaym1, eccen, mvelpp, lambm0, obliqr, declinm1, eccf ) call t_stopf('init_orbd') - + call InitDaylength(bounds_proc, declin=declin, declinm1=declinm1) - + ! Initialize maximum daylength, based on latitude and maximum declination - ! maximum declination hardwired for present-day orbital parameters, + ! maximum declination hardwired for present-day orbital parameters, ! +/- 23.4667 degrees = +/- 0.409571 radians, use negative value for S. Hem do g = bounds_proc%begg,bounds_proc%endg @@ -548,7 +549,7 @@ subroutine initialize2( ) end if ! ------------------------------------------------------------------------ - ! Initialize component data structures + ! Initialize component data structures ! ------------------------------------------------------------------------ ! Note: new logic is in place that sets all the history fields to spval so @@ -556,7 +557,7 @@ subroutine initialize2( ) ! First put in history calls for subgrid data structures - these cannot appear in the ! module for the subgrid data definition due to circular dependencies that are introduced - + data2dptr => col_pp%dz(:,-nlevsno+1:0) col_pp%dz(bounds_proc%begc:bounds_proc%endc,:) = spval call hist_addfld2d (fname='SNO_Z', units='m', type2d='levsno', & @@ -580,7 +581,7 @@ subroutine initialize2( ) else allocate(ep_betr, source=create_betr_simulation_alm()) endif - + call SnowOptics_init( ) ! SNICAR optical parameters: call SnowAge_init( ) ! SNICAR aging parameters: @@ -600,7 +601,7 @@ subroutine initialize2( ) if (use_century_decomp) then ! Note that init_decompcascade_bgc needs cnstate_vars to be initialized call init_decompcascade_bgc(bounds_proc, cnstate_vars, soilstate_vars) - else + else ! Note that init_decompcascade_cn needs cnstate_vars to be initialized call init_decompcascade_cn(bounds_proc, cnstate_vars) end if @@ -615,14 +616,14 @@ subroutine initialize2( ) ! ------------------------------------------------------------------------ ! The time manager needs to be initialized before thes called is made, since - ! the step size is needed. + ! the step size is needed. call t_startf('init_accflds') call atm2lnd_vars%initAccBuffer(bounds_proc) - + call top_as%InitAccBuffer(bounds_proc) - + call top_af%InitAccBuffer(bounds_proc) call veg_es%InitAccBuffer(bounds_proc) @@ -640,7 +641,7 @@ subroutine initialize2( ) call t_stopf('init_accflds') ! ------------------------------------------------------------------------ - ! Initializate dynamic subgrid weights (for prescribed transient Patches, + ! Initializate dynamic subgrid weights (for prescribed transient Patches, ! and/or dynamic landunits); note that these will be overwritten in a ! restart run ! ------------------------------------------------------------------------ @@ -661,13 +662,13 @@ subroutine initialize2( ) end if if (use_cn .and. n_drydep > 0 .and. drydep_method == DD_XLND) then - ! Must do this also when drydeposition is used so that estimates of monthly + ! Must do this also when drydeposition is used so that estimates of monthly ! differences in LAI can be computed call SatellitePhenologyInit(bounds_proc) end if ! ------------------------------------------------------------------------ - ! On restart only - process the history namelist. + ! On restart only - process the history namelist. ! ------------------------------------------------------------------------ ! Later the namelist from the restart file will be used. This allows basic @@ -678,7 +679,7 @@ subroutine initialize2( ) end if ! ------------------------------------------------------------------------ - ! Read restart/initial info + ! Read restart/initial info ! ------------------------------------------------------------------------ if (nsrest == nsrStartup) then @@ -688,13 +689,13 @@ subroutine initialize2( ) if (masterproc) then write(iulog,*)'Using cold start initial conditions ' end if - else + else if (masterproc) then write(iulog,*)'Interpolating initial conditions from ',trim(finidat_interp_source),& ' and creating new initial conditions ', trim(finidat_interp_dest) end if end if - else + else if (masterproc) then write(iulog,*)'Reading initial conditions from ',trim(finidat) end if @@ -727,7 +728,7 @@ subroutine initialize2( ) alm_fates, glc2lnd_vars, crop_vars) end if - + ! ------------------------------------------------------------------------ ! If appropriate, create interpolated initial conditions ! ------------------------------------------------------------------------ @@ -778,7 +779,7 @@ subroutine initialize2( ) ep_betr, & alm_fates, glc2lnd_vars, crop_vars) - ! Reset finidat to now be finidat_interp_dest + ! Reset finidat to now be finidat_interp_dest ! (to be compatible with routines still using finidat) finidat = trim(finidat_interp_dest) @@ -794,7 +795,7 @@ subroutine initialize2( ) if(use_betr)then call ep_betr%set_active(bounds_proc, col_pp) - endif + endif ! ------------------------------------------------------------------------ ! Initialize nitrogen deposition ! ------------------------------------------------------------------------ @@ -805,7 +806,7 @@ subroutine initialize2( ) call ndep_interp(bounds_proc, atm2lnd_vars) call t_stopf('init_ndep') end if - + ! ------------------------------------------------------------------------ ! Initialize phosphorus deposition ! ------------------------------------------------------------------------ @@ -816,14 +817,14 @@ subroutine initialize2( ) call pdep_interp(bounds_proc, atm2lnd_vars) call t_stopf('init_pdep') end if - + ! ------------------------------------------------------------------------ - ! Initialize active history fields. + ! Initialize active history fields. ! ------------------------------------------------------------------------ - ! This is only done if not a restart run. If a restart run, then this - ! information has already been obtained from the restart data read above. + ! This is only done if not a restart run. If a restart run, then this + ! information has already been obtained from the restart data read above. ! Note that routine hist_htapes_build needs time manager information, ! so this call must be made after the restart information has been read. @@ -836,7 +837,7 @@ subroutine initialize2( ) ! ------------------------------------------------------------------------ ! The following is called for both initial and restart runs and must - ! must be called after the restart file is read + ! must be called after the restart file is read call atm2lnd_vars%initAccVars(bounds_proc) call top_as%InitAccVars(bounds_proc) @@ -848,11 +849,11 @@ subroutine initialize2( ) end if call cnstate_vars%initAccVars(bounds_proc) - !------------------------------------------------------------ + !------------------------------------------------------------ ! Read monthly vegetation - !------------------------------------------------------------ + !------------------------------------------------------------ - ! Even if CN is on, and dry-deposition is active, read CLMSP annual vegetation + ! Even if CN is on, and dry-deposition is active, read CLMSP annual vegetation ! to get estimates of monthly LAI if ( n_drydep > 0 .and. drydep_method == DD_XLND )then @@ -862,11 +863,15 @@ subroutine initialize2( ) ! This needs to be done even if CN is on! call interpMonthlyVeg(bounds_proc, canopystate_vars) end if + elseif ( use_fates_sp ) then + ! If fates has satellite phenology enabled, get the monthly veg values + ! prior to the first call to SatellitePhenology() + call interpMonthlyVeg(bounds_proc, canopystate_inst) end if - !------------------------------------------------------------ + !------------------------------------------------------------ ! Determine gridcell averaged properties to send to atm - !------------------------------------------------------------ + !------------------------------------------------------------ if (nsrest == nsrStartup) then call t_startf('init_map2gc') @@ -874,11 +879,11 @@ subroutine initialize2( ) call t_stopf('init_map2gc') end if - !------------------------------------------------------------ + !------------------------------------------------------------ ! Initialize sno export state to send to glc - !------------------------------------------------------------ + !------------------------------------------------------------ - if (create_glacier_mec_landunit) then + if (create_glacier_mec_landunit) then !$OMP PARALLEL DO PRIVATE (nc, bounds_clump) do nc = 1,nclumps call get_clump_bounds(nc, bounds_clump) @@ -892,9 +897,9 @@ subroutine initialize2( ) !$OMP END PARALLEL DO end if - !------------------------------------------------------------ + !------------------------------------------------------------ ! Deallocate wt_nat_patch - !------------------------------------------------------------ + !------------------------------------------------------------ ! wt_nat_patch was allocated in initialize1, but needed to be kept around through ! initialize2 for some consistency checking; now it can be deallocated @@ -904,15 +909,26 @@ subroutine initialize2( ) ! -------------------------------------------------------------- ! Initialise the FATES model state structure cold-start ! -------------------------------------------------------------- - + if ( use_fates .and. .not.is_restart() .and. finidat == ' ') then + ! If fates is using satellite phenology mode, make sure to call the SatellitePhenology + ! procedure prior to init_coldstart which will eventually call leaf_area_profile + if ( use_fates_sp ) then + !$OMP PARALLEL DO PRIVATE (nc, bounds_clump) + do nc = 1,nclumps + call get_clump_bounds(nc, bounds_clump) + call SatellitePhenology(bounds_clump, filter(nc)%num_nolakep, filter(nc)%nolakep, & + water_inst%waterdiagnosticbulk_inst, canopystate_inst) + end do + !$OMP END PARALLEL DO + end if call alm_fates%init_coldstart(canopystate_vars, soilstate_vars, frictionvel_vars) end if ! topo_glc_mec was allocated in initialize1, but needed to be kept around through ! initialize2 because it is used to initialize other variables; now it can be ! deallocated - + ! topo_glc_mec was allocated in initialize1, but needed to be kept around through @@ -934,9 +950,9 @@ subroutine initialize2( ) call t_stopf('init_elm_interface_data & pflotran') !------------------------------------------------------------ - !------------------------------------------------------------ + !------------------------------------------------------------ ! Write log output for end of initialization - !------------------------------------------------------------ + !------------------------------------------------------------ call t_startf('init_wlog') if (masterproc) then From 26346b25b5544c1161d7b3cf2743e791194256e8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 22:30:00 -0700 Subject: [PATCH 039/301] adding fates sp mode flags to controMod --- components/elm/src/main/controlMod.F90 | 94 ++++++++++++++------------ 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index e7a1d5116045..c83bf0775f02 100644 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -10,7 +10,7 @@ module controlMod ! Display the file in a browser to see it neatly formatted in html. ! ! !USES: - use elm_varctl + use elm_varctl use shr_kind_mod , only: r8 => shr_kind_r8, SHR_KIND_CL use shr_nl_mod , only: shr_nl_find_group_name use shr_const_mod , only: SHR_CONST_CDAY @@ -19,20 +19,20 @@ module controlMod use spmdMod , only: masterproc use decompMod , only: clump_pproc use elm_varpar , only: maxpatch_pft, maxpatch_glcmec, more_vertlayers - use histFileMod , only: max_tapes, max_namlen - use histFileMod , only: hist_empty_htapes, hist_dov2xy, hist_avgflag_pertape, hist_type1d_pertape + use histFileMod , only: max_tapes, max_namlen + use histFileMod , only: hist_empty_htapes, hist_dov2xy, hist_avgflag_pertape, hist_type1d_pertape use histFileMod , only: hist_nhtfrq, hist_ndens, hist_mfilt, hist_fincl1, hist_fincl2, hist_fincl3 use histFileMod , only: hist_fincl4, hist_fincl5, hist_fincl6, hist_fexcl1, hist_fexcl2, hist_fexcl3 use histFileMod , only: hist_fexcl4, hist_fexcl5, hist_fexcl6 - use LakeCon , only: deepmixing_depthcrit, deepmixing_mixfact + use LakeCon , only: deepmixing_depthcrit, deepmixing_mixfact use AllocationMod , only: suplnitro use AllocationMod , only: suplphos use ColumnDataType , only: nfix_timeconst use NitrifDenitrifMod , only: no_frozen_nitrif_denitrif use C14DecayMod , only: use_c14_bombspike, atm_c14_filename use SoilLittVertTranspMod , only: som_adv_flux, max_depth_cryoturb - use VerticalProfileMod , only: exponential_rooting_profile, rootprof_exp - use VerticalProfileMod , only: surfprof_exp, pftspecific_rootingprofile + use VerticalProfileMod , only: exponential_rooting_profile, rootprof_exp + use VerticalProfileMod , only: surfprof_exp, pftspecific_rootingprofile use SharedParamsMod , only: anoxia_wtsat use CanopyStateType , only: perchroot, perchroot_alt use CanopyHydrologyMod , only: CanopyHydrology_readnl @@ -41,7 +41,7 @@ module controlMod use FrictionVelocityMod , only: implicit_stress, atm_gustiness use elm_varcon , only: h2osno_max use elm_varctl , only: use_dynroot - use AllocationMod , only: nu_com_phosphatase,nu_com_nfix + use AllocationMod , only: nu_com_phosphatase,nu_com_nfix use elm_varctl , only: nu_com, use_var_soil_thick use seq_drydep_mod , only: drydep_method, DD_XLND, n_drydep use elm_varctl , only: forest_fert_exp @@ -190,7 +190,7 @@ subroutine control_init( ) namelist /elm_inparm/ & use_pheno_flux_limiter - + namelist /elm_inparm/ & suplnitro,suplphos namelist /elm_inparm/ & @@ -214,9 +214,9 @@ subroutine control_init( ) ! lake_melt_icealb is of dimension numrad ! Glacier_mec info - namelist /elm_inparm/ & + namelist /elm_inparm/ & maxpatch_glcmec, glc_smb, glc_do_dynglacier, glcmec_downscale_rain_snow_convert, & - glcmec_downscale_longwave, glc_snow_persistence_max_days, glc_grid, fglcmask + glcmec_downscale_longwave, glc_snow_persistence_max_days, glc_grid, fglcmask ! Other options @@ -238,7 +238,7 @@ subroutine control_init( ) som_adv_flux, max_depth_cryoturb ! C and N input vertical profiles - namelist /elm_inparm/ & + namelist /elm_inparm/ & exponential_rooting_profile, rootprof_exp, surfprof_exp, pftspecific_rootingprofile namelist /elm_inparm / no_frozen_nitrif_denitrif @@ -252,17 +252,19 @@ subroutine control_init( ) use_fates_ed_prescribed_phys, & use_fates_inventory_init, & fates_inventory_ctrl_filename, & - use_fates_fixed_biogeog, & + use_fates_fixed_biogeog, & + use_fates_nocomp, & + use_fates_sp, & fates_parteh_mode namelist /elm_inparm / use_betr - + namelist /elm_inparm / use_lai_streams namelist /elm_inparm/ & use_c14_bombspike, atm_c14_filename - ! All old cpp-ifdefs are below and have been converted to namelist variables + ! All old cpp-ifdefs are below and have been converted to namelist variables ! max number of plant functional types in naturally vegetated landunit namelist /elm_inparm/ maxpatch_pft @@ -328,7 +330,7 @@ subroutine control_init( ) if (masterproc) then ! ---------------------------------------------------------------------- - ! Read namelist from standard input. + ! Read namelist from standard input. ! ---------------------------------------------------------------------- if ( len_trim(NLFilename) == 0 )then @@ -344,7 +346,7 @@ subroutine control_init( ) call endrun(msg='ERROR reading elm_inparm namelist'//errMsg(__FILE__, __LINE__)) end if end if - + call relavu( unitn ) ! ---------------------------------------------------------------------- @@ -366,7 +368,7 @@ subroutine control_init( ) endif end do - ! Override start-type (can only override to branch (3) and only + ! Override start-type (can only override to branch (3) and only ! if the driver is a startup type if ( override_nsrest /= nsrest )then if ( override_nsrest /= nsrBranch .and. nsrest /= nsrStartup )then @@ -376,14 +378,14 @@ subroutine control_init( ) end if call elm_varctl_set( nsrest_in=override_nsrest ) end if - + if (maxpatch_glcmec > 0) then create_glacier_mec_landunit = .true. else create_glacier_mec_landunit = .false. end if - ! Check compatibility with the FATES model + ! Check compatibility with the FATES model if ( use_fates ) then use_voc = .false. @@ -392,12 +394,12 @@ subroutine control_init( ) call endrun(msg=' ERROR: use_cn and use_fates cannot both be set to true.'//& errMsg(__FILE__, __LINE__)) end if - + if ( use_crop ) then call endrun(msg=' ERROR: use_crop and use_fates cannot both be set to true.'//& errMsg(__FILE__, __LINE__)) end if - + if ( n_drydep > 0 .and. drydep_method /= DD_XLND ) then call endrun(msg=' ERROR: dry deposition via ML Welsey is not compatible with FATES.'//& errMsg(__FILE__, __LINE__)) @@ -439,7 +441,7 @@ subroutine control_init( ) call endrun(msg=' ERROR:: CROP and C13/C14 can NOT be on at the same time'//& errMsg(__FILE__, __LINE__)) end if - + if (use_crop .and. .not. create_crop_landunit) then call endrun(msg=' ERROR: prognostic crop Patches require create_crop_landunit=.true.'//& errMsg(__FILE__, __LINE__)) @@ -449,8 +451,8 @@ subroutine control_init( ) call endrun(msg=' ERROR: ero_ccycle = .true. requires erosion model active.'//& errMsg(__FILE__, __LINE__)) end if - - if (use_lch4 .and. use_vertsoilc) then + + if (use_lch4 .and. use_vertsoilc) then anoxia = .true. else anoxia = .false. @@ -489,7 +491,7 @@ subroutine control_init( ) !For future version, I suggest to put the following two calls inside their !own modules, which are called from their own initializing methods call init_hydrology( NLFilename ) - + call CanopyHydrology_readnl( NLFilename ) ! ---------------------------------------------------------------------- @@ -497,14 +499,14 @@ subroutine control_init( ) ! ---------------------------------------------------------------------- call control_spmd() - + if (use_pflotran) then call elm_pf_readnl(NLFilename) end if if (use_betr) then call betr_readNL( NLFilename, use_c13, use_c14) - endif + endif ! ---------------------------------------------------------------------- ! consistency checks @@ -519,7 +521,7 @@ subroutine control_init( ) ! Check on run type if (nsrest == iundef) then - call endrun(msg=' ERROR:: must set nsrest'//& + call endrun(msg=' ERROR:: must set nsrest'//& errMsg(__FILE__, __LINE__)) end if if (nsrest == nsrBranch .and. nrevsn == ' ') then @@ -529,7 +531,7 @@ subroutine control_init( ) ! Consistency settings for co2_ppvm if ( (co2_ppmv <= 0.0_r8) .or. (co2_ppmv > 3000.0_r8) ) then - call endrun(msg=' ERROR: co2_ppmv is out of a reasonable range'//& + call endrun(msg=' ERROR: co2_ppmv is out of a reasonable range'//& errMsg(__FILE__, __LINE__)) end if @@ -597,9 +599,9 @@ end subroutine control_init subroutine control_spmd() ! ! !DESCRIPTION: - ! Distribute namelist data all processors. All program i/o is - ! funnelled through the master processor. Processor 0 either - ! reads restart/history data from the disk and distributes + ! Distribute namelist data all processors. All program i/o is + ! funnelled through the master processor. Processor 0 either + ! reads restart/history data from the disk and distributes ! it to all processors, or collects data from ! all processors and writes it to disk. ! @@ -635,7 +637,7 @@ subroutine control_spmd() call mpi_bcast (use_crop, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_voc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_snicar_frc, 1, MPI_LOGICAL, 0, mpicom, ier) - call mpi_bcast (use_snicar_ad, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (use_snicar_ad, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_vancouver, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_mexicocity, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_noio, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -671,7 +673,7 @@ subroutine control_spmd() ! BGC call mpi_bcast (co2_type, len(co2_type), MPI_CHARACTER, 0, mpicom, ier) - + if (use_cn .or. use_fates) then call mpi_bcast (suplnitro, len(suplnitro), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (nfix_timeconst, 1, MPI_REAL8, 0, mpicom, ier) @@ -684,7 +686,7 @@ subroutine control_spmd() ! This group of flags has always been sent for both use_cn or otherwise ! So I did not change this. It does not seem a liability to broadcast ! in all run types (RGK 07-2020) - + call mpi_bcast (suplphos, len(suplphos), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (nu_com, len(nu_com), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (nu_com_phosphatase, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -702,7 +704,7 @@ subroutine control_spmd() call mpi_bcast (use_c13, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_c14, 1, MPI_LOGICAL, 0, mpicom, ier) - + call mpi_bcast (fates_spitfire_mode, 1, MPI_INTEGER, 0, mpicom, ier) call mpi_bcast (fates_paramfile, len(fates_paramfile) , MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (use_fates_logging, 1, MPI_LOGICAL, 0, mpicom, ier) @@ -710,6 +712,8 @@ subroutine control_spmd() call mpi_bcast (use_fates_cohort_age_tracking, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_ed_st3, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_fixed_biogeog, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (use_fates_nocomp, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (use_fates_sp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_ed_prescribed_phys, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_fates_inventory_init, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (fates_inventory_ctrl_filename, len(fates_inventory_ctrl_filename), & @@ -736,7 +740,7 @@ subroutine control_spmd() call mpi_bcast (pftspecific_rootingprofile, 1, MPI_LOGICAL, 0, mpicom, ier) end if - if ((use_cn .or. use_fates)) then + if ((use_cn .or. use_fates)) then call mpi_bcast (no_frozen_nitrif_denitrif, 1, MPI_LOGICAL, 0, mpicom, ier) end if @@ -824,7 +828,7 @@ subroutine control_spmd() call mpi_bcast (use_elm_interface, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_elm_bgc, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (use_pflotran, 1, MPI_LOGICAL, 0, mpicom, ier) - + !cpl_bypass call mpi_bcast (metdata_type, len(metdata_type), MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (metdata_bypass, len(metdata_bypass), MPI_CHARACTER, 0, mpicom, ier) @@ -935,13 +939,13 @@ subroutine control_print () write(iulog,*) ' Supplemental Nitrogen mode is set to run over Patches: ', & trim(suplnitro) end if - + if (nfix_timeconst /= 0._r8) then write(iulog,*) ' nfix_timeconst, timescale for smoothing npp in N fixation term: ', nfix_timeconst else write(iulog,*) ' nfix_timeconst == zero, use standard N fixation scheme. ' end if - + write(iulog,*) ' spinup_state, (0 = normal mode; 1 = AD spinup) : ', spinup_state if ( spinup_state .eq. 0 ) then write(iulog,*) ' model is currently NOT in AD spinup mode.' @@ -953,7 +957,7 @@ subroutine control_print () call endrun(msg=' error: spinup_state can only have integer value of 0 or 1'//& errMsg(__FILE__, __LINE__)) end if - + write(iulog,*) ' override_bgc_restart_mismatch_dump : ', override_bgc_restart_mismatch_dump end if if (suplphos /= suplpNon)then @@ -964,14 +968,14 @@ subroutine control_print () if (use_cn .and. use_vertsoilc) then write(iulog, *) ' som_adv_flux, the advection term in soil mixing (m/s) : ', som_adv_flux write(iulog, *) ' max_depth_cryoturb (m) : ', max_depth_cryoturb - + write(iulog, *) ' exponential_rooting_profile : ', exponential_rooting_profile write(iulog, *) ' rootprof_exp : ', rootprof_exp write(iulog, *) ' surfprof_exp : ', surfprof_exp write(iulog, *) ' pftspecific_rootingprofile : ', pftspecific_rootingprofile write(iulog, *) ' dynamic roots : ', use_dynroot end if - + if (use_cn) then write(iulog, *) ' use_c13 : ', use_c13 write(iulog, *) ' use_c14 : ', use_c14 @@ -994,7 +998,7 @@ subroutine control_print () write(iulog,*) ' glc number of elevation classes =', maxpatch_glcmec write(iulog,*) ' glc grid for glacier mask file = ',trim(glc_grid) write(iulog,*) ' glc glacier mask file = ',trim(fglcmask) - + write(iulog,*) ' Max snow depth (mm) =', h2osno_max if (glcmec_downscale_rain_snow_convert) then write(iulog,*) ' Rain and snow will be converted based on surface temperature' @@ -1083,6 +1087,8 @@ subroutine control_print () write(iulog, *) ' use_fates_ed_prescribed_phys = ',use_fates_ed_prescribed_phys write(iulog, *) ' use_fates_inventory_init = ',use_fates_inventory_init write(iulog, *) ' use_fates_fixed_biogeog = ', use_fates_fixed_biogeog + write(iulog, *) ' use_fates_nocomp = ', use_fates_nocomp + write(iulog, *) ' use_fates_sp = ', use_fates_sp write(iulog, *) ' fates_inventory_ctrl_filename = ',fates_inventory_ctrl_filename end if From 4073cbe25d29bfd18d0c4c7060df45d204b065da Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 17 Aug 2021 22:47:12 -0700 Subject: [PATCH 040/301] adding sp_pftorder_index to Veg type --- .../elm/src/data_types/VegetationType.F90 | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/components/elm/src/data_types/VegetationType.F90 b/components/elm/src/data_types/VegetationType.F90 index 7b249b37c9c6..c077bb1fe0e4 100644 --- a/components/elm/src/data_types/VegetationType.F90 +++ b/components/elm/src/data_types/VegetationType.F90 @@ -64,15 +64,16 @@ module VegetationType logical , pointer :: active (:) => null() ! true=>do computations on this patch ! Fates relevant types - logical , pointer :: is_veg (:) => null() ! This is an ACTIVE fates patch - logical , pointer :: is_bareground (:) => null() ! ? - real(r8), pointer :: wt_ed (:) => null() ! TODO mv ? can this be removed - logical , pointer :: is_fates (:) => null() ! true for patch vector space reserved - ! for FATES. - ! this is static and is true for all - ! patches within fates jurisdiction - ! including patches which are not currently - ! associated with a FATES linked-list patch + logical , pointer :: is_veg (:) => null() ! This is an ACTIVE fates patch + logical , pointer :: is_bareground (:) => null() ! ? + real(r8), pointer :: wt_ed (:) => null() ! TODO mv ? can this be removed + real(r8), pointer :: sp_pftorder_index (:) => null() ! index to map 'p' onto the order of ED patches in SP mode. + logical , pointer :: is_fates (:) => null() ! true for patch vector space reserved + ! for FATES. + ! this is static and is true for all + ! patches within fates jurisdiction + ! including patches which are not currently + ! associated with a FATES linked-list patch contains procedure, public :: Init => veg_pp_init @@ -119,6 +120,7 @@ subroutine veg_pp_init(this, begp, endp) allocate(this%is_veg (begp:endp)); this%is_veg (:) = .false. allocate(this%is_bareground (begp:endp)); this%is_bareground (:) = .false. allocate(this%wt_ed (begp:endp)); this%wt_ed (:) = nan + allocate(this%sp_pftorder_index (begp:endp)); this%sp_pftorder_index (:) = nan end if end subroutine veg_pp_init @@ -147,6 +149,7 @@ subroutine veg_pp_clean(this) deallocate(this%is_veg) deallocate(this%is_bareground) deallocate(this%wt_ed) + deallocate(this%sp_pftorder_index) end if end subroutine veg_pp_clean From 6f4bb9f0c4ef5574d164892c0017fdfdfaf7bfd2 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Fri, 20 Aug 2021 18:13:43 -0500 Subject: [PATCH 041/301] mods to init RJ runs with RJ qsat --- components/homme/src/test_src/dry_planar.F90 | 65 +++++++++++++++++--- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/components/homme/src/test_src/dry_planar.F90 b/components/homme/src/test_src/dry_planar.F90 index ffc9a5784ca1..ec1b9afa9d42 100644 --- a/components/homme/src/test_src/dry_planar.F90 +++ b/components/homme/src/test_src/dry_planar.F90 @@ -26,8 +26,12 @@ module dry_planar_tests p0 = 100000.d0, &! reference pressure (Pa) kappa = Rd/cp +!consts for kessler-defined qsat real(rl), parameter :: bubble_const1=3.8, bubble_const2=17.27, bubble_const3=273.0, bubble_const4=36.0 - +!consts for RJ-defined qsat + real(rl), parameter :: bubble_t0_const=273.16, bubble_epsilo=Rgas/Rwater_vapor, bubble_e0=610.78 +!this one is used in dcmip as 2.5e6 instead of the one in cam, 2.501e6 + real(rl), parameter :: bubble_latvap=2.5e6 real(rl):: zi(nlevp), zm(nlev) ! z coordinates real(rl):: ddn_hyai(nlevp), ddn_hybi(nlevp) ! vertical derivativess of hybrid coefficients @@ -209,7 +213,8 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) real(rl):: zero_mid_init(np,np,nlev),zero_int_init(np,np,nlevp), & dp_init(np,np,nlev),ps_init(np,np), & phis_init(np,np,nlevp),t_init(np,np,nlev),p_init(np,np,nlev), & - zi_init(np,np,nlevp), zm_init(np,np,nlev) + zi_init(np,np,nlevp), zm_init(np,np,nlev), & + rh if (qsize < 3 .and. bubble_moist) then call abortmp('planar moist bubble requires at least 3 tracers') @@ -228,7 +233,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) print *, 'bubble_zradius', bubble_zradius print *, 'bubble_cosine', bubble_cosine print *, 'bubble_moist', bubble_moist - print *, 'bubble_moist_dq', bubble_moist_dq + print *, 'bubble_moist_dq (RH param)', bubble_moist_dq print *, 'bubble_prec_type (0 is Kessler (default), 1 is RJ)', bubble_prec_type endif @@ -287,9 +292,24 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) !for the background state if (bubble_moist) then !build specific humidity at saturation qs as in dcmip2016-kessler + + if(bubble_prec_type == 0) then + !kessler do k=1,nlevp qi_s(k) = bubble_const1 / pi(k) * exp( bubble_const2 * (Ti(k) - bubble_const3) / ( Ti(k) - bubble_const4 ) ) enddo + elseif(bubble_prec_type == 1) then + !RJ physics + do k=1,nlevp + !dcmip line + !qsat = epsilo * e0 / p(k) * exp(-(latvap/rh2o) * ((one/t(k))-(one/T0))) + qi_s(k) = bubble_epsilo * bubble_e0 / pi(k) * & + exp(-(bubble_latvap/Rwater_vapor) * ((1.0/Ti(k))-(1.0/bubble_t0_const))) + enddo + else + call abortmp('planar moist bubble bubble_prec_type should be 0 or 1') + endif + else qi_s(1:nlevp) = 0.0 endif @@ -330,9 +350,10 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) (y-bubble_xycenter) * (y-bubble_xycenter) / bubble_xyradius / bubble_xyradius ) endif - !set pot. temperature on interfaces - if ( rr < 1.0 ) then +#if 0 +!old + if ( rr < 1.0 ) then if (bubble_cosine) then offset = cos(rr*dd_pi / 2.0 ) th0(k) = bubble_T0 + bubble_dT * offset @@ -342,15 +363,33 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) th0(k) = bubble_T0 + bubble_dT qi(k) = qi_s(k) + bubble_moist_dq endif - else - !set to reference profile th0(k) = bubble_T0 qi(k) = qi_s(k) - endif - +#else +!new + rh = 0.75 + if ( rr < 1.0 ) then + if (bubble_cosine) then + offset = cos(rr*dd_pi / 2.0 ) + th0(k) = bubble_T0 + bubble_dT * offset + rh = rh + bubble_moist_dq * offset + !rh=1.0 + else + !0/1 nonsmooth function + th0(k) = bubble_T0 + bubble_dT + rh = rh + bubble_moist_dq + !rh=1.0 + endif + else + !set to reference profile + th0(k) = bubble_T0 + !rh = ... + endif + qi(k) = rh * qi_s(k) +#endif enddo ! k loop !set theta on midlevels and then T from theta, exner @@ -361,6 +400,9 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) if (bubble_moist) then do k=1,nlev elem(ie)%state%Q(i,j,k,1) = ( qi(k) + qi(k+1) ) / 2.0 + +!this is not good FOR KESSLER, temp hack +! elem(ie)%state%Q(i,j,k,2) = ( qi_s(k) + qi_s(k+1) ) / 2.0 enddo else elem(ie)%state%Q(i,j,:,1) = 0.0 @@ -378,7 +420,10 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) !Q (np,np,nlev,qsize_d) !Qdp (np,np,nlev,qsize_d,2) if (bubble_moist) then + ii=2 +!!!! not KESSLER!, temp hack +! ii=3 else ii=1 endif @@ -390,7 +435,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) elem(ie)%state%Q(:,:,:,ii:qsize) = 0.0 !sets hydro phi from (perturbed) theta and pressure, checks for hydrostatic balance after that, saves a state - !call tests_finalize(elem(ie),hvcoord) + call tests_finalize(elem(ie),hvcoord) enddo end subroutine bubble_init From fc8ad50c0d5f05b700e576a2fe3f3d1d5728eb07 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 22 Aug 2021 15:36:09 -0500 Subject: [PATCH 042/301] reorg and new nl variable --- components/homme/src/share/control_mod.F90 | 7 ++-- components/homme/src/share/namelist_mod.F90 | 9 +++-- components/homme/src/test_src/dry_planar.F90 | 40 +++----------------- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index bd535a5c0aad..eea85e8e1faf 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -269,9 +269,10 @@ module control_mod real (kind=real_kind), public :: bubble_xyradius = 2000.0!bubble radius along x or y axis real (kind=real_kind), public :: bubble_zradius = 1500.0 !bubble radius along z axis logical, public :: bubble_cosine = .TRUE. !bubble uniform or cosine - logical, public :: bubble_moist = .FALSE. ! - real (kind=real_kind), public :: bubble_moist_dq = 0.0 !bubble dQ parameter - integer, public :: bubble_prec_type = 0 !0 kessler, 1 rj + logical, public :: bubble_moist = .FALSE. ! + real (kind=real_kind), public :: bubble_moist_drh = 0.0 !bubble dRH parameter + real (kind=real_kind), public :: bubble_rh_background = 0.0 !bubble RH parameter + integer, public :: bubble_prec_type = 0 !0 kessler, 1 rj logical, protected :: case_planar_bubble = .FALSE. public :: set_planar_defaults diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index c4dd7787886e..9cdda69eb07b 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -118,7 +118,8 @@ module namelist_mod bubble_zradius, & bubble_cosine, & bubble_moist, & - bubble_moist_dq, & + bubble_moist_drh, & + bubble_rh_background, & bubble_prec_type, & case_planar_bubble #endif @@ -352,7 +353,8 @@ subroutine readnl(par) bubble_zradius, & bubble_cosine, & bubble_moist, & - bubble_moist_dq, & + bubble_moist_drh, & + bubble_rh_background, & bubble_prec_type namelist /vert_nl/ & vform, & @@ -835,7 +837,8 @@ subroutine readnl(par) call MPI_bcast(bubble_zradius ,1,MPIreal_t ,par%root,par%comm,ierr) call MPI_bcast(bubble_cosine ,1,MPIlogical_t,par%root,par%comm,ierr) call MPI_bcast(bubble_moist ,1,MPIlogical_t,par%root,par%comm,ierr) - call MPI_bcast(bubble_moist_dq ,1,MPIreal_t,par%root,par%comm,ierr) + call MPI_bcast(bubble_moist_drh ,1,MPIreal_t,par%root,par%comm,ierr) + call MPI_bcast(bubble_rh_background ,1,MPIreal_t,par%root,par%comm,ierr) call MPI_bcast(bubble_prec_type, 1, MPIinteger_t, par%root,par%comm,ierr) call MPI_bcast(case_planar_bubble,1,MPIlogical_t,par%root,par%comm,ierr) diff --git a/components/homme/src/test_src/dry_planar.F90 b/components/homme/src/test_src/dry_planar.F90 index ec1b9afa9d42..935b9781e786 100644 --- a/components/homme/src/test_src/dry_planar.F90 +++ b/components/homme/src/test_src/dry_planar.F90 @@ -195,7 +195,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) use control_mod, only: bubble_T0, bubble_dT, bubble_xycenter, bubble_zcenter, bubble_ztop, & bubble_xyradius,bubble_zradius, bubble_cosine, & - bubble_moist, bubble_moist_dq, bubble_prec_type + bubble_moist, bubble_moist_drh, bubble_prec_type, bubble_rh_background use physical_constants, only: Lx, Ly, Sx, Sy use element_ops, only: set_elem_state @@ -233,7 +233,8 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) print *, 'bubble_zradius', bubble_zradius print *, 'bubble_cosine', bubble_cosine print *, 'bubble_moist', bubble_moist - print *, 'bubble_moist_dq (RH param)', bubble_moist_dq + print *, 'bubble_moist_drh', bubble_moist_drh + print *, 'bubble_rh_background', bubble_rh_background print *, 'bubble_prec_type (0 is Kessler (default), 1 is RJ)', bubble_prec_type endif @@ -330,6 +331,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) enddo; enddo ! set the rest of conditions for each element + rh=bubble_rh_background do ie = nets,nete do j=1,np; do i=1,np @@ -350,46 +352,22 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) (y-bubble_xycenter) * (y-bubble_xycenter) / bubble_xyradius / bubble_xyradius ) endif - -#if 0 -!old - if ( rr < 1.0 ) then - if (bubble_cosine) then - offset = cos(rr*dd_pi / 2.0 ) - th0(k) = bubble_T0 + bubble_dT * offset - qi(k) = qi_s(k) + bubble_moist_dq * offset - else - !0/1 nonsmooth function - th0(k) = bubble_T0 + bubble_dT - qi(k) = qi_s(k) + bubble_moist_dq - endif - else - !set to reference profile - th0(k) = bubble_T0 - qi(k) = qi_s(k) - endif -#else -!new rh = 0.75 if ( rr < 1.0 ) then if (bubble_cosine) then offset = cos(rr*dd_pi / 2.0 ) th0(k) = bubble_T0 + bubble_dT * offset - rh = rh + bubble_moist_dq * offset - !rh=1.0 + rh = rh + bubble_moist_drh * offset else !0/1 nonsmooth function th0(k) = bubble_T0 + bubble_dT - rh = rh + bubble_moist_dq - !rh=1.0 + rh = rh + bubble_moist_drh endif else !set to reference profile th0(k) = bubble_T0 - !rh = ... endif qi(k) = rh * qi_s(k) -#endif enddo ! k loop !set theta on midlevels and then T from theta, exner @@ -400,9 +378,6 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) if (bubble_moist) then do k=1,nlev elem(ie)%state%Q(i,j,k,1) = ( qi(k) + qi(k+1) ) / 2.0 - -!this is not good FOR KESSLER, temp hack -! elem(ie)%state%Q(i,j,k,2) = ( qi_s(k) + qi_s(k+1) ) / 2.0 enddo else elem(ie)%state%Q(i,j,:,1) = 0.0 @@ -420,10 +395,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) !Q (np,np,nlev,qsize_d) !Qdp (np,np,nlev,qsize_d,2) if (bubble_moist) then - ii=2 -!!!! not KESSLER!, temp hack -! ii=3 else ii=1 endif From a14ae9fe8be1dcc6c7dcc75b0ecd8f7f81d85983 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 23 Aug 2021 10:27:43 -0500 Subject: [PATCH 043/301] change test nl to match new paprameters --- .../homme/test/reg_test/namelists/thetanh-moist-bubble.nl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl b/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl index 6583961704cc..a2f4ac2f3298 100644 --- a/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl +++ b/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl @@ -55,7 +55,8 @@ bubble_moist=.true. bubble_T0=300.0 bubble_dT=2.0 - bubble_moist_dq=0.2 + bubble_rh_background=0.7 + bubble_moist_drh=0.2 bubble_prec_type=0 / &vert_nl From f8eb5c41d6ea2a9338dffaa16447b1a8eabb2ebb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 23 Aug 2021 16:21:00 -0500 Subject: [PATCH 044/301] fixing hardcoded rh --- components/homme/src/test_src/dry_planar.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/components/homme/src/test_src/dry_planar.F90 b/components/homme/src/test_src/dry_planar.F90 index 935b9781e786..d3c4cc308fb2 100644 --- a/components/homme/src/test_src/dry_planar.F90 +++ b/components/homme/src/test_src/dry_planar.F90 @@ -352,7 +352,6 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) (y-bubble_xycenter) * (y-bubble_xycenter) / bubble_xyradius / bubble_xyradius ) endif - rh = 0.75 if ( rr < 1.0 ) then if (bubble_cosine) then offset = cos(rr*dd_pi / 2.0 ) From 3966ef21d0a4f554510a0fd9a76c409dd157e325 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 24 Aug 2021 13:22:49 -0500 Subject: [PATCH 045/301] yet another fix for background rh --- components/homme/src/test_src/dry_planar.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/test_src/dry_planar.F90 b/components/homme/src/test_src/dry_planar.F90 index d3c4cc308fb2..b640c778f6c1 100644 --- a/components/homme/src/test_src/dry_planar.F90 +++ b/components/homme/src/test_src/dry_planar.F90 @@ -331,7 +331,6 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) enddo; enddo ! set the rest of conditions for each element - rh=bubble_rh_background do ie = nets,nete do j=1,np; do i=1,np @@ -352,6 +351,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) (y-bubble_xycenter) * (y-bubble_xycenter) / bubble_xyradius / bubble_xyradius ) endif + rh=bubble_rh_background if ( rr < 1.0 ) then if (bubble_cosine) then offset = cos(rr*dd_pi / 2.0 ) From 3c24edc07890a5f9dfea8911acd8c028740e7adf Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Wed, 8 Sep 2021 10:17:52 -0700 Subject: [PATCH 046/301] Zero calvingThickness in li_calve_ice Zero calvingThickness in calving driver routine. This is necessary for combining von Mises or eigencalving with damage threshold calving. --- .../src/mode_forward/mpas_li_calving.F | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F index bfec77abc081..d15551e61095 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_calving.F @@ -192,7 +192,18 @@ subroutine li_calve_ice(domain, err) block => block % next end do endif - + + ! Zero calvingThickness here instead of in individual subroutines. + ! This is necessary when using damage threshold calving with other + ! calving routines + block => domain % blocklist + do while (associated(block)) + call mpas_pool_get_subpool(block % structs, 'geometry', geometryPool) + call mpas_pool_get_array(geometryPool, 'calvingThickness', calvingThickness) + calvingThickness(:) = 0.0_RKIND + + block => block % next + end do ! compute calvingThickness based on the calving_config option if (trim(config_calving) == 'thickness_threshold') then @@ -3006,8 +3017,6 @@ subroutine apply_calving_damage_threshold(meshPool, geometryPool, scratchPool, d growMask => growMaskField % array growMask(:) = 0 - calvingThickness(:) = 0.0_RKIND - ! define seed and grow masks for flood fill. where ( li_mask_is_dynamic_margin(cellMask) .and. (damage .ge. config_damage_calving_threshold) ) seedMask = 1 From b54679f0f0774415616c53b73fd7f8b81f90b8bf Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 13 Sep 2021 10:17:32 -0500 Subject: [PATCH 047/301] remove test_finalize call from init --- components/homme/src/test_src/dry_planar.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/test_src/dry_planar.F90 b/components/homme/src/test_src/dry_planar.F90 index b640c778f6c1..93a37252e0c8 100644 --- a/components/homme/src/test_src/dry_planar.F90 +++ b/components/homme/src/test_src/dry_planar.F90 @@ -406,7 +406,7 @@ subroutine bubble_init(elem,hybrid,hvcoord,nets,nete,f) elem(ie)%state%Q(:,:,:,ii:qsize) = 0.0 !sets hydro phi from (perturbed) theta and pressure, checks for hydrostatic balance after that, saves a state - call tests_finalize(elem(ie),hvcoord) + !call tests_finalize(elem(ie),hvcoord) enddo end subroutine bubble_init From 4ee59fe28d1fee2ced7fa02ec53c2990a5c63b72 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 13 Sep 2021 16:31:18 -0600 Subject: [PATCH 048/301] Homme: fixes and more tests for ForcingFunctor * Add BFB unit tests for adjustment=true * Fix several bugs, exposed by the new tests --- .../homme/src/share/prim_driver_base.F90 | 22 +- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 22 +- .../thetal_kokkos_ut/forcing_interface.F90 | 6 +- .../thetal_kokkos_ut/forcing_ut.cpp | 203 +++++++++--------- 4 files changed, 140 insertions(+), 113 deletions(-) diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 377d8a8f0486..a05530aee54c 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1570,7 +1570,7 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) logical, intent(in) :: adjustment ! local - integer :: i,j,k,ie,q + integer :: i,j,k,q real (kind=real_kind) :: fq real (kind=real_kind) :: dp(np,np,nlev), ps(np,np), dp_adj(np,np,nlev) real (kind=real_kind) :: phydro(np,np,nlev) ! hydrostatic pressure @@ -1586,6 +1586,13 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) real (kind=real_kind) :: dpnh_dp_i(np,np,nlevp) #endif +#ifdef HOMMEXX_BFB_TESTING + ! BFB comparison with C++ requires to perform the reduction + ! of FQ over the whole column *before* adding to ps + real (kind=real_kind) :: sum_fq(np,np) + sum_fq = 0 +#endif + #ifdef MODEL_THETA_L if (dt_remap_factor==0) then adjust_ps=.true. ! stay on reference levels for Eulerian case @@ -1604,7 +1611,6 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) ! after calling this routine, ps_v may not be valid and should not be used elem%state%ps_v(:,:,np1)=0 - #ifdef MODEL_THETA_L !compute temperatue and NH perturbation pressure before Q tendency do k=1,nlev @@ -1623,6 +1629,7 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) #endif if (adjustment) then + ! hard adjust Q from physics. negativity check done in physics do k=1,nlev do j=1,np @@ -1638,13 +1645,24 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) fq = dp(i,j,k)*( elem%derived%FQ(i,j,k,q) -& elem%state%Q(i,j,k,q)) ! force ps to conserve mass: +#ifdef HOMMEXX_BFB_TESTING + sum_fq(i,j) = sum_fq(i,j) + fq +#else ps(i,j)=ps(i,j) + fq +#endif dp_adj(i,j,k)=dp_adj(i,j,k) + fq ! ps = ps0+sum(dp(k)) endif enddo end do end do end do +#ifdef HOMMEXX_BFB_TESTING + do j=1,np + do i=1,np + ps(i,j) = ps(i,j) + sum_fq(i,j) + end do + end do +#endif else ! end of adjustment ! apply forcing to Qdp elem%derived%FQps(:,:)=0 diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index b680374665ef..44f0b9ebba32 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -312,7 +312,7 @@ class ForcingFunctor auto pnh = Homme::subview(m_pnh,kv.ie,igp,jgp); auto exner = Homme::subview(m_exner,kv.ie,igp,jgp); if (m_hydrostatic) { - auto p_i = Homme::subview(m_pi_i,kv.team_idx,igp,jgp); + auto p_i = Homme::subview(m_pi_i,kv.team_idx,igp,jgp); m_elem_ops.compute_hydrostatic_p(kv,dp,p_i,pnh); m_eos.compute_exner(kv,pnh,exner); } else { @@ -347,13 +347,17 @@ class ForcingFunctor auto qdp = Homme::subview(m_tracers.qdp,kv.ie,m_np1_qdp,0,igp,jgp); auto dp_adj = Homme::subview(m_dp_adj,kv.ie,igp,jgp); if (m_adjustment) { + Real added_mass = 0; Dispatch::parallel_reduce( kv.team, Kokkos::ThreadVectorRange(kv.team, NUM_PHYSICAL_LEV), [&](const int &k, Real &accumulator) { const int ilev = k / VECTOR_SIZE; const int ivec = k % VECTOR_SIZE; accumulator += dp(ilev)[ivec]*(fq(ilev)[ivec]-q(ilev)[ivec]); - },ps); + },added_mass); + Kokkos::single(Kokkos::PerThread(kv.team),[&](){ + ps += added_mass; + }); if (!m_adjust_ps) { Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { @@ -405,12 +409,12 @@ class ForcingFunctor qdp_ilev += compute_fqdt_pack(ilev,fq,qdp); } + // Update tracers concentration if (m_moist) { - dp(ilev) = dp_adj(ilev); + Q(ilev) = qdp_ilev/dp_adj(ilev); + } else { + Q(ilev) = qdp_ilev/dp(ilev); } - - // Update tracers concentration - Q(ilev) = qdp_ilev/dp(ilev); }); }); } @@ -442,11 +446,12 @@ class ForcingFunctor const Real ps = m_state.m_ps_v(kv.ie,m_np1,igp,jgp); if (m_moist) { - // Need to update pnh and exner. Currently, pnh is storing pnh-pi - + // Need to update dp, pnh and exner. Currently, pnh is storing pnh-pi if (m_adjust_ps) { Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { + dp(ilev) = dp_adj(ilev); + pnh(ilev) += m_hvcoord.ps0*m_hvcoord.hybrid_am(ilev) + ps *m_hvcoord.hybrid_bm(ilev); }); @@ -457,6 +462,7 @@ class ForcingFunctor Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { pnh(ilev) += exner(ilev); + dp(ilev) = dp_adj(ilev); }); } diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/forcing_interface.F90 index 6276f8481aab..e7c805ce935e 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_interface.F90 @@ -127,7 +127,7 @@ subroutine dynamics_forcing_f90(dt,np1) bind(c) enddo end subroutine dynamics_forcing_f90 - subroutine tracers_forcing_f90(dt,np1,np1_qdp,hydrostatic,moist) bind(c) + subroutine tracers_forcing_f90(dt,np1,np1_qdp,hydrostatic,moist,adjustment) bind(c) use control_mod, only: use_moisture, theta_hydrostatic_mode use dimensions_mod, only: nelemd use prim_driver_base, only: applyCAMforcing_tracers @@ -136,7 +136,7 @@ subroutine tracers_forcing_f90(dt,np1,np1_qdp,hydrostatic,moist) bind(c) ! real (kind=c_double), intent(in) :: dt integer (kind=c_int), intent(in) :: np1,np1_qdp - logical (kind=c_bool), intent(in) :: hydrostatic, moist + logical (kind=c_bool), intent(in) :: hydrostatic, moist, adjustment ! ! Locals ! @@ -164,7 +164,7 @@ subroutine tracers_forcing_f90(dt,np1,np1_qdp,hydrostatic,moist) bind(c) elem(ie)%derived%FPHI = fphi(:,:,:,ie) ! Apply forcing - call applyCAMforcing_tracers(elem(ie),hvcoord,np1,np1_qdp,dt,.false.) + call applyCAMforcing_tracers(elem(ie),hvcoord,np1,np1_qdp,dt,LOGICAL(adjustment)) ! Copy outputs to C ptrs q(:,:,:,:,ie) = elem(ie)%state%q diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 8a686ac46d6c..1bf7d32fbb6f 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -39,7 +39,7 @@ void set_forcing_pointers_f90 (Real*& q_ptr, Real*& fq_ptr, Real*& qdp_ptr, Real*& dp_ptr, Real*& phinh_ptr, Real*& ps_ptr, Real*& fm_ptr, Real*& ft_ptr, Real*& fvtheta_ptr, Real*& fphi_ptr); -void tracers_forcing_f90 (const Real& dt, const int& np1, const int& np1_qdp, const bool& hydrostatic, const bool& moist); +void tracers_forcing_f90 (const Real& dt, const int& np1, const int& np1_qdp, const bool& hydrostatic, const bool& moist, const bool& adjustment); void dynamics_forcing_f90 (const Real& dt, const int& np1); void cleanup_f90(); } // extern "C" @@ -156,68 +156,111 @@ TEST_CASE("forcing", "forcing") { SECTION("tracers") { std::cout << "Testing tracers forcing.\n"; for (const bool hydrostatic : {true,false}) { - std::cout << " -> " << (hydrostatic ? "hydrostatic" : "non-hydrostatic") << "\n"; + std::cout << " -> hydrostatic mode: " << (hydrostatic ? "true" : "false") << "\n"; for (const MoistDry moisture : {MoistDry::DRY,MoistDry::MOIST}) { - std::cout << " -> " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; - - // Reset state, tracers, and forcing to the original random values - state.randomize(seed, 10*hv.ps0, hv.ps0, hv.hybrid_ai0); - tracers.randomize(seed); - forcing.randomize(seed); - - // Sync views - sync_to_host(tracers.Q,q_f90); - sync_to_host(tracers.fq,fq_f90); - sync_to_host(tracers.qdp,qdp_f90); - - sync_to_host(state.m_v,v_f90); - sync_to_host(state.m_w_i,w_f90); - sync_to_host(state.m_vtheta_dp,vtheta_f90); - sync_to_host(state.m_dp3d,dp_f90); - sync_to_host(state.m_phinh_i,phinh_f90); - // ps has same layout in cxx and f90 - Kokkos::deep_copy(ps_f90,state.m_ps_v); - - sync_to_host<3>(forcing.m_fm,fm_f90); - sync_to_host(forcing.m_ft,ft_f90); - sync_to_host(forcing.m_fvtheta,fvtheta_f90); - sync_to_host(forcing.m_fphi,fphi_f90); - - p.theta_hydrostatic_mode = hydrostatic; - - // Create the functor - FunctorsBuffersManager fbm; - ForcingFunctor ff; - - // Create and set buffers in the forcing functor - fbm.request_size(ff.requested_buffer_size()); - fbm.allocate(); - ff.init_buffers(fbm); - - // Run tracers forcing (cxx and f90) - ff.tracers_forcing(dt,np1,np1_qdp,false,moisture); - tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST); - - // Compare answers - Kokkos::deep_copy(h_dp,state.m_dp3d); - Kokkos::deep_copy(h_fphi,forcing.m_fphi); - Kokkos::deep_copy(h_fvtheta,forcing.m_fvtheta); - Kokkos::deep_copy(h_fq,tracers.fq); - Kokkos::deep_copy(h_qdp,tracers.qdp); - Kokkos::deep_copy(h_q,tracers.Q); - for (int ie=0; ie moisture: " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; + for (const bool adjustment : {true,false}) { + std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; + + // Reset state, tracers, and forcing to the original random values + state.randomize(seed, 10*hv.ps0, hv.ps0, hv.hybrid_ai0); + tracers.randomize(seed); + forcing.randomize(seed); + + // Sync views + sync_to_host(tracers.Q,q_f90); + sync_to_host(tracers.fq,fq_f90); + sync_to_host(tracers.qdp,qdp_f90); + + sync_to_host(state.m_v,v_f90); + sync_to_host(state.m_w_i,w_f90); + sync_to_host(state.m_vtheta_dp,vtheta_f90); + sync_to_host(state.m_dp3d,dp_f90); + sync_to_host(state.m_phinh_i,phinh_f90); + // ps has same layout in cxx and f90 + Kokkos::deep_copy(ps_f90,state.m_ps_v); + + sync_to_host<3>(forcing.m_fm,fm_f90); + sync_to_host(forcing.m_ft,ft_f90); + sync_to_host(forcing.m_fvtheta,fvtheta_f90); + sync_to_host(forcing.m_fphi,fphi_f90); + + p.theta_hydrostatic_mode = hydrostatic; + + // Create the functor + FunctorsBuffersManager fbm; + ForcingFunctor ff; + + // Create and set buffers in the forcing functor + fbm.request_size(ff.requested_buffer_size()); + fbm.allocate(); + ff.init_buffers(fbm); + + // Run tracers forcing (cxx and f90) + ff.tracers_forcing(dt,np1,np1_qdp,adjustment,moisture); + tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST,adjustment); + + // Compare answers + Kokkos::deep_copy(h_dp,state.m_dp3d); + Kokkos::deep_copy(h_fphi,forcing.m_fphi); + Kokkos::deep_copy(h_fvtheta,forcing.m_fvtheta); + Kokkos::deep_copy(h_fq,tracers.fq); + Kokkos::deep_copy(h_qdp,tracers.qdp); + Kokkos::deep_copy(h_q,tracers.Q); + for (int ie=0; ie Date: Mon, 13 Sep 2021 16:25:25 -0600 Subject: [PATCH 049/301] Homme: added lower/upper bound for randomization of tracers/forcing views --- components/homme/src/share/cxx/Tracers.cpp | 5 ++--- components/homme/src/share/cxx/Tracers.hpp | 2 +- components/homme/src/theta-l_kokkos/cxx/ElementsForcing.cpp | 5 +++-- components/homme/src/theta-l_kokkos/cxx/ElementsForcing.hpp | 2 +- components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/homme/src/share/cxx/Tracers.cpp b/components/homme/src/share/cxx/Tracers.cpp index 2314160e401d..e94769c890df 100644 --- a/components/homme/src/share/cxx/Tracers.cpp +++ b/components/homme/src/share/cxx/Tracers.cpp @@ -41,13 +41,12 @@ void Tracers::init(const int num_elems, const int num_tracers) m_inited = true; } -void Tracers::randomize(const int seed) { +void Tracers::randomize(const int seed, const Real min, const Real max) { // Check tracers were inited assert (m_inited); - constexpr Real min_value = 0.015625; std::mt19937_64 engine(seed); - std::uniform_real_distribution random_dist(min_value, 1.0 / min_value); + std::uniform_real_distribution random_dist(min, max); genRandArray(qdp, engine, random_dist); genRandArray(qtens_biharmonic, engine, random_dist); diff --git a/components/homme/src/share/cxx/Tracers.hpp b/components/homme/src/share/cxx/Tracers.hpp index 6f8e70e533bd..ec16fe2dbb71 100644 --- a/components/homme/src/share/cxx/Tracers.hpp +++ b/components/homme/src/share/cxx/Tracers.hpp @@ -17,7 +17,7 @@ struct Tracers { void init (const int num_elems, const int num_tracers); - void randomize(const int seed); + void randomize (const int seed, const Real min = -1.0, const Real max = 1.0); void pull_qdp(CF90Ptr &state_qdp); void push_qdp(F90Ptr &state_qdp) const; diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.cpp index 2520f5c3303d..7fc720ceb6ad 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.cpp @@ -17,12 +17,13 @@ void ElementsForcing::init (const int num_elems) { m_fphi = ExecViewManaged("F_Phi", m_num_elems); } -void ElementsForcing::randomize (const int seed) { +void ElementsForcing::randomize (const int seed, const Real min_f, const Real max_f) { // Check forcing was inited assert (m_num_elems>0); + assert (max_f > min_f); std::mt19937_64 engine(seed); - std::uniform_real_distribution random_dist(-1.0, 1.0); + std::uniform_real_distribution random_dist(min_f, max_f); genRandArray(m_fm, engine, random_dist); genRandArray(m_fvtheta, engine, random_dist); diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.hpp b/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.hpp index d56fd0f1de31..ece6b1c65a2c 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsForcing.hpp @@ -17,7 +17,7 @@ class ElementsForcing { ElementsForcing() = default; void init (const int num_elems); - void randomize (const int seed); + void randomize (const int seed, const Real min_f = -1.0, const Real max_f = 1.0); int num_elems () const { return m_num_elems; } diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 1bf7d32fbb6f..323f632bf1d6 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -164,7 +164,7 @@ TEST_CASE("forcing", "forcing") { // Reset state, tracers, and forcing to the original random values state.randomize(seed, 10*hv.ps0, hv.ps0, hv.hybrid_ai0); - tracers.randomize(seed); + tracers.randomize(seed,-1e-3,1e-3); forcing.randomize(seed); // Sync views From 3ab0794ca17c3bd600c05292bf39c8a3a464196a Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 15 Sep 2021 16:30:08 -0600 Subject: [PATCH 050/301] Hommexx: fixed a couple of compiler warning in release builds --- components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp | 3 +-- components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp index 9b8297ac95b8..1f14073d556b 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/CaarFunctorImpl.hpp @@ -288,8 +288,7 @@ struct CaarFunctorImpl { m_buffers.grad_phinh_i = decltype(m_buffers.grad_phinh_i)(mem,nslots); mem += m_buffers.grad_phinh_i.size(); - int used_mem = (reinterpret_cast(mem) - fbm.get_memory()); - assert (used_mem==requested_buffer_size()); + assert ((reinterpret_cast(mem) - fbm.get_memory())==requested_buffer_size()); } void init_boundary_exchanges (const std::shared_ptr& bm_exchange) { diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp index b43d4e051d79..e41646312161 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp @@ -38,8 +38,7 @@ void RefStates::compute(const bool hydrostatic, ElementOps elem_ops; elem_ops.init(hvcoord); - const int num_elems = dp_ref.extent_int(0); - assert(num_elems==m_num_elems); + assert(dp_ref.extent_int(0)==m_num_elems); // Local copies, to avoid cuda issues with *this auto l_dp_ref = dp_ref; From fb7f5e807944bcf2d9d2b4550a122c951e92672a Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 16 Sep 2021 18:41:12 -0600 Subject: [PATCH 051/301] Add damage to advection scheme This commit adds damage to the advection scheme. It only occurs if config_damage_advection is .true., and it is .false. by default. We may want to change the namelist controls in the future, perhaps having damage always advected when the damage calving scheme is on. But this makes it simpler for testing. The advection scheme is 3d but damage is a 2d field. When damage is added to the tracer array, the 2d value is assigned at all vertical levels. After advection, the tracer array result is converted back to a 2d field by a layerThickness-weighted average. Any accumulation at the ice surface or bottom is accumulated with a damage value of 0. --- .../mpas-albany-landice/src/Registry.xml | 6 ++- .../src/mode_forward/mpas_li_advection.F | 52 +++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index 0bc79d5609b8..1ebf9bc7ebe6 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -42,7 +42,7 @@ - + Date: Thu, 16 Sep 2021 20:40:22 -0700 Subject: [PATCH 052/301] Add halo update for damage advection Damage advection requires halo update. May want to put this in an if-statement and only use if config_damage_advection = .true. --- .../src/mode_forward/mpas_li_time_integration_fe.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe.F b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe.F index e36e7ee13e0b..1fd380483935 100644 --- a/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe.F +++ b/components/mpas-albany-landice/src/mode_forward/mpas_li_time_integration_fe.F @@ -784,6 +784,7 @@ subroutine advection_solver(domain, err) call mpas_dmpar_field_halo_exch(domain, 'temperature') call mpas_dmpar_field_halo_exch(domain, 'waterFrac') call mpas_dmpar_field_halo_exch(domain, 'enthalpy') + call mpas_dmpar_field_halo_exch(domain, 'damage') call mpas_timer_stop("halo updates") From 7ffcbf5a691297adfee82e7a54eb37e24ad2da6f Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Fri, 17 Sep 2021 08:54:55 -0700 Subject: [PATCH 053/301] Add config_restore_thickness_after_advection This namelist option resets thickness to the previous timestep's values after advection has taken place. This is useful for spinning up tracer fields like damage while holding ice thickness constant. --- components/mpas-albany-landice/src/Registry.xml | 6 +++++- .../src/mode_forward/mpas_li_time_integration_fe.F | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/src/Registry.xml b/components/mpas-albany-landice/src/Registry.xml index 0bc79d5609b8..a6bbe01a8e76 100644 --- a/components/mpas-albany-landice/src/Registry.xml +++ b/components/mpas-albany-landice/src/Registry.xml @@ -115,7 +115,11 @@ + /> + -lnd/clm2/paramdata/fates_params_default_12pft_api16.c210527.nc +lnd/clm2/paramdata/fates_params_api.16.1.0_12pft_c210630.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc diff --git a/components/elm/src/main/elm_initializeMod.F90 b/components/elm/src/main/elm_initializeMod.F90 index cc30e31fe459..9dd03b5d0256 100644 --- a/components/elm/src/main/elm_initializeMod.F90 +++ b/components/elm/src/main/elm_initializeMod.F90 @@ -661,6 +661,10 @@ subroutine initialize2( ) call SatellitePhenologyInit(bounds_proc) end if + if (use_fates_sp) then + call SatellitePhenologyInit(bounds_proc) + end if + if (use_cn .and. n_drydep > 0 .and. drydep_method == DD_XLND) then ! Must do this also when drydeposition is used so that estimates of monthly ! differences in LAI can be computed From a273f68b869ca047b425d3a603f550219bb22294 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 20 Oct 2021 18:34:52 -0500 Subject: [PATCH 081/301] add forcing call and debugging --- .../homme/src/share/cxx/prim_driver.cpp | 7 ++--- .../homme/src/share/prim_driver_base.F90 | 4 +-- .../homme/src/test_src/held_suarez_mod.F90 | 6 ++++- .../cxx/cxx_f90_interface_theta.cpp | 8 +++--- .../src/theta-l_kokkos/prim_driver_mod.F90 | 27 +++++++++++++++++-- 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index 951367f1225e..ead9c0355aec 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -104,9 +104,10 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np if ( ! independent_time_steps) { tl.update_tracers_levels(params.dt_tracer_factor); -#ifndef CAM - apply_test_forcing (); -#endif +/// remove +//#ifndef CAM +// apply_test_forcing (); +//#endif // Apply forcing. // In standalone mode, params.ftype == ForcingAlg::FORCING_DEBUG diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index a05530aee54c..929f42aa44a1 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1100,7 +1100,7 @@ subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, single_column, tl, hvco ! compute HOMME test case forcing ! by calling it here, it mimics eam forcings computations in standalone ! homme. - call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) +! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) #endif call applyCAMforcing_remap(elem,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) @@ -1331,7 +1331,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute #ifndef CAM ! Compute test forcing over tracer time step. - call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_q,nets,nete,tl) +! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_q,nets,nete,tl) #endif #ifdef CAM diff --git a/components/homme/src/test_src/held_suarez_mod.F90 b/components/homme/src/test_src/held_suarez_mod.F90 index 49dd53e0cec4..4234e27c8d8f 100644 --- a/components/homme/src/test_src/held_suarez_mod.F90 +++ b/components/homme/src/test_src/held_suarez_mod.F90 @@ -112,7 +112,11 @@ subroutine hs_forcing(elemin,hvcoord,nm1,nm1_Q,dt) enddo endif - + elemin%derived%FM(:,:,1:3,:) = 0.0 +elemin%derived%FQ(:,:,:,:) =0.0 +elemin%derived%FT(:,:,:) = 0.0 + + end subroutine hs_forcing function hs_v_forcing(hvcoord,ps,v,npts,nlevels) result(hs_v_frc) diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 16036f157461..440dec48f061 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -147,11 +147,11 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, // TODO Parse a fortran string and set this properly. For now, our code does // not depend on this except to throw an error in apply_test_forcing. std::string test_name(*test_case); - if (test_name=="jw_baroclinic") { + //if (test_name=="jw_baroclinic") { params.test_case = TestCase::JW_BAROCLINIC; - } else { - Errors::runtime_abort("Error! Unknown test case '" + test_name + "'.\n"); - } + //} else { + // Errors::runtime_abort("Error! Unknown test case '" + test_name + "'.\n"); + //} // Now this structure can be used safely params.params_set = true; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index f793f4a7ba87..63b23e438609 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -343,7 +343,8 @@ end subroutine prim_init_kokkos_functors subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hvcoord,nsubstep) use iso_c_binding, only : c_int, c_ptr, c_loc - use control_mod, only : qsplit, rsplit, statefreq, disable_diagnostics + use control_mod, only : qsplit, rsplit, statefreq, disable_diagnostics, & + dt_remap_factor, dt_tracer_factor use dimensions_mod, only : nelemd use element_state, only : elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, & @@ -353,7 +354,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv use hybrid_mod, only : hybrid_t use hybvcoord_mod, only : hvcoord_t use kinds, only : real_kind - use time_mod, only : timelevel_t, nextOutputStep, nsplit + use time_mod, only : timelevel_t, nextOutputStep, nsplit, TimeLevel_Qdp use control_mod, only : statefreq use parallel_mod, only : abortmp use perf_mod, only : t_startf, t_stopf @@ -361,6 +362,9 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv use theta_f2c_mod, only : prim_run_subcycle_c, cxx_push_results_to_f90 #ifndef SCREAM use theta_f2c_mod, only : push_forcing_to_c +#endif +#if !defined(CAM) && !defined(SCREAM) + use test_mod, only : compute_test_forcing #endif ! ! Inputs @@ -383,6 +387,9 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv type (c_ptr) :: elem_state_dp3d_ptr, elem_state_Qdp_ptr, elem_state_Q_ptr, elem_state_ps_v_ptr type (c_ptr) :: elem_derived_omega_p_ptr + integer :: n0_qdp, np1_qdp + real(kind=real_kind) :: dt_remap + if (nets/=1 .or. nete/=nelemd) then call abortmp ('We don''t allow to call C routines from a horizontally threaded region') endif @@ -399,6 +406,22 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv compute_diagnostics = .false. endif + !dt_q = dt*dt_tracer_factor + if (dt_remap_factor == 0) then + dt_remap = dt + else + dt_remap = dt*dt_remap_factor + end if + + call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) + +!call from driver mod +! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) + +#if !defined(CAM) && !defined(SCREAM) +! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) +#endif + #ifndef SCREAM ! Scream already computes all forcing using the same pointers ! stored in Hommexx, so the forcing is already up to date From cb25ba216bdda7bcb68a8c11e7dd32d5223bd5bd Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 20 Oct 2021 21:45:39 -0500 Subject: [PATCH 082/301] this version (also, with remap=1 instead of 3 in HS nl) bfb --- components/homme/src/share/prim_driver_base.F90 | 4 ++-- components/homme/src/test_src/held_suarez_mod.F90 | 4 ++-- components/homme/src/theta-l_kokkos/prim_driver_mod.F90 | 2 +- components/homme/test/reg_test/namelists/thetaHS.nl | 4 ++-- .../test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake | 4 ++-- .../homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 929f42aa44a1..a05530aee54c 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1100,7 +1100,7 @@ subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, single_column, tl, hvco ! compute HOMME test case forcing ! by calling it here, it mimics eam forcings computations in standalone ! homme. -! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) #endif call applyCAMforcing_remap(elem,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete) @@ -1331,7 +1331,7 @@ subroutine prim_step_flexible(hybrid, elem, nets, nete, dt, tl, hvcoord, compute #ifndef CAM ! Compute test forcing over tracer time step. -! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_q,nets,nete,tl) + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_q,nets,nete,tl) #endif #ifdef CAM diff --git a/components/homme/src/test_src/held_suarez_mod.F90 b/components/homme/src/test_src/held_suarez_mod.F90 index 4234e27c8d8f..99c4508fbada 100644 --- a/components/homme/src/test_src/held_suarez_mod.F90 +++ b/components/homme/src/test_src/held_suarez_mod.F90 @@ -112,9 +112,9 @@ subroutine hs_forcing(elemin,hvcoord,nm1,nm1_Q,dt) enddo endif - elemin%derived%FM(:,:,1:3,:) = 0.0 +! elemin%derived%FM(:,:,1:3,:) = 0.0 elemin%derived%FQ(:,:,:,:) =0.0 -elemin%derived%FT(:,:,:) = 0.0 +!elemin%derived%FT(:,:,:) = 0.0 end subroutine hs_forcing diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 63b23e438609..dc26cb120e9c 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -419,7 +419,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv ! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) #if !defined(CAM) && !defined(SCREAM) -! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) #endif #ifndef SCREAM diff --git a/components/homme/test/reg_test/namelists/thetaHS.nl b/components/homme/test/reg_test/namelists/thetaHS.nl index 81dae86eb104..b3780f72b243 100644 --- a/components/homme/test/reg_test/namelists/thetaHS.nl +++ b/components/homme/test/reg_test/namelists/thetaHS.nl @@ -15,7 +15,7 @@ restartfile = "./R0001" runtype = 0 mesh_file = '/dev/null' tstep = 100 -rsplit = 3 +rsplit = 1 !3 qsplit = 1 integration = "explicit" smooth = 0 @@ -24,7 +24,7 @@ nu_div = 3.4e-8 ! test this too? nu_p = -1 nu_q = -1 nu_s = -1 -nu_top = 2.5e5 +nu_top = 0 !2.5e5 se_ftype = 0 limiter_option = 9 vert_remap_q_alg = 1 diff --git a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake index d316f5d9fd0d..2c9845069e8f 100644 --- a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake @@ -14,8 +14,8 @@ SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - held_suarez1.nc - held_suarez2.nc) + held_suarez01.nc + held_suarez02.nc) #DO NOT MOD SET (HOMME_TEST_TIME_STEP 600) diff --git a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake index 84560a6d27c8..9ed79f574002 100644 --- a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake +++ b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake @@ -14,8 +14,8 @@ SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - held_suarez1.nc - held_suarez2.nc) + held_suarez01.nc + held_suarez02.nc) #DO NOT MOD SET (HOMME_TEST_TIME_STEP 600) From ec0af098c9313095e92fbb99ed92e14aa8ef390c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 20 Oct 2021 22:23:16 -0500 Subject: [PATCH 083/301] a bfb version --- components/homme/src/test_src/held_suarez_mod.F90 | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/components/homme/src/test_src/held_suarez_mod.F90 b/components/homme/src/test_src/held_suarez_mod.F90 index 99c4508fbada..34a969257f9b 100644 --- a/components/homme/src/test_src/held_suarez_mod.F90 +++ b/components/homme/src/test_src/held_suarez_mod.F90 @@ -112,9 +112,18 @@ subroutine hs_forcing(elemin,hvcoord,nm1,nm1_Q,dt) enddo endif -! elemin%derived%FM(:,:,1:3,:) = 0.0 -elemin%derived%FQ(:,:,:,:) =0.0 -!elemin%derived%FT(:,:,:) = 0.0 +#if 0 +!bfb version +elemin%derived%FM(:,:,1:3,:) = 0.0 +elemin%derived%FQ(:,:,:,:) = 0.01 +elemin%derived%FT(:,:,:) = 0.0 +#endif + +elemin%derived%FM(:,:,1:3,:) = 0.0 +!elemin%derived%FQ(:,:,:,:) = 0.0 +elemin%derived%FT(:,:,:) = 0.0 + + end subroutine hs_forcing From ace3b16b5e0db1dbf77eea69e6be1e1eca733676 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 22 Oct 2021 15:04:34 -0600 Subject: [PATCH 084/301] Be consistent with case of LINUX --- .../cmake_macros/{intel_Linux.cmake => intel_LINUX.cmake} | 0 cime_config/machines/config_compilers.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename cime_config/machines/cmake_macros/{intel_Linux.cmake => intel_LINUX.cmake} (100%) diff --git a/cime_config/machines/cmake_macros/intel_Linux.cmake b/cime_config/machines/cmake_macros/intel_LINUX.cmake similarity index 100% rename from cime_config/machines/cmake_macros/intel_Linux.cmake rename to cime_config/machines/cmake_macros/intel_LINUX.cmake diff --git a/cime_config/machines/config_compilers.xml b/cime_config/machines/config_compilers.xml index 5e6590dc240b..37fcc6ce3778 100644 --- a/cime_config/machines/config_compilers.xml +++ b/cime_config/machines/config_compilers.xml @@ -588,7 +588,7 @@ flags should be captured within MPAS CMake files. - + -mcmodel medium -shared-intel From a7343dd6196ba0542c78250fb12aeda48002a768 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 22 Oct 2021 15:05:29 -0600 Subject: [PATCH 085/301] Completed set/append mix and unused warning checks --- cime_config/machines/scripts/evaluator | 118 ++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 11 deletions(-) diff --git a/cime_config/machines/scripts/evaluator b/cime_config/machines/scripts/evaluator index 6e444e773a67..43f8f27b6631 100755 --- a/cime_config/machines/scripts/evaluator +++ b/cime_config/machines/scripts/evaluator @@ -13,6 +13,13 @@ import xml.etree.ElementTree as ET UNIVERSALS = ["universal"] +# Regexes for parsing our cmake macros +SET_RE = re.compile(r'^\s*set[(](\w+)\s+"([^"]+)"[)]\s*$') +APPEND_RE = re.compile(r'^\s*string[(]APPEND\s+(\w+)\s+"([^"]+)"[)]\s*$') +IF_RE = re.compile(r'^\s*if\s*[(]([^)]+)[)]\s*$') +ENDIF_RE = re.compile(r'^\s*endif') +EXE_RE = re.compile(r'^\s*execute_process') + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -46,6 +53,9 @@ def find_machs_comps_oss(config_machines): ############################################################################### """ Return a dict mapping MACH -> (OS, COMPILERS), compilers, OSs + + Created by parsing the config_machines.xml file. The map describes the relationship + between machines and OSes and compilers """ with open(config_machines, "r") as fd: tree = ET.parse(fd) @@ -56,7 +66,10 @@ def find_machs_comps_oss(config_machines): oss = set() for child in root: - assert child.tag == "machine", f"Unexpected child.tag {child.tag}" + assert child.tag in ["machine", "default_run_suffix"], f"Unexpected child.tag {child.tag}" + if child.tag == "default_run_suffix": + continue + assert "MACH" in child.attrib, f"Missing MACH attribute for child.tag {child.tag}" mach = child.attrib["MACH"].strip() @@ -82,30 +95,113 @@ def find_machs_comps_oss(config_machines): return machs, list(all_comps), list(oss) ############################################################################### -def find_suspicious_set(all_files, machs): +def parse_macro(macro_path): +############################################################################### + """ + Parse one of our cmake macros, return (settings, appends), both lists + of (varname, value) tuples. + """ + settings = [] + appends = [] + with macro_path.open("r") as fd: + for line in fd.readlines(): + line = line.strip() + if line != "": + set_m = SET_RE.match(line) + append_m = APPEND_RE.match(line) + if_m = IF_RE.match(line) + endif_m = ENDIF_RE.match(line) + exe_m = EXE_RE.match(line) + + if set_m: + var, val = set_m.groups() + settings.append((var, val)) + elif append_m: + var, val = append_m.groups() + appends.append((var, val)) + elif if_m or endif_m or exe_m: + pass # for now + else: + assert False, f"In {macro_path}, Could not parse line: {line}" + + return settings, appends + ############################################################################### +def get_all_macros(macro_path_pl, universal, the_os, mach, comp): +############################################################################### + """ + Get all existing macro files for this configuration. Returns them in load order + as full pathlib.Paths + """ + universal_macro = f"{universal}.cmake" + compiler_macro = f"{comp}.cmake" + os_macro = f"{the_os}.cmake" + machine_macro = f"{mach}.cmake" + comp_os_macro = f"{comp}_{the_os}.cmake" + comp_mach_macro = f"{comp}_{mach}.cmake" + + results = [] + for macro_name in [universal_macro, compiler_macro, os_macro, machine_macro, comp_os_macro, comp_mach_macro]: + curr_macro_path_pl = macro_path_pl / macro_name + if curr_macro_path_pl.exists(): + results.append(curr_macro_path_pl) + + return results + +############################################################################### +def find_suspicious_set(macro_path_pl, machs): +############################################################################### + warnings = [] + for universal in UNIVERSALS: + for mach, data in machs.items(): + the_os = data[0] + for comp in data[1]: + macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) + sets = set() + appends = set() + for macro_pl in macro_pls: + curr_settings, curr_appends = parse_macro(macro_pl) + sets.update ([var for var, val in curr_settings]) + appends.update([var for var, val in curr_appends]) + + mixes = sets & appends + if mixes: + warnings.append(f"Macro {macro_pl} began to mix sets and appends for vars: {', '.join(mixes)}") + print(warnings[-1]) + + return warnings + +############################################################################### +def find_unused_macros(macro_path_pl, machs): +############################################################################### + warnings = [] + unused_macro_files = set([item for item in macro_path_pl.glob("*.cmake") if item.name != "Macros.cmake"]) for universal in UNIVERSALS: for mach, data in machs.items(): the_os = data[0] for comp in data[1]: - universal_macro = f"{universal}.cmake" - compiler_macro = f"{comp}.cmake" - os_macro = f"{the_os}.cmake" - machine_macro = f"{mach}.cmake" - comp_os_macro = f"{comp}_{the_os}.cmake" - comp_mach_macro = f"{comp}_{mach}.cmake" + macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) + unused_macro_files = unused_macro_files - set(macro_pls) + + for unused_macro_file in unused_macro_files: + warnings.append(f"Macro {unused_macro_file} is unused") + print(warnings[-1]) - x + return warnings ############################################################################### -def evaluate(macro_path, machine, compiler): +def evaluate(macro_path, machine=None, compiler=None): ############################################################################### macro_path_pl = pathlib.Path(macro_path) - config_machines_pl = macro_path / ".." / "config_machines.xml" + config_machines_pl = macro_path_pl / ".." / "config_machines.xml" machs, all_comps, oss = find_machs_comps_oss(config_machines_pl) + warnings = [] + warnings.extend(find_suspicious_set(macro_path_pl, machs)) + warnings.extend(find_unused_macros(macro_path_pl, machs)) + return warnings == [] ############################################################################### def _main_func(description): From bad61dd82e31557186a4af8d9d2376ec35231cd8 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 25 Oct 2021 17:48:43 -0600 Subject: [PATCH 086/301] Homme(xx) SL: Remove a number of -Wall warnings. --- components/homme/src/share/compose/compose_cedr.cpp | 3 +-- .../src/share/compose/compose_slmm_departure_point.cpp | 1 - .../src/share/compose/compose_slmm_departure_point.hpp | 1 - .../homme/src/share/compose/compose_slmm_islmpi.cpp | 2 +- .../src/share/compose/compose_slmm_islmpi_adp.cpp | 1 - components/homme/src/share/compose/compose_test.hpp | 1 + .../homme/src/share/cxx/ComposeTransportImpl.hpp | 4 ++-- .../src/share/cxx/ComposeTransportImplGeneral.cpp | 10 ++++------ .../src/share/cxx/ComposeTransportImplTrajectory.cpp | 2 +- .../homme/test_execs/thetal_kokkos_ut/compose_ut.cpp | 10 +++++----- 10 files changed, 15 insertions(+), 20 deletions(-) diff --git a/components/homme/src/share/compose/compose_cedr.cpp b/components/homme/src/share/compose/compose_cedr.cpp index febd9ad38edb..1e72c7484c74 100644 --- a/components/homme/src/share/compose/compose_cedr.cpp +++ b/components/homme/src/share/compose/compose_cedr.cpp @@ -431,8 +431,7 @@ CDR::CDR (Int cdr_alg_, Int ngblcell_, Int nlclcell_, Int nlev_, Int qsize_, cdr_over_super_levels(threed && Alg::is_caas(alg)), caas_in_suplev(alg == Alg::qlt_super_level_local_caas && nsublev > 1), hard_zero(hard_zero_), - p(p_), inited_tracers_(false), - run(cdr_alg_ != 42) + p(p_), run(cdr_alg_ != 42), inited_tracers_(false) { const Int n_id_in_suplev = caas_in_suplev ? 1 : nsublev; if (Alg::is_qlt(alg)) { diff --git a/components/homme/src/share/compose/compose_slmm_departure_point.cpp b/components/homme/src/share/compose/compose_slmm_departure_point.cpp index 60d21e9f92b4..b2ab270b4917 100644 --- a/components/homme/src/share/compose/compose_slmm_departure_point.cpp +++ b/components/homme/src/share/compose/compose_slmm_departure_point.cpp @@ -26,7 +26,6 @@ void fill_normals (LocalMesh& m) { // continuous rather than periodic coordinate values. void make_continuous (const Plane& p, LocalMesh& m) { slmm_assert(m.tgt_elem >= 0 && m.tgt_elem <= nslices(m.e)); - const auto tcell = slice(m.e, m.tgt_elem); // Geometric center of cell. const auto getctr = [&] (const Int ie, Real ctr[3]) { const auto cell = slice(m.e, ie); diff --git a/components/homme/src/share/compose/compose_slmm_departure_point.hpp b/components/homme/src/share/compose/compose_slmm_departure_point.hpp index 8d67d8bffa26..de912c542160 100644 --- a/components/homme/src/share/compose/compose_slmm_departure_point.hpp +++ b/components/homme/src/share/compose/compose_slmm_departure_point.hpp @@ -261,7 +261,6 @@ void fill_perim (LocalMesh& m) { std::vector external_edges; find_external_edges(m, external_edges); const Int nee = external_edges.size(); - const bool already = m.perimp.size() > 0; m.perimp = typename LocalMesh::Ints("perimp", nee); m.perimnml = typename LocalMesh::Ints("perimnml", nee); for (Int k = 0; k < nee; ++k) { diff --git a/components/homme/src/share/compose/compose_slmm_islmpi.cpp b/components/homme/src/share/compose/compose_slmm_islmpi.cpp index b7af839d3943..7313d1c851ae 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi.cpp @@ -566,7 +566,7 @@ void comm_lid_on_rank (IslMpi& cm, const Rank2Gids& rank2rmtgids, mpi::waitany(count, reqs.data(), &idx); const Int* rmtlids = recv.data() + recvptr[idx]; const auto& gids = rank2rmtgids.at(recvrank[idx]); - slmm_assert(recvptr[idx+1] - recvptr[idx] == gids.size()); + slmm_assert(recvptr[idx+1] - recvptr[idx] == static_cast(gids.size())); Int i = 0; for (auto gid: gids) gid2rmt_owning_lid[gid] = rmtlids[i++]; diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_adp.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_adp.cpp index 56520eda99f2..d3f4e76c1af2 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_adp.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_adp.cpp @@ -84,7 +84,6 @@ void analyze_dep_points (IslMpi& cm, const Int& nets, const Int& nete, const auto& plane = cm.advecter->get_plane(); #ifdef COMPOSE_PORT const auto myrank = cm.p->rank(); - const Int nrmtrank = static_cast(cm.ranks.size()) - 1; const Int np2 = cm.np2, nlev = cm.nlev; const auto& own_dep_mask = cm.own_dep_mask; cm.bla.zero(); diff --git a/components/homme/src/share/compose/compose_test.hpp b/components/homme/src/share/compose/compose_test.hpp index 3e2e290e2a77..9a3e7d4b350b 100644 --- a/components/homme/src/share/compose/compose_test.hpp +++ b/components/homme/src/share/compose/compose_test.hpp @@ -135,6 +135,7 @@ class InitialCondition { case CosineBells: return inputs[2]; case SlottedCylinders: return inputs[3]; case CorrelatedCosineBells: return inputs[4]; + case nshapes: break; } throw std::runtime_error("Should not be here."); } diff --git a/components/homme/src/share/cxx/ComposeTransportImpl.hpp b/components/homme/src/share/cxx/ComposeTransportImpl.hpp index c8fe92c5088d..9d1b296b2d94 100644 --- a/components/homme/src/share/cxx/ComposeTransportImpl.hpp +++ b/components/homme/src/share/cxx/ComposeTransportImpl.hpp @@ -64,8 +64,8 @@ struct ComposeTransportImpl { Data () : nelemd(-1), qsize(-1), limiter_option(9), cdr_check(0), hv_q(0), - hv_subcycle_q(0), independent_time_steps(false), nu_q(0), hv_scaling(0), - dp_tol(-1), geometry_type(0) + hv_subcycle_q(0), geometry_type(0), nu_q(0), hv_scaling(0), dp_tol(-1), + independent_time_steps(false) {} }; diff --git a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp index 671c66449643..d1b2b921fd9f 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp @@ -20,17 +20,15 @@ static int calc_nslot (const int nelemd) { } ComposeTransportImpl::ComposeTransportImpl () - : m_tp_ne(1,1,1), m_tu_ne(m_tp_ne), // throwaway settings - m_tp_ne_qsize(1,1,1), m_tu_ne_qsize(m_tp_ne_qsize), // throwaway settings - m_tp_ne_hv_q(1,1,1), m_tu_ne_hv_q(m_tp_ne_hv_q) // throwaway settings + : m_tp_ne(1,1,1), m_tp_ne_qsize(1,1,1), m_tp_ne_hv_q(1,1,1), // throwaway settings + m_tu_ne(m_tp_ne), m_tu_ne_qsize(m_tp_ne_qsize), m_tu_ne_hv_q(m_tp_ne_hv_q) { setup(); } ComposeTransportImpl::ComposeTransportImpl (const int num_elems) - : m_tp_ne(1,1,1), m_tu_ne(m_tp_ne), // throwaway settings - m_tp_ne_qsize(1,1,1), m_tu_ne_qsize(m_tp_ne_qsize), // throwaway settings - m_tp_ne_hv_q(1,1,1), m_tu_ne_hv_q(m_tp_ne_hv_q) // throwaway settings + : m_tp_ne(1,1,1), m_tp_ne_qsize(1,1,1), m_tp_ne_hv_q(1,1,1), // throwaway settings + m_tu_ne(m_tp_ne), m_tu_ne_qsize(m_tp_ne_qsize), m_tu_ne_hv_q(m_tp_ne_hv_q) {} void ComposeTransportImpl::setup () { diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index 2822d3a6ef43..45d193c4a677 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -446,7 +446,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto dep_pts = Homme::subview(m_dep_pts, ie); const auto f = [&] (const int i, const int j, const int k) { // dp = p1 - dt v/scale_factor - Scalar dp[3], r = 0; + Scalar dp[3]; for (int d = 0; d < 3; ++d) { const auto vel_cart = (vec_sphere2cart(0,d,i,j)*vstar(0,i,j,k) + vec_sphere2cart(1,d,i,j)*vstar(1,i,j,k)); diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 0ff378e3d5c4..5428f016aa51 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -392,14 +392,14 @@ TEST_CASE ("compose_transport_testing") { ct.test_2d(bfb, nmax, eval_c); if (s.get_comm().root()) { const Real f = bfb ? 0 : 1; - const auto n = s.nlev*s.qsize; + const int n = s.nlev*s.qsize; // When not a BFB build, still expect l2 error to be the same to a few digits. - for (size_t i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e-3)); + for (int i = 0; i < n; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*1e-3)); // Mass conservation error should be within a factor of 10 of each other. - for (size_t i = n; i < n + s.qsize; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*10)); + for (int i = n; i < n + s.qsize; ++i) REQUIRE(almost_equal(eval_f[i], eval_c[i], f*10)); // And mass conservation itself should be small. - for (size_t i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_f[i]) <= 20*tol); - for (size_t i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_c[i]) <= 20*tol); + for (int i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_f[i]) <= 20*tol); + for (int i = n; i < n + s.qsize; ++i) REQUIRE(std::abs(eval_c[i]) <= 20*tol); //todo add an l2 ceiling for some select tracers as a function of ne } } From 24f8f5c7437215811b80f4bd40e2dfb895d395d8 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 25 Oct 2021 19:29:23 -0600 Subject: [PATCH 087/301] Homme(xx)/SL: Fix more -Wall -pedantic warnings. --- components/homme/src/share/compose/compose_cedr.cpp | 2 +- components/homme/src/share/compose_mod.F90 | 13 +++++++------ components/homme/src/share/cxx/GllFvRemapImpl.cpp | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/homme/src/share/compose/compose_cedr.cpp b/components/homme/src/share/compose/compose_cedr.cpp index 1e72c7484c74..69a940d0ec8b 100644 --- a/components/homme/src/share/compose/compose_cedr.cpp +++ b/components/homme/src/share/compose/compose_cedr.cpp @@ -358,7 +358,7 @@ struct ReproSumReducer : send = typename RealList::HostMirror("send", nlocal*count); recv = typename RealList::HostMirror("recv", count); } - cedr_assert(send.size() == nlocal*count); + cedr_assert(static_cast(send.size()) == nlocal*count); ko::deep_copy(send, ConstRealList(sendbuf, nlocal*count)); sendptr = send.data(); rcvptr = recv.data(); diff --git a/components/homme/src/share/compose_mod.F90 b/components/homme/src/share/compose_mod.F90 index 7f3d4171b8e4..afa0b661070f 100644 --- a/components/homme/src/share/compose_mod.F90 +++ b/components/homme/src/share/compose_mod.F90 @@ -106,11 +106,12 @@ subroutine cedr_sl_set_dp3d(ie, dp3d, tl_np1) bind(c) end subroutine cedr_sl_set_dp3d subroutine cedr_sl_set_dp(ie, dp) bind(c) + use iso_c_binding , only : c_int, c_double use kinds , only : real_kind use dimensions_mod, only : nlev, np use element_state, only : timelevels - integer, value, intent(in) :: ie - real(kind=real_kind), intent(in) :: dp(np,np,nlev) + integer(kind=c_int), value, intent(in) :: ie + real(kind=c_double), intent(in) :: dp(np,np,nlev) end subroutine cedr_sl_set_dp subroutine cedr_sl_set_Q(ie, Q) bind(c) @@ -128,25 +129,25 @@ end subroutine cedr_sl_set_pointers_end subroutine cedr_sl_run_global(minq, maxq, nets, nete) bind(c) use iso_c_binding, only: c_int, c_double use dimensions_mod, only : nlev, np, qsize + integer(kind=c_int), value, intent(in) :: nets, nete real(kind=c_double), intent(in) :: minq(np,np,nlev,qsize,nets:nete) real(kind=c_double), intent(in) :: maxq(np,np,nlev,qsize,nets:nete) - integer(kind=c_int), value, intent(in) :: nets, nete end subroutine cedr_sl_run_global subroutine cedr_sl_run_local(minq, maxq, nets, nete, use_ir, limiter_option) bind(c) use iso_c_binding, only: c_int, c_double use dimensions_mod, only : nlev, np, qsize + integer(kind=c_int), value, intent(in) :: nets, nete, use_ir, limiter_option real(kind=c_double), intent(in) :: minq(np,np,nlev,qsize,nets:nete) real(kind=c_double), intent(in) :: maxq(np,np,nlev,qsize,nets:nete) - integer(kind=c_int), value, intent(in) :: nets, nete, use_ir, limiter_option end subroutine cedr_sl_run_local subroutine cedr_sl_check(minq, maxq, nets, nete) bind(c) use iso_c_binding, only: c_int, c_double use dimensions_mod, only : nlev, np, qsize + integer(kind=c_int), value, intent(in) :: nets, nete real(kind=c_double), intent(in) :: minq(np,np,nlev,qsize,nets:nete) real(kind=c_double), intent(in) :: maxq(np,np,nlev,qsize,nets:nete) - integer(kind=c_int), value, intent(in) :: nets, nete end subroutine cedr_sl_check subroutine slmm_init_local_mesh(ie, neigh_corners, num_neighbors, pinside, & @@ -424,9 +425,9 @@ subroutine compose_repro_sum(send, recv, nlocal, nfld, comm) bind(c) use repro_sum_mod, only: repro_sum #endif + integer(kind=c_int), value, intent(in) :: nlocal, nfld, comm real(kind=c_double), intent(in) :: send(nlocal,nfld) real(kind=c_double), intent(out) :: recv(nfld) - integer(kind=c_int), value, intent(in) :: nlocal, nfld, comm call repro_sum(send, recv, nlocal, nlocal, nfld, commid=comm) end subroutine compose_repro_sum diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 1ca51dcc85a5..8415975e8778 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -287,7 +287,6 @@ g2f_mixing_ratio (const KernelVariables& kv, const int np2, const int nf2, const const WT& w1, const WT& w2, const int iqf, const QT& qf) { using g = GllFvRemapImpl; using Kokkos::parallel_for; - const auto ttrg = Kokkos::TeamThreadRange(kv.team, np2); const auto ttrf = Kokkos::TeamThreadRange(kv.team, nf2); const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev); From 4339c8a1f009dd41ba683dd3e3db9a5a9aa7a5d4 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 25 Oct 2021 19:30:42 -0600 Subject: [PATCH 088/301] Homme/SL: Work around an nvcc issue. As described in SCREAM issue #1277, a certain combination of cuda and gcc versions leads to erroneously parsed code. Work around that issue. --- .../homme/src/share/compose/cedr_bfb_tree_allreduce.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/compose/cedr_bfb_tree_allreduce.cpp b/components/homme/src/share/compose/cedr_bfb_tree_allreduce.cpp index 4a2ac33fa30c..aadbfaecb004 100644 --- a/components/homme/src/share/compose/cedr_bfb_tree_allreduce.cpp +++ b/components/homme/src/share/compose/cedr_bfb_tree_allreduce.cpp @@ -83,8 +83,12 @@ ::allreduce (const ConstRealList& send, const RealList& recv, const bool transpo const auto nf = nfield_; cedr_assert(ns.levels[0].nodes.size() == static_cast(nlocal_)); - // We want to be behaviorally const but still permit lazy finish_setup. - if ( ! bd_.size()) const_cast(this)->finish_setup(); + { // We want to be behaviorally const but still permit lazy finish_setup. + // Cuda 10.1.105 with GCC 8.5.0 incorrectly misses the const_cast; for + // some reason, adding remove_const fixes that. + typedef typename std::remove_const::type MeLcl; + if ( ! bd_.size()) const_cast(this)->finish_setup(); + } const auto buf_host = get_send_host(send); From 51111f3fbb5bdc0c24ac7e2c2f583ef3a164fce8 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 26 Oct 2021 14:14:15 -0600 Subject: [PATCH 089/301] Homme/SL: Fix all remaining Cuda/GCC -Wall -pedantic warnings. This commit removes all remaining warnings in builds with Cuda 10.1.105 and GCC 7.2.0 and 8.5.0. The remaining warnings were (1) 'warning: dynamic initialization in unreachable code' in various spots in compose/ and (2) 'warning: calling a __host__ function from a __host__ __device__ function is not allowed' in HommeFormatArray functions. These are benign, but the goal is to make composec++ and composef90 build free of warnings. (This was already true in non-Cuda builds.) --- .../homme/src/share/compose/compose_cedr.cpp | 7 +++-- .../src/share/compose/compose_cedr_qlt.cpp | 18 ++++++++----- .../src/share/compose/compose_cedr_qlt.hpp | 1 + .../homme/src/share/compose/compose_homme.hpp | 27 ++++++++++++++++++- .../homme/src/share/compose/compose_slmm.cpp | 14 +++++----- .../compose/compose_slmm_islmpi_pack.cpp | 18 ++++++++----- .../share/compose/compose_slmm_islmpi_q.cpp | 18 ++++++++----- 7 files changed, 71 insertions(+), 32 deletions(-) diff --git a/components/homme/src/share/compose/compose_cedr.cpp b/components/homme/src/share/compose/compose_cedr.cpp index 69a940d0ec8b..2b497db6647b 100644 --- a/components/homme/src/share/compose/compose_cedr.cpp +++ b/components/homme/src/share/compose/compose_cedr.cpp @@ -484,11 +484,10 @@ void CDR::init_tracers (const bool need_conservation) { template void CDR::get_buffers_sizes (size_t& s1, size_t &s2) { - if (ko::OnGpu::value) { + if (ko::OnGpu::value) s1 = s2 = 0; - return; - } - cdr->get_buffers_sizes(s1, s2); + else + cdr->get_buffers_sizes(s1, s2); } template diff --git a/components/homme/src/share/compose/compose_cedr_qlt.cpp b/components/homme/src/share/compose/compose_cedr_qlt.cpp index 911632fa555a..04f0f938414f 100644 --- a/components/homme/src/share/compose/compose_cedr_qlt.cpp +++ b/components/homme/src/share/compose/compose_cedr_qlt.cpp @@ -178,18 +178,24 @@ QLT::QLT (const cedr::mpi::Parallel::Ptr& p, const cedr::Int& ncells, : cedr::qlt::QLT(p, ncells, tree, options) { if (vertical_levels) { - cedr_throw_if(ko::OnGpu::value, - "QLT does not yet support vertical_levels on gpu."); - vld_ = std::make_shared(vertical_levels); + if (ko::OnGpu::value) + cedr_throw_if(ko::OnGpu::value, + "QLT does not yet support vertical_levels on gpu."); + else + vld_ = std::make_shared(vertical_levels); } } template void QLT::run () { - if (ko::OnGpu::value) { + if (ko::OnGpu::value) Super::run(); - return; - } + else + runimpl(); +} + +template +void QLT::runimpl () { static const int mpitag = 42; using cedr::Int; using cedr::Real; diff --git a/components/homme/src/share/compose/compose_cedr_qlt.hpp b/components/homme/src/share/compose/compose_cedr_qlt.hpp index dd51b77a7679..008fbf1801b3 100644 --- a/components/homme/src/share/compose/compose_cedr_qlt.hpp +++ b/components/homme/src/share/compose/compose_cedr_qlt.hpp @@ -17,6 +17,7 @@ class QLT : public cedr::qlt::QLT { void reconcile_vertical(const Int problem_type, const Int bd_os, const Int bis, const Int bie); + void runimpl(); public: QLT(const cedr::mpi::Parallel::Ptr& p, const cedr::Int& ncells, diff --git a/components/homme/src/share/compose/compose_homme.hpp b/components/homme/src/share/compose/compose_homme.hpp index a4bcab8bb644..c1a9e6bb050b 100644 --- a/components/homme/src/share/compose/compose_homme.hpp +++ b/components/homme/src/share/compose/compose_homme.hpp @@ -142,7 +142,13 @@ struct HommeFormatArray { static_assert(rank == 2, "rank 2 array"); assert(i >= 0); assert(ie_data_ptr[ie]); + // These routines are not used on the GPU, but they can be called from + // KOKKOS_FUNCTIONs on CPU in GPU builds. Avoid nvcc warnings as follows: +#ifdef __CUDA_ARCH__ + return unused(); +#else return *(ie_data_ptr[ie] + i); +#endif } COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& k, const Int& lev) const { @@ -151,7 +157,11 @@ struct HommeFormatArray { assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev); +#ifdef __CUDA_ARCH__ + return unused(); +#else return *(ie_data_ptr[ie] + lev*np2 + k); +#endif } COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& q_or_timelev, const Int& k, @@ -162,7 +172,11 @@ struct HommeFormatArray { assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q_or_timelev); +#ifdef __CUDA_ARCH__ + return unused(); +#else return *(ie_data_ptr[ie] + (q_or_timelev*nlev + lev)*np2 + k); +#endif } COMPOSE_FORCEINLINE_FUNCTION T& operator() (const Int& ie, const Int& timelev, const Int& q, const Int& k, @@ -174,7 +188,11 @@ struct HommeFormatArray { assert(lev >= 0); assert(ie_data_ptr[ie]); check(ie, k, lev, q, timelev); +#ifdef __CUDA_ARCH__ + return unused(); +#else return *(ie_data_ptr[ie] + ((timelev*qsized + q)*nlev + lev)*np2 + k); +#endif } private: @@ -182,10 +200,17 @@ struct HommeFormatArray { std::vector ie_data_ptr; const Int nlev, qsized, ntimelev; +#ifdef KOKKOS_ENABLE_CUDA + COMPOSE_INLINE_FUNCTION static T& unused () { + static T unused = 0; + return unused; + } +#endif + COMPOSE_FORCEINLINE_FUNCTION void check (Int ie, Int k = -1, Int lev = -1, Int q_or_timelev = -1, Int timelev = -1) const { -#ifdef COMPOSE_BOUNDS_CHECK +#if defined COMPOSE_BOUNDS_CHECK && ! defined __CUDA_ARCH__ assert(ie >= 0 && ie < static_cast(ie_data_ptr.size())); if (k >= 0) assert(k < np2); if (lev >= 0) assert(lev < nlev); diff --git a/components/homme/src/share/compose/compose_slmm.cpp b/components/homme/src/share/compose/compose_slmm.cpp index 3d43a766bcb5..7006ff14c618 100644 --- a/components/homme/src/share/compose/compose_slmm.cpp +++ b/components/homme/src/share/compose/compose_slmm.cpp @@ -316,15 +316,15 @@ void slmm_init_impl ( void slmm_query_bufsz (homme::Int* sendsz, homme::Int* recvsz) { slmm_assert(homme::g_csl_mpi); - if (ko::OnGpu::value) { + if (ko::OnGpu::value) *sendsz = *recvsz = 0; - return; + else { + homme::Int s = 0, r = 0; + for (const auto e : homme::g_csl_mpi->sendsz) s += e; + for (const auto e : homme::g_csl_mpi->recvsz) r += e; + *sendsz = s; + *recvsz = r; } - homme::Int s = 0, r = 0; - for (const auto e : homme::g_csl_mpi->sendsz) s += e; - for (const auto e : homme::g_csl_mpi->recvsz) r += e; - *sendsz = s; - *recvsz = r; } void slmm_init_plane (homme::Real Sx, homme::Real Sy, homme::Real Lx, homme::Real Ly) { diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp index de66e7ef18e7..4fe325728dfa 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_pack.cpp @@ -135,14 +135,8 @@ void pack_dep_points_sendbuf_pass1_scan (IslMpi& cm) { *#x) *#lev *#lid *#rank */ template -void pack_dep_points_sendbuf_pass1 (IslMpi& cm) { +void pack_dep_points_sendbuf_pass1_noscan (IslMpi& cm) { #ifdef COMPOSE_PORT -# ifndef COMPOSE_PACK_NOSCAN - if (ko::OnGpu::value) { - pack_dep_points_sendbuf_pass1_scan(cm); - return; - } -# endif ko::fence(); deep_copy(cm.nx_in_rank_h, cm.nx_in_rank); deep_copy(cm.nx_in_lid_h, cm.nx_in_lid); @@ -215,6 +209,16 @@ void pack_dep_points_sendbuf_pass1 (IslMpi& cm) { #endif } +template +void pack_dep_points_sendbuf_pass1 (IslMpi& cm) { +#if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN + if (ko::OnGpu::value) + pack_dep_points_sendbuf_pass1_scan(cm); + else +#endif + pack_dep_points_sendbuf_pass1_noscan(cm); +} + template void pack_dep_points_sendbuf_pass2 (IslMpi& cm, const DepPoints& dep_points) { const auto myrank = cm.p->rank(); diff --git a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp index b49c228b545d..c6633a0f5aeb 100644 --- a/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp +++ b/components/homme/src/share/compose/compose_slmm_islmpi_q.cpp @@ -594,13 +594,7 @@ void calc_rmt_q_pass2 (IslMpi& cm) { #endif // COMPOSE_PORT template -void calc_rmt_q_pass1 (IslMpi& cm) { -#if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN - if (ko::OnGpu::value) { - calc_rmt_q_pass1_scan(cm); - return; - } -#endif +void calc_rmt_q_pass1_noscan (IslMpi& cm) { const Int nrmtrank = static_cast(cm.ranks.size()) - 1; #ifdef COMPOSE_PORT_SEPARATE_VIEWS for (Int ri = 0; ri < nrmtrank; ++ri) @@ -667,6 +661,16 @@ void calc_rmt_q_pass1 (IslMpi& cm) { deep_copy(cm.rmt_qs_extrema, cm.rmt_qs_extrema_h); } +template +void calc_rmt_q_pass1 (IslMpi& cm) { +#if defined COMPOSE_PORT && ! defined COMPOSE_PACK_NOSCAN + if (ko::OnGpu::value) + calc_rmt_q_pass1_scan(cm); + else +#endif + calc_rmt_q_pass1_noscan(cm); +} + template void calc_rmt_q (IslMpi& cm) { { slmm::Timer t("09_rmt_q_pass1"); From f9b723d76a987a8dd981827f3fe214214be7c781 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 26 Oct 2021 14:51:54 -0600 Subject: [PATCH 090/301] Homme/SL: Add an assertion to codify 'unused'. --- components/homme/src/share/compose/compose_homme.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/homme/src/share/compose/compose_homme.hpp b/components/homme/src/share/compose/compose_homme.hpp index c1a9e6bb050b..d53cf90ccf2d 100644 --- a/components/homme/src/share/compose/compose_homme.hpp +++ b/components/homme/src/share/compose/compose_homme.hpp @@ -203,6 +203,7 @@ struct HommeFormatArray { #ifdef KOKKOS_ENABLE_CUDA COMPOSE_INLINE_FUNCTION static T& unused () { static T unused = 0; + assert(0); return unused; } #endif From 30f42d3122dea1b998db8c007444e5b62c245567 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 27 Oct 2021 16:13:40 -0500 Subject: [PATCH 091/301] minor --- .../homme/src/test_src/held_suarez_mod.F90 | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/components/homme/src/test_src/held_suarez_mod.F90 b/components/homme/src/test_src/held_suarez_mod.F90 index 34a969257f9b..7bcd69353834 100644 --- a/components/homme/src/test_src/held_suarez_mod.F90 +++ b/components/homme/src/test_src/held_suarez_mod.F90 @@ -112,20 +112,6 @@ subroutine hs_forcing(elemin,hvcoord,nm1,nm1_Q,dt) enddo endif -#if 0 -!bfb version -elemin%derived%FM(:,:,1:3,:) = 0.0 -elemin%derived%FQ(:,:,:,:) = 0.01 -elemin%derived%FT(:,:,:) = 0.0 -#endif - -elemin%derived%FM(:,:,1:3,:) = 0.0 -!elemin%derived%FQ(:,:,:,:) = 0.0 -elemin%derived%FT(:,:,:) = 0.0 - - - - end subroutine hs_forcing function hs_v_forcing(hvcoord,ps,v,npts,nlevels) result(hs_v_frc) @@ -263,6 +249,10 @@ subroutine hs0_init_state(elem, hybrid, hvcoord,nets,nete,Tinit) elem(ie)%state%v(:,:,:,:,nm1)=elem(ie)%state%v(:,:,:,:,n0) elem(ie)%state%v(:,:,:,:,np1)=elem(ie)%state%v(:,:,:,:,n0) +#ifdef MODEL_THETA_L + elem(ie)%state%w_i = 0.0 +#endif + temperature(:,:,:)=Tinit ! if topo file was given in the namelist, PHIS was initilized in prim_main @@ -314,6 +304,7 @@ subroutine hs0_init_state(elem, hybrid, hvcoord,nets,nete,Tinit) endif ps=elem(ie)%state%ps_v(:,:,n0) call set_thermostate(elem(ie),ps,temperature,hvcoord) + end do end subroutine hs0_init_state From 62ab0ddea7dcb019513cd1989b01dd5d0b631974 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 27 Oct 2021 15:22:06 -0700 Subject: [PATCH 092/301] Add a flag to remove a certain optimization for one MPAS file -- only for GNU compilers. Only needed for gnu9 and higher, but is safe with lower versions. --- cime_config/machines/Depends.gnu.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cime_config/machines/Depends.gnu.cmake b/cime_config/machines/Depends.gnu.cmake index 81087ea2460f..25c4fba8ba98 100644 --- a/cime_config/machines/Depends.gnu.cmake +++ b/cime_config/machines/Depends.gnu.cmake @@ -1 +1,11 @@ e3sm_add_flags("eam/src/dynamics/fv/geopk.F90" "-fcray-pointer") + +list(APPEND MPAS_ICE_SHORTWAVE + ${CMAKE_BINARY_DIR}/core_seaice/column/ice_shortwave.f90 +) + +if (NOT DEBUG) + foreach(ITEM IN LISTS MPAS_ICE_SHORTWAVE) + e3sm_add_flags("${ITEM}" "-fno-tree-pta") # avoids an error that shows up in solver with gnu9 and higher + endforeach() +endif() From 2fdbcc8182c14ac47d874b69a7a695c3d80fb902 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 27 Oct 2021 23:55:30 -0500 Subject: [PATCH 093/301] logic around push to/from c --- components/homme/CMakeLists.txt | 9 ++ components/homme/cmake/HommeMacros.cmake | 4 + components/homme/src/share/control_mod.F90 | 4 +- components/homme/src/test_mod.F90 | 25 +++-- .../src/theta-l_kokkos/prim_driver_mod.F90 | 94 +++++++++++++++---- 5 files changed, 108 insertions(+), 28 deletions(-) diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 77cbb78a2d92..f90021e1441a 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -53,6 +53,15 @@ IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) MESSAGE(STATUS "This configuration builds without PIO library and NetCDF calls") ENDIF() +#option to discard forcings call and push to/from F +#set it on in cmake file +SET(BUILD_HOMMEXX_BENCHMARK_NOFORCING OFF CACHE BOOL "") +if(BUILD_HOMMEXX_BENCHMARK_NOFORCING) + message(STATUS "Build without test forcing and pushes to/from F, PERFORMANCE ONLY") +else() + message(STATUS "Option BUILD_HOMMEXX_BENCHMARK_NOFORCING is OFF") +endif() + # Add the cmake/CMakeLists.txt file SET (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 5d10ef2d0fdf..08f86a1327c5 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -116,6 +116,10 @@ macro(createTestExec execName execType macroNP macroNC IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() + IF(BUILD_HOMMEXX_BENCHMARK_NOFORCING) + TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMMEXX_BENCHMARK_NOFORCING) + ENDIF() + IF (CXXLIB_SUPPORTED_CACHE) MESSAGE(STATUS " Linking Fortran with -cxxlib") diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index bd535a5c0aad..03bbd31f1fd3 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -117,7 +117,9 @@ module control_mod integer , public :: partmethod ! partition methods character(len=MAX_STRING_LEN) , public :: topology = "cube" ! options: "cube", "plane" character(len=MAX_STRING_LEN) , public :: geometry = "sphere" ! options: "sphere", "plane" - character(len=MAX_STRING_LEN) , public :: test_case + character(len=MAX_STRING_LEN) , public :: test_case + !most tests don't have forcing + logical , public :: test_with_forcing = .false. integer , public :: tasknum integer , public :: statefreq ! output frequency of synopsis of system state (steps) integer , public :: restartfreq diff --git a/components/homme/src/test_mod.F90 b/components/homme/src/test_mod.F90 index eddc4bbe01c5..dd4cc4b336d2 100644 --- a/components/homme/src/test_mod.F90 +++ b/components/homme/src/test_mod.F90 @@ -4,7 +4,7 @@ module test_mod -use control_mod, only: test_case, sub_case, dt_remap_factor, runtype, bubble_moist +use control_mod, only: test_case, sub_case, dt_remap_factor, runtype, bubble_moist, test_with_forcing use dimensions_mod, only: np, nlev, nlevp, qsize use derivative_mod, only: derivative_t, gradient_sphere use element_mod, only: element_t @@ -55,6 +55,8 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n type(timelevel_t), intent(in) :: tl ! time level sctructure integer, intent(in) :: nets,nete ! start, end element index +!which ones are with forcing???? + ! init calls for any runtype select case(test_case) case('asp_baroclinic'); @@ -72,15 +74,15 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('dcmip2012_test2_2'); case('dcmip2012_test3'); case('dcmip2012_test4'); - case('dcmip2016_test1'); call dcmip2016_init(); + case('dcmip2016_test1'); call dcmip2016_init(); test_with_forcing = .true. ; case('dcmip2016_test1_pg1', 'dcmip2016_test1_pg2', 'dcmip2016_test1_pg3', 'dcmip2016_test1_pg4') - call dcmip2016_init(); - case('dcmip2016_test2'); call dcmip2016_init(); - case('dcmip2016_test3'); call dcmip2016_init(); + call dcmip2016_init(); test_with_forcing = .true. ; + case('dcmip2016_test2'); call dcmip2016_init(); test_with_forcing = .true. ; + case('dcmip2016_test3'); call dcmip2016_init(); test_with_forcing = .true. ; case('mtest1'); case('mtest2'); case('mtest3'); - case('held_suarez0'); + case('held_suarez0'); test_with_forcing = .true. ; case('jw_baroclinic'); case('planar_hydro_gravity_wave'); case('planar_nonhydro_gravity_wave'); @@ -88,14 +90,17 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('planar_nonhydro_mtn_wave'); case('planar_schar_mtn_wave'); case('planar_rising_bubble'); - if (bubble_moist) call dcmip2016_init(); + if (bubble_moist) then + call dcmip2016_init(); + test_with_forcing = .true. ; + endif case('planar_density_current'); case('planar_baroclinic_instab'); case('planar_moist_rising_bubble'); case('planar_moist_density_current'); - case('planar_moist_baroclinic_instab'); - case('planar_tropical_cyclone'); - case('planar_supercell'); + case('planar_moist_baroclinic_instab'); + case('planar_tropical_cyclone'); + case('planar_supercell'); case default; call abortmp('unrecognized test case') endselect diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index dc26cb120e9c..2a47d71e898c 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -22,6 +22,9 @@ module prim_driver_mod public :: prim_init_ref_states_views public :: prim_init_diags_views + logical, private :: compute_forcing_and_push_to_c + logical, private :: push_to_f + contains subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) @@ -406,7 +409,8 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv compute_diagnostics = .false. endif - !dt_q = dt*dt_tracer_factor + call init_logic_for_push_to_c() + if (dt_remap_factor == 0) then dt_remap = dt else @@ -415,21 +419,14 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) -!call from driver mod -! call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) - -#if !defined(CAM) && !defined(SCREAM) - call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) -#endif - -#ifndef SCREAM - ! Scream already computes all forcing using the same pointers - ! stored in Hommexx, so the forcing is already up to date - call t_startf('push_to_cxx') - call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & - elem_derived_FPHI, elem_derived_FQ) - call t_stopf('push_to_cxx') -#endif + if (compute_forcing_and_push_to_c) then +! if ( .true. ) then + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) + call t_startf('push_to_cxx') + call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & + elem_derived_FPHI, elem_derived_FQ) + call t_stopf('push_to_cxx') + endif call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep) @@ -439,7 +436,11 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv tl%n0 = n0_c + 1 tl%np1 = np1_c + 1 - if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then + call init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics) + + if (push_to_f) then +! if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then +! if (.true.) then ! Set pointers to states elem_state_v_ptr = c_loc(elem_state_v) elem_state_w_i_ptr = c_loc(elem_state_w_i) @@ -507,4 +508,63 @@ subroutine setup_element_pointers (elem) enddo end subroutine setup_element_pointers + + +!the next 2 routines have logic for push to/from F and for forcing routine +! +!there are 3 cases: +! +!performance: +! (no forcing, no push to c) -> (subcycle) -> (no push to f) +! +!test without forcing (can be performant if output is only at the end): +! (no forcing, no push to c) -> (subcycle) -> (push to f only for output/diagnostics) +! +!test with forcing (not performant): +! (always forcing and push to c) -> (subcycle) -> (always push to f) + + subroutine init_logic_for_push_to_c() + + use control_mod, only: test_with_forcing + + compute_forcing_and_push_to_c = .false. + + ! Scream already computes all forcing using the same pointers + ! stored in Hommexx, so the forcing is already up to date +#if defined(HOMMEXX_BENCHMARK_NOFORCING) || defined(SCREAM) + +#else + if(test_with_forcing)then + compute_forcing_and_push_to_c = .true. + endif +#endif + + end subroutine init_logic_for_push_to_c + + + subroutine init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics) + + use control_mod, only: test_with_forcing + use time_mod, only : timelevel_t + + type (TimeLevel_t), intent(in) :: tl + integer, intent(in) :: statefreq, nextOutputStep + logical, intent(in) :: compute_diagnostics + +#ifdef HOMMEXX_BENCHMARK_NOFORCING + push_to_f = .false. +#else + + if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then + push_to_f = .true. + endif + if (test_with_forcing) then + push_to_f = .true. + endif + +#endif + + end subroutine init_logic_for_push_to_f + + end module From 106c802d5180c43b4e3c437a5c168ceac1b95f71 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 31 Oct 2021 21:43:28 -0500 Subject: [PATCH 094/301] remove unused routine --- components/homme/src/share/cxx/prim_step.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index b6ce10e77de1..aaaebf944bdf 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -135,7 +135,6 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { const bool forcing_0or2 = (params.ftype == ForcingAlg::FORCING_DEBUG || params.ftype == ForcingAlg::FORCING_2); apply_forcing = forcing_0or2; - apply_test_forcing(); #endif if (apply_forcing) { From cef3e972257658b76bc7cd3dfda84be6d391e25a Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 31 Oct 2021 21:45:03 -0500 Subject: [PATCH 095/301] add boolean to the rest of tests --- components/homme/src/test_mod.F90 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/homme/src/test_mod.F90 b/components/homme/src/test_mod.F90 index dd4cc4b336d2..4dfb60c68f21 100644 --- a/components/homme/src/test_mod.F90 +++ b/components/homme/src/test_mod.F90 @@ -70,8 +70,8 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n case('dcmip2012_test1_2'); case('dcmip2012_test1_3'); case('dcmip2012_test2_0'); - case('dcmip2012_test2_1'); - case('dcmip2012_test2_2'); + case('dcmip2012_test2_1'); test_with_forcing = .true. ; + case('dcmip2012_test2_2'); test_with_forcing = .true. ; case('dcmip2012_test3'); case('dcmip2012_test4'); case('dcmip2016_test1'); call dcmip2016_init(); test_with_forcing = .true. ; @@ -79,9 +79,9 @@ subroutine set_test_initial_conditions(elem, deriv, hybrid, hvcoord, tl, nets, n call dcmip2016_init(); test_with_forcing = .true. ; case('dcmip2016_test2'); call dcmip2016_init(); test_with_forcing = .true. ; case('dcmip2016_test3'); call dcmip2016_init(); test_with_forcing = .true. ; - case('mtest1'); - case('mtest2'); - case('mtest3'); + case('mtest1'); test_with_forcing = .true. ; + case('mtest2'); test_with_forcing = .true. ; + case('mtest3'); test_with_forcing = .true. ; case('held_suarez0'); test_with_forcing = .true. ; case('jw_baroclinic'); case('planar_hydro_gravity_wave'); From 33958d7f47f28835983ae91f6fccd8880a0ffd6d Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 31 Oct 2021 21:48:19 -0500 Subject: [PATCH 096/301] redo set of tests --- .../homme/test/reg_test/namelists/thetahs1.nl | 78 +++++++++++++++++++ .../homme/test/reg_test/namelists/thetahs2.nl | 78 +++++++++++++++++++ .../test/reg_test/run_tests/test-list.cmake | 4 +- .../test/reg_test/run_tests/testing-utils.sh | 3 + .../run_tests/theta-fhs1-kokkos.cmake | 18 +++++ .../test/reg_test/run_tests/theta-fhs1.cmake | 18 +++++ .../run_tests/theta-fhs2-kokkos.cmake | 18 +++++ .../test/reg_test/run_tests/theta-fhs2.cmake | 18 +++++ 8 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 components/homme/test/reg_test/namelists/thetahs1.nl create mode 100644 components/homme/test/reg_test/namelists/thetahs2.nl create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs1-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs1.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs2-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs2.cmake diff --git a/components/homme/test/reg_test/namelists/thetahs1.nl b/components/homme/test/reg_test/namelists/thetahs1.nl new file mode 100644 index 000000000000..060e68860e05 --- /dev/null +++ b/components/homme/test/reg_test/namelists/thetahs1.nl @@ -0,0 +1,78 @@ +&ctl_nl +vthreads = 1 +NThreads = 1 +partmethod = 4 +topology = "cube" +test_case = "held_suarez0" +u_perturb = 1 +rotate_grid = 0 +ne = ${HOMME_TEST_NE} +qsize = 1 +ndays = ${HOMME_TEST_NDAYS} +statefreq = 9999 +restartfreq = 43200 +restartfile = "./R0001" +runtype = 0 +mesh_file = '/dev/null' +tstep = 100 +rsplit = 3 +qsplit = 1 +transport_alg = 0 +integration = "explicit" +smooth = 0 +nu = 3.4e-8 +nu_div = 3.4e-8 ! test this too? +nu_p = -1 +nu_q = -1 +nu_s = -1 +nu_top = 0 !2.5e5 +se_ftype = 0 +limiter_option = 9 +vert_remap_q_alg = 1 +hypervis_scaling = 3.0 +hypervis_order = 2 +hypervis_subcycle = 5 +hypervis_subcycle_tom = 6 +theta_hydrostatic_mode = false +theta_advect_form = 1 +tstep_type = 10 +moisture = 'notdry' +/ +&solver_nl +precon_method = "identity" +maxits = 500 +tol = 1.e-9 +/ +&filter_nl +filter_type = "taylor" +transfer_type = "bv" +filter_freq = 0 +filter_mu = 0.04D0 +p_bv = 12.0D0 +s_bv = .666666666666666666D0 +wght_fm = 0.10D0 +kcut_fm = 2 +/ +&vert_nl +vform = "ccm" +vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' +vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' +/ + +&prof_inparm +profile_outpe_num = 100 +profile_single_file = .true. +/ + +! timunits: 0= steps, 1=days, 2=hours +&analysis_nl + interp_gridtype = 2 + output_timeunits = 1,1 + output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} + output_start_time = 0,0 + output_end_time = 30000,30000 + output_varnames1 = 'ps','zeta','u','v','T' + output_varnames2 = 'Q','Q2','Q3','Q4' + io_stride = 8 + output_type = 'netcdf' +/ diff --git a/components/homme/test/reg_test/namelists/thetahs2.nl b/components/homme/test/reg_test/namelists/thetahs2.nl new file mode 100644 index 000000000000..bf76dc17a70d --- /dev/null +++ b/components/homme/test/reg_test/namelists/thetahs2.nl @@ -0,0 +1,78 @@ +&ctl_nl +vthreads = 1 +NThreads = 1 +partmethod = 4 +topology = "cube" +test_case = "held_suarez0" +u_perturb = 1 +rotate_grid = 0 +ne = ${HOMME_TEST_NE} +qsize = 1 +ndays = ${HOMME_TEST_NDAYS} +statefreq = 9999 +restartfreq = 43200 +restartfile = "./R0001" +runtype = 0 +mesh_file = '/dev/null' +tstep = 100 +dt_remap_factor = 3 +dt_tracer_factor = 6 +transport_alg = 12 +integration = "explicit" +smooth = 0 +nu = 3.4e-8 +nu_div = 3.4e-8 ! test this too? +nu_p = -1 +nu_q = -1 +nu_s = -1 +nu_top = 0 !2.5e5 +se_ftype = 0 +limiter_option = 9 +vert_remap_q_alg = 1 +hypervis_scaling = 3.0 +hypervis_order = 2 +hypervis_subcycle = 5 +hypervis_subcycle_tom = 6 +theta_hydrostatic_mode = false +theta_advect_form = 1 +tstep_type = 10 +moisture = 'notdry' +/ +&solver_nl +precon_method = "identity" +maxits = 500 +tol = 1.e-9 +/ +&filter_nl +filter_type = "taylor" +transfer_type = "bv" +filter_freq = 0 +filter_mu = 0.04D0 +p_bv = 12.0D0 +s_bv = .666666666666666666D0 +wght_fm = 0.10D0 +kcut_fm = 2 +/ +&vert_nl +vform = "ccm" +vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' +vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' +/ + +&prof_inparm +profile_outpe_num = 100 +profile_single_file = .true. +/ + +! timunits: 0= steps, 1=days, 2=hours +&analysis_nl + interp_gridtype = 2 + output_timeunits = 1,1 + output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} + output_start_time = 0,0 + output_end_time = 30000,30000 + output_varnames1 = 'ps','zeta','u','v','T' + output_varnames2 = 'Q','Q2','Q3','Q4' + io_stride = 8 + output_type = 'netcdf' +/ diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index 5a48195ef0c4..b847937f1551 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -90,9 +90,11 @@ IF (BUILD_HOMME_THETA_KOKKOS) ) #manually added, create namelists and cmake files for F and cxx by hand + #all have to be named theta-f* to make setup work (this could be fixed) SET(TESTS_WITHOUT_AUTO theta-f1-tt10-hvs1-hvst0-r2-qz10-nutopoff-GB-sl - theta-f1-tt10-HS + theta-fhs1 + theta-fhs2 ) #all tests that will be used for cxx-vs-F bfb testing diff --git a/components/homme/test/reg_test/run_tests/testing-utils.sh b/components/homme/test/reg_test/run_tests/testing-utils.sh index d934c86f2769..b111caaae89b 100644 --- a/components/homme/test/reg_test/run_tests/testing-utils.sh +++ b/components/homme/test/reg_test/run_tests/testing-utils.sh @@ -725,6 +725,9 @@ diffCprncOutput() { # Parse the output file to determine if they were identical DIFF_RESULT=`grep -ae 'diff_test' ${cprncOutputFile} | awk '{ print $8 }'` + +# Dimension time differs + if [ "${DIFF_RESULT}" == IDENTICAL ] ; then # check for missing variables NUMVARS_RESULT=`grep -ae 'were not found on' ${cprncOutputFile} | awk '{ print $5 }'` diff --git a/components/homme/test/reg_test/run_tests/theta-fhs1-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fhs1-kokkos.cmake new file mode 100644 index 000000000000..b9d880870595 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs1-kokkos.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs1-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs1.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fhs1.cmake b/components/homme/test/reg_test/run_tests/theta-fhs1.cmake new file mode 100644 index 000000000000..bdbdc736a32b --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs1.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs1) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs1.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fhs2-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fhs2-kokkos.cmake new file mode 100644 index 000000000000..a6b8d7dc1a13 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs2-kokkos.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs2-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs2.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fhs2.cmake b/components/homme/test/reg_test/run_tests/theta-fhs2.cmake new file mode 100644 index 000000000000..2b25fce4093f --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs2.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs2) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs2.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) From 27da2952e8dc40c22478682e08272efadc03726b Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 31 Oct 2021 21:48:49 -0500 Subject: [PATCH 097/301] change dt for _flexible routine --- components/homme/src/theta-l_kokkos/prim_driver_mod.F90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 2a47d71e898c..cc168f09532a 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -391,7 +391,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv type (c_ptr) :: elem_derived_omega_p_ptr integer :: n0_qdp, np1_qdp - real(kind=real_kind) :: dt_remap + real(kind=real_kind) :: dt_remap, dt_q if (nets/=1 .or. nete/=nelemd) then call abortmp ('We don''t allow to call C routines from a horizontally threaded region') @@ -411,6 +411,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv call init_logic_for_push_to_c() + dt_q = dt*dt_tracer_factor if (dt_remap_factor == 0) then dt_remap = dt else @@ -421,7 +422,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv if (compute_forcing_and_push_to_c) then ! if ( .true. ) then - call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,dt_remap,nets,nete,tl) + call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) call t_startf('push_to_cxx') call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & elem_derived_FPHI, elem_derived_FQ) From 68ec7056a51ada7624bd2990e73f13dd09ba26a8 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 1 Nov 2021 00:01:05 -0600 Subject: [PATCH 098/301] Homme/SL: Fix qsize_d vs qsize usage. Hommexx's qdp uses qsize_d, but q now uses qsize so that SCREAM's AD can doesn't need to know qsize_d. Fix the Hommexx <-> SL view handoff that handles these details. This commit affects only the Hommexx dycore; the F90 dycore uses a different interface. --- components/homme/src/share/compose/compose_hommexx.cpp | 5 ++--- .../homme/src/share/cxx/ComposeTransportImplGeneral.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/homme/src/share/compose/compose_hommexx.cpp b/components/homme/src/share/compose/compose_hommexx.cpp index 7b9f493a7c2e..fdc6cdf4bcf5 100644 --- a/components/homme/src/share/compose/compose_hommexx.cpp +++ b/components/homme/src/share/compose/compose_hommexx.cpp @@ -30,12 +30,11 @@ void set_views (const SetView& spheremp, const auto nel = spheremp.extent_int(0); const auto np2 = spheremp.extent_int(1)*spheremp.extent_int(1); const auto nlev = dp.extent_int(3); - const auto qsize_d = qdp.extent_int(2); ta.spheremp = View(spheremp.data(), nel, np2); ta.dp = View(dp.data(), nel, np2, nlev); ta.dp3d = View(dp3d.data(), nel, dp3d.extent_int(1), np2, nlev); - ta.qdp = View(qdp.data(), nel, qdp.extent_int(1), qsize_d, np2, nlev); - ta.q = View(q.data(), nel, qsize_d, np2, nlev); + ta.qdp = View(qdp.data(), nel, qdp.extent_int(1), qdp.extent_int(2), np2, nlev); + ta.q = View(q.data(), nel, q.extent_int(1), np2, nlev); ta.dep_points = View(dep_points.data(), nel, dep_points.extent_int(1), np2); #else slmm_throw_if(true, "Running a Hommexx code path with the non-Hommexx build" diff --git a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp index 671c66449643..095b3bb104db 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplGeneral.cpp @@ -68,10 +68,11 @@ void ComposeTransportImpl::reset (const SimulationParams& params) { d.m_divdp.data() : s.m_dp3d.data()), nel, (independent_time_steps ? 1 : NUM_TIME_LEVELS), np, np, nlev), - homme::compose::SetView(reinterpret_cast(t.qdp.data()), - nel, Q_NUM_TIME_LEVELS, QSIZE_D, np, np, nlev), + homme::compose::SetView( + reinterpret_cast(t.qdp.data()), + nel, t.qdp.extent_int(1), t.qdp.extent_int(2), np, np, nlev), homme::compose::SetView (reinterpret_cast(t.Q.data()), - nel, QSIZE_D, np, np, nlev), + nel, t.Q.extent_int(1), np, np, nlev), m_data.dep_pts); } m_data.independent_time_steps = independent_time_steps; From 1f3fe54c563db7394b7675146c61fa1c4e5b6627 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 1 Nov 2021 00:15:20 -0600 Subject: [PATCH 099/301] Homme/SL: Fix uninit'ed memory in compose_ut test. When ! defined HOMMEXX_BFB_TESTING, RemapStateProvider.hpp needs to know theta_hydrostatic_mode. Init this parameter in the unit tests. --- .../homme/test_execs/thetal_kokkos_ut/compose_interface.F90 | 4 +++- components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 index 76cb8417fd7d..4296b6609a08 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_interface.F90 @@ -15,7 +15,8 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, use edge_mod_base, only: initEdgeBuffer, edge_g use control_mod, only: transport_alg, semi_lagrange_cdr_alg, semi_lagrange_cdr_check, & semi_lagrange_hv_q, limiter_option, nu_q, hypervis_subcycle_q, hypervis_order, & - vert_remap_q_alg, qsplit, rsplit, dt_remap_factor, dt_tracer_factor + vert_remap_q_alg, qsplit, rsplit, dt_remap_factor, dt_tracer_factor, & + theta_hydrostatic_mode use geometry_interface_mod, only: GridVertex use bndry_mod, only: sort_neighbor_buffer_mapping use reduction_mod, only: initreductionbuffer, red_sum, red_min, red_max @@ -43,6 +44,7 @@ subroutine init_compose_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, rsplit = 1 dt_tracer_factor = -1 dt_remap_factor = -1 + theta_hydrostatic_mode = .true. hypervis_order = 2 semi_lagrange_hv_q = hv_q diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 0ff378e3d5c4..2a003187cae0 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -132,6 +132,7 @@ struct Session { p.dt_tracer_factor = -1; p.dt_remap_factor = -1; p.params_set = true; + p.theta_hydrostatic_mode = true; const auto hyai = cmvdc(h.hybrid_ai); const auto hybi = cmvdc(h.hybrid_bi); From 91075474fe6bc22f8cda704dae1f474f9ed46988 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 1 Nov 2021 01:25:45 -0500 Subject: [PATCH 100/301] Homme: Modify the new standalone test's parameters. --- components/homme/test/reg_test/namelists/thetahs2.nl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/homme/test/reg_test/namelists/thetahs2.nl b/components/homme/test/reg_test/namelists/thetahs2.nl index bf76dc17a70d..a696f7c6dd1d 100644 --- a/components/homme/test/reg_test/namelists/thetahs2.nl +++ b/components/homme/test/reg_test/namelists/thetahs2.nl @@ -28,10 +28,11 @@ nu_s = -1 nu_top = 0 !2.5e5 se_ftype = 0 limiter_option = 9 -vert_remap_q_alg = 1 +vert_remap_q_alg = 10 hypervis_scaling = 3.0 hypervis_order = 2 hypervis_subcycle = 5 +hypervis_subcycle_q = 6 hypervis_subcycle_tom = 6 theta_hydrostatic_mode = false theta_advect_form = 1 From 2a62584fd5acc875d2b5a8734b18e1e9d400ee87 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 1 Nov 2021 16:40:47 -0500 Subject: [PATCH 101/301] fix ftype2 for standalone --- .../theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 440dec48f061..d70af7eb7d7d 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -225,14 +225,12 @@ void push_forcing_to_c (F90Ptr elem_derived_FM, const SimulationParams ¶ms = Context::singleton().get(); Tracers &tracers = Context::singleton().get(); - if (params.ftype == ForcingAlg::FORCING_DEBUG) { - if (tracers.fq.data() == nullptr) { - tracers.fq = decltype(tracers.fq)("fq", num_elems); - } - HostViewUnmanaged fq_f90( - elem_derived_FQ, num_elems); - sync_to_device(fq_f90, tracers.fq); + if (tracers.fq.data() == nullptr) { + tracers.fq = decltype(tracers.fq)("fq", num_elems); } + HostViewUnmanaged fq_f90( + elem_derived_FQ, num_elems); + sync_to_device(fq_f90, tracers.fq); } void init_reference_element_c (CF90Ptr& deriv, CF90Ptr& mass) From f49c8a1397503af05abdbb178fa58c73ac138fb1 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Mon, 1 Nov 2021 16:26:19 -0600 Subject: [PATCH 102/301] Homme/SL: Move assignment of var outside of ifdef CAM block. This variable is used independently of whether CAM is defined. --- components/homme/src/share/cxx/prim_step.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index aaaebf944bdf..8cdf8c294713 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -128,12 +128,12 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { tl.update_tracers_levels(params.dt_tracer_factor); + const bool forcing_0or2 = (params.ftype == ForcingAlg::FORCING_DEBUG || + params.ftype == ForcingAlg::FORCING_2); bool apply_forcing; #ifdef CAM apply_forcing = params.ftype == ForcingAlg::FORCING_DEBUG; #else - const bool forcing_0or2 = (params.ftype == ForcingAlg::FORCING_DEBUG || - params.ftype == ForcingAlg::FORCING_2); apply_forcing = forcing_0or2; #endif From aa75dc936f086b5725d09273a219a22c52977224 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Fri, 29 Oct 2021 12:36:24 -0600 Subject: [PATCH 103/301] Homme: In FV->GLL remap, fix input handling of q for ftype 0. The previous implementation was based on a misunderstanding of the input Q type when ftype is 0. I had thought both the input (Q) and the output (FQ) were tendencies when ftype=0. But in fact neither is, just like when ftype > 0. In other words, gllfvremap_mod is independent of ftype. Also add another FV<->GLL API test to the C++ unit test. --- components/homme/src/share/cxx/GllFvRemap.cpp | 12 +- components/homme/src/share/cxx/GllFvRemap.hpp | 6 +- .../homme/src/share/cxx/GllFvRemapImpl.cpp | 53 +----- .../homme/src/share/cxx/GllFvRemapImpl.hpp | 6 +- components/homme/src/share/gllfvremap_mod.F90 | 153 +++++------------- .../homme/src/share/gllfvremap_util_mod.F90 | 91 +++++------ .../homme/src/test_src/dcmip16_wrapper.F90 | 32 +++- .../thetal_kokkos_ut/gllfvremap_interface.F90 | 45 +++--- .../thetal_kokkos_ut/gllfvremap_ut.cpp | 108 +++++++++---- 9 files changed, 221 insertions(+), 285 deletions(-) diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index 8d15e669328e..8dab69c27a34 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -15,16 +15,12 @@ namespace Homme { -void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, int ftype, +void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, bool theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); auto& s = c.get(); - assert(ftype >= 0 && ftype <= 4); - s.ftype = (ftype == 0 ? ForcingAlg::FORCING_0 : - ftype == 1 ? ForcingAlg::FORCING_1 : - ForcingAlg::FORCING_2); auto& g = c.get(); g.init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f); @@ -68,9 +64,9 @@ ::run_dyn_to_fv_phys (const int time_idx, const Phys1T& ps, const Phys1T& phis, } void GllFvRemap -::run_fv_phys_to_dyn (const int time_idx, const Real dt, - const CPhys2T& T, const CPhys3T& uv, const CPhys3T& q) { - m_impl->run_fv_phys_to_dyn(time_idx, dt, T, uv, q); +::run_fv_phys_to_dyn (const int time_idx, const CPhys2T& T, const CPhys3T& uv, + const CPhys3T& q) { + m_impl->run_fv_phys_to_dyn(time_idx, T, uv, q); } void GllFvRemap::run_fv_phys_to_dyn_dss () { m_impl->run_fv_phys_to_dyn_dss(); } diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 06116c8ddf40..5f1e6b863cf0 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -49,8 +49,8 @@ class GllFvRemap { const Phys3T& uv, // q(ie,col,idx,lev) const Phys3T& q); - void run_fv_phys_to_dyn(const int time_idx, const Real dt, - const CPhys2T& T, const CPhys3T& uv, const CPhys3T& q); + void run_fv_phys_to_dyn(const int time_idx, const CPhys2T& T, const CPhys3T& uv, + const CPhys3T& q); void run_fv_phys_to_dyn_dss(); private: @@ -58,7 +58,7 @@ class GllFvRemap { }; extern "C" void -init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, int ftype, +init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, const bool theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 1ca51dcc85a5..15365d544e26 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -44,9 +44,9 @@ GllFvRemapImpl::GllFvRemapImpl () m_forcing(Context::singleton().get()), m_geometry(Context::singleton().get()), m_tracers(Context::singleton().get()), - m_tp_ne(1,1,1), m_tu_ne(m_tp_ne), // throwaway settings - m_tp_ne_qsize(1,1,1), m_tu_ne_qsize(m_tp_ne_qsize), // throwaway settings - m_tp_ne_dss(1,1,1), m_tu_ne_dss(m_tp_ne_dss) // throwaway settings + // throwaway settings + m_tp_ne(1,1,1), m_tp_ne_qsize(1,1,1), m_tp_ne_dss(1,1,1), + m_tu_ne(m_tp_ne), m_tu_ne_qsize(m_tp_ne_qsize), m_tu_ne_dss(m_tp_ne_dss) {} void GllFvRemapImpl::reset (const SimulationParams& params) { @@ -130,7 +130,6 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, " nf must be > 1.", Errors::err_not_implemented); auto& sp = Context::singleton().get(); - m_data.q_adjustment = sp.ftype != ForcingAlg::FORCING_0; m_data.use_moisture = sp.moisture == MoistDry::MOIST; // Only in the unit test gllfvremap_ut does theta_hydrostatic_mode not already // == sp.theta_hydrostatic_mode. @@ -287,7 +286,6 @@ g2f_mixing_ratio (const KernelVariables& kv, const int np2, const int nf2, const const WT& w1, const WT& w2, const int iqf, const QT& qf) { using g = GllFvRemapImpl; using Kokkos::parallel_for; - const auto ttrg = Kokkos::TeamThreadRange(kv.team, np2); const auto ttrf = Kokkos::TeamThreadRange(kv.team, nf2); const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlev); @@ -532,8 +530,8 @@ ::run_dyn_to_fv_phys (const int timeidx, const Phys1T& ps, const Phys1T& phis, c } void GllFvRemapImpl:: -run_fv_phys_to_dyn (const int timeidx, const Real dt, - const CPhys2T& Ts, const CPhys3T& uvs, const CPhys3T& qs) { +run_fv_phys_to_dyn (const int timeidx, const CPhys2T& Ts, const CPhys3T& uvs, + const CPhys3T& qs) { #ifdef MODEL_THETA_L using Kokkos::parallel_for; @@ -668,7 +666,6 @@ run_fv_phys_to_dyn (const int timeidx, const Real dt, const auto q_g = m_tracers.Q; const auto fq = m_tracers.fq; const auto qlim = m_tracers.qlim; - const bool q_adjustment = m_data.q_adjustment; const auto tu_ne_qsize = m_tu_ne_qsize; const auto feq = KOKKOS_LAMBDA (const MT& team) { @@ -687,7 +684,7 @@ run_fv_phys_to_dyn (const int timeidx, const Real dt, gll_metdet_ie(&gll_metdet(ie,0,0), np2); const EVU dp_fv_ie(&dp_fv(ie,0,0,0), nf2, nlevpk); - if (q_adjustment) { + { // Get limiter bounds. const evus2 qf_ie(&r2w(1,0,0,0), nf2, nlevpk); loop_ik(ttrf, tvr, [&] (int i, int k) { qf_ie(i,k) = q(ie,i,iq,k); }); @@ -716,36 +713,6 @@ run_fv_phys_to_dyn (const int timeidx, const Real dt, // GLL Q1 const evus_np2_nlev fq_ie(&fq(ie,iq,0,0,0)); loop_ik(ttrg, tvr, [&] (int i, int k) { fq_ie(i,k) = qg_ie(i,k) + dqg_ie(i,k); }); - } else { - // FV Q_ten - const evus2 dqf_ie(&r2w(0,0,0,0), nf2, nlevpk); - loop_ik(ttrf, tvr, [&] (int i, int k) { dqf_ie(i,k) = dt*q(ie,i,iq,k)/dp_fv_ie(i,k); }); - kv.team_barrier(); - // GLL Q_ten - const evucs_np2_nlev dp_g_ie(&dp_g(ie,timeidx,0,0,0)); - const evus_np2_nlev dqg_ie(rw2.data()); - f2g_scalar_dp(kv, nf2, np2, nlevpk, f2g_remapd, fv_metdet_ie, gll_metdet_ie, - dp_fv_ie, dp_g_ie, dqf_ie, evus_np2_nlev(rw1.data()), dqg_ie); - kv.team_barrier(); - // GLL Q1 - const evucs_np2_nlev qg_ie(&q_g(ie,iq,0,0,0)); - const evus_np2_nlev fq_ie(&fq(ie,iq,0,0,0)); - loop_ik(ttrg, tvr, [&] (int i, int k) { fq_ie(i,k) = qg_ie(i,k) + dqg_ie(i,k); }); - kv.team_barrier(); - // GLL Q0 -> FV Q0 - const evus2 qf_ie(&r2w(1,0,0,0), nf2, nlevpk); - g2f_mixing_ratio( - kv, np2, nf2, nlevpk, g2f_remapd, gll_metdet_ie, - w_ff, fv_metdet_ie, dp_g_ie, dp_fv_ie, qg_ie, - evus_np2_nlev(rw1.data()), evus_np2_nlev(rw2.data()), - 0, evus3(qf_ie.data(), nf2, 1, nlevpk)); - kv.team_barrier(); - // FV Q1 - loop_ik(ttrf, tvr, [&] (int i, int k) { qf_ie(i,k) += dqf_ie(i,k); }); - kv.team_barrier(); - // Get limiter bounds. - calc_extrema(kv, nf2, nlevpk, qf_ie, - evus1(&qlim(ie,iq,0,0), nlevpk), evus1(&qlim(ie,iq,1,0), nlevpk)); } }; Kokkos::fence(); @@ -771,14 +738,6 @@ run_fv_phys_to_dyn (const int timeidx, const Real dt, const evus_np2_nlev fq_ie(&fq(ie,iq,0,0,0)); limiter_clip_and_sum(kv.team, np2, nlevpk, 1, gll_spheremp_ie, qmin, qmax, dp_g_ie, evus_np2_nlev(rw1.data()), fq_ie); - if ( ! q_adjustment) { - kv.team_barrier(); - const auto ttrg = Kokkos::TeamThreadRange(kv.team, np2); - const auto tvr = Kokkos::ThreadVectorRange(kv.team, nlevpk); - loop_ik(ttrg, tvr, [&] (int i, int k) { - fq_ie(i,k) = dp_g_ie(i,k)*(fq_ie(i,k) - qg_ie(i,k))/dt; - }); - } }; Kokkos::fence(); parallel_for(m_tp_ne_qsize, geq); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.hpp b/components/homme/src/share/cxx/GllFvRemapImpl.hpp index 98e966d2eb5b..b4bab84e6091 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.hpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.hpp @@ -60,7 +60,7 @@ struct GllFvRemapImpl { struct Data { int nelemd, qsize, nf2, n_dss_fld; - bool q_adjustment, use_moisture, theta_hydrostatic_mode; + bool use_moisture, theta_hydrostatic_mode; static constexpr int nbuf1 = 2, nbuf2 = 1; Buf1 buf1[nbuf1]; @@ -113,8 +113,8 @@ struct GllFvRemapImpl { void run_dyn_to_fv_phys(const int time_idx, const Phys1T& ps, const Phys1T& phis, const Phys2T& T, const Phys2T& omega, const Phys3T& uv, const Phys3T& q); - void run_fv_phys_to_dyn(const int time_idx, const Real dt, - const CPhys2T& T, const CPhys3T& uv, const CPhys3T& q); + void run_fv_phys_to_dyn(const int time_idx, const CPhys2T& T, const CPhys3T& uv, + const CPhys3T& q); void run_fv_phys_to_dyn_dss(); /* Compute pressure level increments on the FV grid given ps on the FV grid. diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index 2b05ab1ac699..95ff46148c71 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -27,7 +27,8 @@ module gllfvremap_mod ! * pg1's OOA is boosted to ~1.6. ! This module works with cubed_sphere_map 0 and 2, but only 2 will ! work in the fully coupled E3SM. - ! This module supports all ftype values 0 to 4. + ! This module supports all ftype values 0 to 4, as the calculations + ! in this module are independent of ftype. ! We find in practice that pg1 is too coarse. To see this, run the ! Homme-standalone test dcmip2016_test1_pg1 and compare results with ! dcmip2016_test1_pg2 and dcmip2016_test1 (np4). pg2 and np4 fields are @@ -182,7 +183,6 @@ subroutine gfr_init(par, elem, nphys, check, boost_pg1) use dimensions_mod, only: nlev use parallel_mod, only: parallel_t, abortmp use quadrature_mod, only : gausslobatto, quadrature_t - use control_mod, only: ftype type (parallel_t), intent(in) :: par type (element_t), intent(in) :: elem(:) @@ -203,8 +203,8 @@ subroutine gfr_init(par, elem, nphys, check, boost_pg1) gfr%tolfac = one if (par%masterproc) then write(iulog,*) 'gfr> Running with dynamics and physics on separate grids (physgrid).' - write(iulog, '(a,i3,a,i2,a,i2,a,l2)') 'gfr> init nphys', nphys, ' check', gfr%check, & - ' ftype', ftype, ' boost_pg1', gfr%boost_pg1 + write(iulog, '(a,i3,a,i2,a,l2)') 'gfr> init nphys', nphys, ' check', gfr%check, & + ' boost_pg1', gfr%boost_pg1 if (nphys == 1) then ! Document state of pg1. dcmip2016_test1 shows it is too coarse. For ! boost_pg1 = true, stepon's DSS loop needs to be separated from its @@ -261,13 +261,13 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET - use control_mod, only: ftype, theta_hydrostatic_mode + use control_mod, only: theta_hydrostatic_mode use iso_c_binding, only: c_bool interface - subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, ftype, theta_hydrostatic_mode, & + subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) use iso_c_binding, only: c_bool, c_int, c_double - integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max, ftype + integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max logical (c_bool), value, intent(in) :: theta_hydrostatic_mode real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd @@ -277,7 +277,7 @@ end subroutine init_gllfvremap_c end interface logical (c_bool) :: thm thm = theta_hydrostatic_mode - call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, ftype, thm, & + call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif end subroutine gfr_init_hxx @@ -375,17 +375,15 @@ subroutine gfr_dyn_to_fv_phys_hybrid(hybrid, nt, hvcoord, elem, nets, nete, & end do if (gfr%check > 0) then - nerr = check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & - .true., .true., q) + nerr = check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, .true., q) if (nerr > 0) gfr%check_ok = .false. end if end subroutine gfr_dyn_to_fv_phys_hybrid - subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, T, uv, q) - ! Remap T, uv, q states or tendencies from FV to GLL grids. - ! If ftype is in 1:4, then q is the full mixing ratio state and - ! dt is not used; if it is not, then q is Qdp tendency, and dt - ! must be the correct physics time step. + subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, hvcoord, elem, nets, nete, T, uv, q) + ! Remap T, uv, and q tendencies and from FV to GLL grids. + ! On input, T and uv are tendencies and q is state. + ! On output, FM and FT are tendencies and FQ is state. #ifdef __INTEL_COMPILER # if __INTEL_COMPILER >= 1700 && __INTEL_COMPILER < 1800 @@ -403,11 +401,9 @@ subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, use dimensions_mod, only: nlev use hybvcoord_mod, only: hvcoord_t use physical_constants, only: p0, kappa - use control_mod, only: ftype type (hybrid_t), intent(in) :: hybrid integer, intent(in) :: nt - real(kind=real_kind), intent(in) :: dt type (hvcoord_t), intent(in) :: hvcoord type (element_t), intent(inout) :: elem(:) integer, intent(in) :: nets, nete @@ -417,11 +413,9 @@ subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, real(kind=real_kind), dimension(np*np,nlev) :: wf1, wf2, dp_fv, p_fv real(kind=real_kind) :: qmin, qmax integer :: ie, nf, nf2, k, qsize, qi, nerr - logical :: q_adjustment nf = gfr%nphys nf2 = nf*nf - q_adjustment = ftype >= 1 .and. ftype <= 4 qsize = size(q,3) do ie = nets,nete @@ -448,54 +442,28 @@ subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, #endif do qi = 1,qsize - if (q_adjustment) then - ! FV Q_ten - ! GLL Q0 -> FV Q0 - call gfr_g2f_mixing_ratio(gfr, ie, elem(ie)%metdet, dp, dp_fv, & - elem(ie)%state%Q(:,:,:,qi), wf1) - ! FV Q_ten = FV Q1 - FV Q0 - wf1(:nf2,:) = q(:nf2,:,qi,ie) - wf1(:nf2,:) - if (nf > 1 .or. .not. gfr%boost_pg1) then - ! GLL Q_ten - call gfr_f2g_scalar_dp(gfr, ie, elem(ie)%metdet, dp_fv, dp, wf1, wg1) - ! GLL Q1 - elem(ie)%derived%FQ(:,:,:,qi) = elem(ie)%state%Q(:,:,:,qi) + wg1 - else - ! GLL Q_ten - do k = 1,nlev - elem(ie)%derived%FQ(:,:,k,qi) = wf1(1,k) - end do - end if - ! Get limiter bounds. - do k = 1,nlev - gfr%qmin(k,qi,ie) = minval(q(:nf2,k,qi,ie)) - gfr%qmax(k,qi,ie) = maxval(q(:nf2,k,qi,ie)) - end do + ! FV Q_ten + ! GLL Q0 -> FV Q0 + call gfr_g2f_mixing_ratio(gfr, ie, elem(ie)%metdet, dp, dp_fv, & + elem(ie)%state%Q(:,:,:,qi), wf1) + ! FV Q_ten = FV Q1 - FV Q0 + wf1(:nf2,:) = q(:nf2,:,qi,ie) - wf1(:nf2,:) + if (nf > 1 .or. .not. gfr%boost_pg1) then + ! GLL Q_ten + call gfr_f2g_scalar_dp(gfr, ie, elem(ie)%metdet, dp_fv, dp, wf1, wg1) + ! GLL Q1 + elem(ie)%derived%FQ(:,:,:,qi) = elem(ie)%state%Q(:,:,:,qi) + wg1 else - ! FV Q_ten - wf1(:nf2,:) = dt*q(:nf2,:,qi,ie)/dp_fv(:nf2,:) - if (nf > 1 .or. .not. gfr%boost_pg1) then - ! GLL Q_ten - call gfr_f2g_scalar_dp(gfr, ie, elem(ie)%metdet, dp_fv, dp, wf1, wg1) - ! GLL Q1 - elem(ie)%derived%FQ(:,:,:,qi) = elem(ie)%state%Q(:,:,:,qi) + wg1 - else - ! GLL Q_ten - do k = 1,nlev - elem(ie)%derived%FQ(:,:,k,qi) = wf1(1,k) - end do - end if - ! GLL Q0 -> FV Q0 - call gfr_g2f_mixing_ratio(gfr, ie, elem(ie)%metdet, dp, dp_fv, & - elem(ie)%state%Q(:,:,:,qi), wf2) - ! FV Q1 - wf2(:nf2,:) = wf2(:nf2,:) + wf1(:nf2,:) - ! Get limiter bounds. + ! GLL Q_ten do k = 1,nlev - gfr%qmin(k,qi,ie) = minval(wf2(:nf2,k)) - gfr%qmax(k,qi,ie) = maxval(wf2(:nf2,k)) + elem(ie)%derived%FQ(:,:,k,qi) = wf1(1,k) end do end if + ! Get limiter bounds. + do k = 1,nlev + gfr%qmin(k,qi,ie) = minval(q(:nf2,k,qi,ie)) + gfr%qmax(k,qi,ie) = maxval(q(:nf2,k,qi,ie)) + end do end do end do @@ -524,17 +492,11 @@ subroutine gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, gfr%qmax(:,qi,ie), dp, wg1, elem(ie)%derived%FQ(:,:,:,qi)) if (nerr > 0) gfr%check_ok = .false. end if - if (.not. q_adjustment) then - ! Convert to a tendency. - elem(ie)%derived%FQ(:,:,:,qi) = & - dp*(elem(ie)%derived%FQ(:,:,:,qi) - elem(ie)%state%Q(:,:,:,qi))/dt - end if end do end do if (gfr%check > 0) then - nerr = check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & - .false., q_adjustment, q) + nerr = check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, .false., q) if (nerr > 0) gfr%check_ok = .false. end if end subroutine gfr_fv_phys_to_dyn_hybrid @@ -731,7 +693,7 @@ subroutine gfr_dyn_to_fv_phys_dom_mt(par, dom_mt, nt, hvcoord, elem, & #endif end subroutine gfr_dyn_to_fv_phys_dom_mt - subroutine gfr_fv_phys_to_dyn_dom_mt(par, dom_mt, nt, dt, hvcoord, elem, T, uv, q) + subroutine gfr_fv_phys_to_dyn_dom_mt(par, dom_mt, nt, hvcoord, elem, T, uv, q) ! Wrapper to the hybrid-threading main routine. use parallel_mod, only: parallel_t @@ -742,7 +704,6 @@ subroutine gfr_fv_phys_to_dyn_dom_mt(par, dom_mt, nt, dt, hvcoord, elem, T, uv, type (parallel_t), intent(in) :: par type (domain1d_t), intent(in) :: dom_mt(:) integer, intent(in) :: nt - real(kind=real_kind), intent(in) :: dt type (hvcoord_t), intent(in) :: hvcoord type (element_t), intent(inout) :: elem(:) real(kind=real_kind), intent(inout) :: T(:,:,:), uv(:,:,:,:), q(:,:,:,:) @@ -755,7 +716,7 @@ subroutine gfr_fv_phys_to_dyn_dom_mt(par, dom_mt, nt, dt, hvcoord, elem, T, uv, !$omp parallel num_threads(hthreads), default(shared), private(nets,nete,hybrid) #endif call gfr_hybrid_create(par, dom_mt, hybrid, nets, nete) - call gfr_fv_phys_to_dyn_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete, T, uv, q) + call gfr_fv_phys_to_dyn_hybrid(hybrid, nt, hvcoord, elem, nets, nete, T, uv, q) #ifdef HORIZ_OPENMP !$omp end parallel #endif @@ -1818,18 +1779,16 @@ subroutine gfr_pg1_reconstruct_topo_hybrid(hybrid, elem, nets, nete) end if end subroutine gfr_pg1_reconstruct_topo_hybrid - subroutine gfr_pg1_reconstruct_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete) + subroutine gfr_pg1_reconstruct_hybrid(hybrid, nt, hvcoord, elem, nets, nete) ! pg1 reconstruction routine for tendencies and states. use element_ops, only: get_field use dimensions_mod, only: nlev, qsize use hybvcoord_mod, only: hvcoord_t use physical_constants, only: p0, kappa - use control_mod, only: ftype type (hybrid_t), intent(in) :: hybrid integer, intent(in) :: nt - real(kind=real_kind), intent(in) :: dt type (hvcoord_t), intent(in) :: hvcoord type (element_t), intent(inout) :: elem(:) integer, intent(in) :: nets, nete @@ -1837,12 +1796,9 @@ subroutine gfr_pg1_reconstruct_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete) real(kind=real_kind), dimension(np,np,nlev) :: dp, p, wr1 real(kind=real_kind) :: qmin, qmax integer :: ie, k, qi, nerr - logical :: q_adjustment if (gfr%nphys /= 1 .or. .not. gfr%boost_pg1) return - q_adjustment = ftype >= 1 .and. ftype <= 4 - do ie = nets,nete dp = elem(ie)%state%dp3d(:,:,:,nt) @@ -1875,11 +1831,6 @@ subroutine gfr_pg1_reconstruct_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete) gfr%qmax(:,qi,ie), dp, wr1, elem(ie)%derived%FQ(:,:,:,qi)) if (nerr > 0) gfr%check_ok = .false. end if - if (.not. q_adjustment) then - ! Convert to a tendency. - elem(ie)%derived%FQ(:,:,:,qi) = & - dp*(elem(ie)%derived%FQ(:,:,:,qi) - elem(ie)%state%Q(:,:,:,qi))/dt - end if end do end do @@ -1913,7 +1864,6 @@ subroutine gfr_pg1_g_reconstruct_scalar_dp(gfr, ie, gll_metdet, dp, g) real(kind=real_kind), intent(inout) :: g(:,:,:) real(kind=real_kind) wr(np*np) - integer :: nlev, k, edgeidx g = dp*g call gfr_pg1_g_reconstruct_scalar(gfr, ie, gll_metdet, g) @@ -1947,7 +1897,7 @@ subroutine gfr_pg1_g_reconstruct_vector(gfr, ie, elem, v) end do end subroutine gfr_pg1_g_reconstruct_vector - subroutine gfr_pg1_reconstruct_dom_mt(par, dom_mt, nt, dt, hvcoord, elem) + subroutine gfr_pg1_reconstruct_dom_mt(par, dom_mt, nt, hvcoord, elem) ! Wrapper to the hybrid-threading main routine. use parallel_mod, only: parallel_t @@ -1958,7 +1908,6 @@ subroutine gfr_pg1_reconstruct_dom_mt(par, dom_mt, nt, dt, hvcoord, elem) type (parallel_t), intent(in) :: par type (domain1d_t), intent(in) :: dom_mt(:) integer, intent(in) :: nt - real(kind=real_kind), intent(in) :: dt type (hvcoord_t), intent(in) :: hvcoord type (element_t), intent(inout) :: elem(:) @@ -1970,7 +1919,7 @@ subroutine gfr_pg1_reconstruct_dom_mt(par, dom_mt, nt, dt, hvcoord, elem) !$omp parallel num_threads(hthreads), default(shared), private(nets,nete,hybrid) #endif call gfr_hybrid_create(par, dom_mt, hybrid, nets, nete) - call gfr_pg1_reconstruct_hybrid(hybrid, nt, dt, hvcoord, elem, nets, nete) + call gfr_pg1_reconstruct_hybrid(hybrid, nt, hvcoord, elem, nets, nete) #ifdef HORIZ_OPENMP !$omp end parallel #endif @@ -2347,7 +2296,7 @@ subroutine set_ps_Q(elem, nets, nete, timeidx, qidx, nlev) end subroutine set_ps_Q function check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & - use_state_Q, q_adjustment, q_f) result(nerr) + use_state_Q, q_f) result(nerr) ! Compare global mass on dynamics and physics grids. @@ -2362,7 +2311,7 @@ function check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & type (hvcoord_t), intent(in) :: hvcoord type (element_t), intent(in) :: elem(:) integer, intent(in) :: nt, nets, nete - logical, intent(in) :: use_state_Q, q_adjustment + logical, intent(in) :: use_state_Q real (kind=real_kind), intent(in) :: q_f(:,:,:,:) integer :: nf, nf2, ie, k, qi, ic, nchunk, qi0, nq, qic, b1, b2, cnt, nerr @@ -2395,19 +2344,11 @@ function check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & else wg = elem(ie)%derived%FQ(:,:,k,qi) end if - if (q_adjustment) then - global_shared_buf(ie,b1) = global_shared_buf(ie,b1) + & - sum(elem(ie)%spheremp(:,:)*dp(:,:,k)*wg) - global_shared_buf(ie,b2) = global_shared_buf(ie,b2) + & - sum(gfr%fv_metdet(:nf2,ie)*gfr%w_ff(:nf2)* & - dp_fv(:nf2,k)*q_f(:nf2,k,qi,ie)) - else - global_shared_buf(ie,b1) = global_shared_buf(ie,b1) + & - sum(elem(ie)%spheremp(:,:)*wg) - global_shared_buf(ie,b2) = global_shared_buf(ie,b2) + & - sum(gfr%fv_metdet(:nf2,ie)*gfr%w_ff(:nf2)* & - q_f(:nf2,k,qi,ie)) - end if + global_shared_buf(ie,b1) = global_shared_buf(ie,b1) + & + sum(elem(ie)%spheremp(:,:)*dp(:,:,k)*wg) + global_shared_buf(ie,b2) = global_shared_buf(ie,b2) + & + sum(gfr%fv_metdet(:nf2,ie)*gfr%w_ff(:nf2)* & + dp_fv(:nf2,k)*q_f(:nf2,k,qi,ie)) end do end do end do @@ -2422,12 +2363,6 @@ function check_global_properties(gfr, hybrid, hvcoord, elem, nt, nets, nete, & end do end do tol = 10*eps - if (.not. q_adjustment) then - ! In the case of Q tendencies, cancellation can lead to arbitrarily large - ! errors in the tendencies. We have no control over that, so just loosen - ! the tolerance a little and likely warn more often. - tol = 1.e3_real_kind*tol - end if if (hybrid%masterthread) then do qi = 1,qsize if (abs(mass(2,qi) - mass(1,qi)) > tol*abs(mass(1,qi))) then diff --git a/components/homme/src/share/gllfvremap_util_mod.F90 b/components/homme/src/share/gllfvremap_util_mod.F90 index 7362dbf6174f..7392b9a3676e 100644 --- a/components/homme/src/share/gllfvremap_util_mod.F90 +++ b/components/homme/src/share/gllfvremap_util_mod.F90 @@ -151,7 +151,7 @@ subroutine set_gll_state(hvcoord, elem, nt1, nt2) do k = 1,nlev s1%p(:,:,k) = hvcoord%hyam(k)*hvcoord%ps0 + hvcoord%hybm(k)*s1%ps s1%dp(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & - (hvcoord%hybi(k+1) - hvcoord%hybi(k))*s1%ps + (hvcoord%hybi(k+1) - hvcoord%hybi(k))*s1%ps end do s1%z = zero s1%zi = zero @@ -236,13 +236,6 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) ! Set FV tendencies. if (tendency) then do ie = nets,nete - if (ftype == 0) then - do k = 1,nlev - wg(:nf,:nf,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & - (hvcoord%hybi(k+1) - hvcoord%hybi(k))* & - reshape(pg_data%ps(:,ie), (/nf,nf/)) - end do - end if do j = 1,nf do i = 1,nf col = nf*(j-1) + i @@ -252,13 +245,7 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) pg_data%T(col,:,ie) = f/dt ! no moisture adjustment => no dp3d adjustment pg_data%q(col,:,1,ie) = zero - if (ftype == 0) then - do q = 2,qsize - pg_data%q(col,:,q,ie) = f*wg(i,j,:)/dt - end do - else - pg_data%q(col,:,2:qsize,ie) = pg_data%q(col,:,2:qsize,ie) + f - end if + pg_data%q(col,:,2:qsize,ie) = pg_data%q(col,:,2:qsize,ie) + f end do end do end do @@ -266,17 +253,22 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) ! Test that if tendencies are 0, then the original fields are unchanged. pg_data%T = zero pg_data%uv = zero - if (ftype == 0) pg_data%q = zero end if ! FV -> GLL. - call gfr_fv_phys_to_dyn(hybrid, nt2, dt, hvcoord, elem, nets, nete, & + call gfr_fv_phys_to_dyn(hybrid, nt2, hvcoord, elem, nets, nete, & pg_data%T, pg_data%uv, pg_data%q) call gfr_f2g_dss(hybrid, elem, nets, nete) - call gfr_pg1_reconstruct(hybrid, nt2, dt, hvcoord, elem, nets, nete) + call gfr_pg1_reconstruct(hybrid, nt2, hvcoord, elem, nets, nete) ! Apply the tendencies. do ie = nets,nete + if (ftype == 0) then + do q = 1, qsize + elem(ie)%derived%FQ(:,:,:,q) = elem(ie)%state%dp3d(:,:,:,nt1)* & + (elem(ie)%derived%FQ(:,:,:,q) - elem(ie)%state%Q(:,:,:,q))/dt + end do + end if call applyCAMforcing_tracers(elem(ie), hvcoord, nt2, nt2, dt, logical(ftype /= 0)) end do call applyCAMforcing_dynamics(elem, hvcoord, nt2, dt, nets, nete) @@ -360,13 +352,13 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) end if end do call wrap_repro_sum(nvars=2, comm=hybrid%par%comm) + rd = sqrt(global_shared_sum(1)/global_shared_sum(2)) + msg = '' + if (.not. tendency .and. rd > 5*eps) then + nerr = nerr + 1 + msg = ' ERROR' + end if if (hybrid%masterthread) then - rd = sqrt(global_shared_sum(1)/global_shared_sum(2)) - msg = '' - if (.not. tendency .and. rd > 5*eps) then - nerr = nerr + 1 - msg = ' ERROR' - end if write(iulog, '(a,i3,a,i3,es12.4,a8)') 'gfrt> test1 q l2', q, ' of', qsize, rd, msg end if end do @@ -426,28 +418,18 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) ! original is Q. qmin1 = one; qmax1 = -one do ie = nets,nete - if (ftype == 0) then - do k = 1,nlev - wg(:nf,:nf,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & - (hvcoord%hybi(k+1) - hvcoord%hybi(k))*reshape(pg_data%ps(:,ie), (/nf,nf/)) - end do - end if pg_data%q(:nf2,:,:,ie) = two*pg_data%q(:nf2,:,:,ie) do q = 2,qsize qmin1(q) = min(qmin1(q), minval(elem(ie)%state%Q(:,:,1,q))) qmax1(q) = max(qmax1(q), maxval(elem(ie)%state%Q(:,:,1,q))) qmin1(q) = min(qmin1(q), minval(pg_data%q(:nf2,1,q,ie))) qmax1(q) = max(qmax1(q), maxval(pg_data%q(:nf2,1,q,ie))) - if (ftype == 0) then - pg_data%q(:nf2,:,q,ie) = & - half*reshape(wg(:nf,:nf,:), (/nf2,nlev/))*pg_data%q(:nf2,:,q,ie)/dt - end if end do end do - call gfr_fv_phys_to_dyn(hybrid, nt2, dt, hvcoord, elem, nets, nete, & + call gfr_fv_phys_to_dyn(hybrid, nt2, hvcoord, elem, nets, nete, & pg_data%T, pg_data%uv, pg_data%q) call gfr_f2g_dss(hybrid, elem, nets, nete) - call gfr_pg1_reconstruct(hybrid, nt2, dt, hvcoord, elem, nets, nete) + call gfr_pg1_reconstruct(hybrid, nt2, hvcoord, elem, nets, nete) ! Don't apply forcings; rather, the forcing fields now have the ! remapped quantities we want to compare against the original. do q = 2, qsize+3 @@ -485,11 +467,6 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) global_shared_buf(ie,3) = sum(wg(:,:,1)*elem(ie)%state%dp3d(:,:,1,nt1)*wg1(:,:,1)) end if else - if (ftype == 0) then - elem(ie)%derived%FQ(:,:,:,q) = elem(ie)%state%Q(:,:,:,q) + & - dt*elem(ie)%derived%FQ(:,:,:,q)/ & - elem(ie)%state%dp3d(:,:,:,nt1) - end if ! Extrema in level 1. qmin2 = min(qmin2, minval(elem(ie)%derived%FQ(:,:,1,q))) qmax2 = max(qmax2, maxval(elem(ie)%derived%FQ(:,:,1,q))) @@ -512,21 +489,24 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) qmax1(q) = ParallelMax(qmax1(q), hybrid) qmin2 = ParallelMin(qmin2, hybrid) qmax2 = ParallelMax(qmax2, hybrid) - if (hybrid%masterthread) then - rd = sqrt(global_shared_sum(1)/global_shared_sum(2)) - write(iulog, '(a,i3,a,i3,es12.4)') 'gfrt> test3 q l2', q, ' of', qsize, rd - b = max(abs(qmin1(q)), abs(qmax1(q))) - if (q <= qsize .and. qmin2 < qmin1(q) - 5*eps*b .or. & - qmax2 > qmax1(q) + 5*eps*b) then - nerr = nerr + 1 + rd = sqrt(global_shared_sum(1)/global_shared_sum(2)) + if (hybrid%masterthread) & + write(iulog, '(a,i3,a,i3,es12.4)') 'gfrt> test3 q l2', q, ' of', qsize, rd + b = max(abs(qmin1(q)), abs(qmax1(q))) + if (q <= qsize .and. qmin2 < qmin1(q) - 5*eps*b .or. & + qmax2 > qmax1(q) + 5*eps*b) then + nerr = nerr + 1 + if (hybrid%masterthread) then write(iulog, '(a,i3,es12.4,es12.4,es12.4,es12.4,a)') 'gfrt> test3 q extrema', & q, qmin1(q), qmin2-qmin1(q), qmax2-qmax1(q), qmax1(q), ' ERROR' end if - if (domass) then - a = global_shared_sum(3) - b = global_shared_sum(4) - if (abs(b - a) > 5*eps*abs(a)) then - nerr = nerr + 1 + end if + if (domass) then + a = global_shared_sum(3) + b = global_shared_sum(4) + if (abs(b - a) > 5*eps*abs(a)) then + nerr = nerr + 1 + if (hybrid%masterthread) then write(iulog, '(a,i3,es12.4,es12.4,es12.4,a)') 'gfrt> test3 q mass', & q, a, b, abs(b - a)/abs(a), ' ERROR' end if @@ -536,9 +516,10 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) end function run function gfr_check_api(hybrid, nets, nete, hvcoord, elem) result(nerr) - ! Drive run. Check nphys 1 through 4, ftypes 1 and 2, and pg1 with + ! Drive run. Check nphys 1 through 4, ftypes 0 and 2, and pg1 with ! and without the OOA boost. + use kinds, only: iulog use hybvcoord_mod, only: hvcoord_t use control_mod, only: ftype use gllfvremap_mod @@ -573,6 +554,8 @@ function gfr_check_api(hybrid, nets, nete, hvcoord, elem) result(nerr) end if !$omp barrier + if (hybrid%masterthread) write(iulog, '(a,i2)') 'gfrt> ftype', ftype + nerr = nerr + run(hybrid, hvcoord, elem, nets, nete, nphys, .false.) nerr = nerr + run(hybrid, hvcoord, elem, nets, nete, nphys, .true.) diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index 73c32284d470..baf71a0910c3 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -619,6 +619,7 @@ subroutine dcmip2016_test1_pg_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl use element_ops, only: get_field use gllfvremap_mod use perf_mod, only: t_startf, t_stopf + use control_mod, only: ftype ! to DSS precl use edge_mod, only: edgevpack_nlyr, edgevunpack_nlyr, edge_g @@ -777,30 +778,44 @@ subroutine dcmip2016_test1_pg_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl call gfr_g_make_nonnegative(elem(ie)%metdet, wrk3(:,:,:1)) precl(:,:,ie) = wrk3(:,:,1) + ! T, uv tendencies pg_data%T(:,:,ie) = reshape((T_fv(:nf,:nf,:) - T0(:nf,:nf,:))/dt, (/ncol,nlev/)) pg_data%uv(:,1,:,ie) = reshape((u_fv(:nf,:nf,:) - u0(:nf,:nf,:))/dt, (/ncol,nlev/)) pg_data%uv(:,2,:,ie) = reshape((v_fv(:nf,:nf,:) - v0(:nf,:nf,:))/dt, (/ncol,nlev/)) - ! ftype = 0, so q is Qdp tendency. + ! q state do i = 1,qsize - pg_data%q(:,:,i,ie) = reshape(dp_fv(:nf,:nf,:)*(Q_fv(:nf,:nf,:,i) - Q0_fv(:nf,:nf,:,i))/dt, & - (/ncol,nlev/)) + pg_data%q(:,:,i,ie) = reshape(Q_fv(:nf,:nf,:,i), (/ncol,nlev/)) end do ! Measure max w and max prect. w is not used in the physics, so ! just look at the GLL values. - max_w = max( max_w , maxval(w ) ) + max_w = max(max_w, maxval(w)) ! ps isn't updated by the physics, so just look at the GLL values. - min_ps = min( min_ps, minval(elem(ie)%state%ps_v(:,:,nt)) ) + min_ps = min(min_ps, minval(elem(ie)%state%ps_v(:,:,nt))) enddo call t_startf('gfr_fv_phys_to_dyn') - call gfr_fv_phys_to_dyn(hybrid, nt, dt, hvcoord, elem, nets, nete, & + call gfr_fv_phys_to_dyn(hybrid, nt, hvcoord, elem, nets, nete, & pg_data%T, pg_data%uv, pg_data%q) call t_stopf('gfr_fv_phys_to_dyn') ! dp_coupling doesn't do the DSS; stepon does. Thus, this DCMIP test ! also needs to do its own DSS. call gfr_f2g_dss(hybrid, elem, nets, nete) - call gfr_pg1_reconstruct(hybrid, nt, dt, hvcoord, elem, nets, nete) + call gfr_pg1_reconstruct(hybrid, nt, hvcoord, elem, nets, nete) + + if (ftype == 0) then + ! Convert FQ from state to Qdp tendency. + do ie = nets,nete + do k = 1,nlev + dp(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & + (hvcoord%hybi(k+1) - hvcoord%hybi(k))*elem(ie)%state%ps_v(:,:,nt) + end do + do i = 1,qsize + elem(ie)%derived%FQ(:,:,:,i) = & + dp*(elem(ie)%derived%FQ(:,:,:,i) - elem(ie)%state%Q(:,:,:,i))/dt + end do + end do + end if call toy_init(rcd) do ie = nets,nete @@ -808,11 +823,12 @@ subroutine dcmip2016_test1_pg_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl dp(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & (hvcoord%hybi(k+1) - hvcoord%hybi(k))*elem(ie)%state%ps_v(:,:,nt) end do + ! Since ftype is 0, FQ is the tendency. Convert to state. do i = 1,2 wrk4(:,:,:,i) = elem(ie)%state%Q(:,:,:,i+3) + dt*elem(ie)%derived%FQ(:,:,:,i+3)/dp end do call toy_rcd(wrk4, rcd) - end do + end do call toy_print(hybrid, tl%nstep, rcd) ! DSS precl diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_interface.F90 index 889190973e46..2f55f50a2496 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_interface.F90 @@ -9,29 +9,28 @@ module physgrid_interface subroutine init_gllfvremap_f90(ne, hyai, hybi, hyam, hybm, ps0, dvv, mp, qsize_in, & is_sphere) bind(c) + use iso_c_binding, only: c_double use hybvcoord_mod, only: set_layer_locations use thetal_test_interface, only: init_f90 use theta_f2c_mod, only: init_elements_c use edge_mod_base, only: initEdgeBuffer, edge_g, initEdgeSBuffer use prim_advection_base, only: edgeAdvQminmax - use geometry_interface_mod, only: GridVertex use bndry_mod, only: sort_neighbor_buffer_mapping use reduction_mod, only: initreductionbuffer, red_sum, red_min, red_max use parallel_mod, only: global_shared_buf, nrepro_vars - use control_mod, only: ftype, use_moisture, theta_hydrostatic_mode + use control_mod, only: use_moisture, theta_hydrostatic_mode - real (real_kind), intent(in) :: hyai(nlevp), hybi(nlevp), hyam(nlev), hybm(nlev) + real (c_double), intent(in) :: hyai(nlevp), hybi(nlevp), hyam(nlev), hybm(nlev) integer (c_int), value, intent(in) :: ne, qsize_in - real (real_kind), value, intent(in) :: ps0 - real (real_kind), intent(out) :: dvv(np,np), mp(np,np) + real (c_double), value, intent(in) :: ps0 + real (c_double), intent(out) :: dvv(np,np), mp(np,np) logical (c_bool), value, intent(in) :: is_sphere - integer :: ie, edgesz + integer :: edgesz if (.not. is_sphere) print *, "NOT IMPL'ED YET" qsize = qsize_in - ftype = 2 use_moisture = .true. theta_hydrostatic_mode = .false. @@ -56,49 +55,50 @@ subroutine run_gfr_test(nerr) bind(c) integer (c_int), intent(out) :: nerr - integer :: ithr type (hybrid_t) :: hybrid type (domain1d_t) :: dom_mt(0:0) dom_mt(0)%start = 1 dom_mt(0)%end = nelemd hybrid = hybrid_create(par, 0, 1) - + nerr = gfr_test(hybrid, dom_mt, hvcoord, deriv, elem) end subroutine run_gfr_test - subroutine gfr_init_f90(nf, ftype_in, thm) bind(c) + subroutine gfr_init_f90(nf, thm) bind(c) use gllfvremap_mod, only: gfr_init - use control_mod, only: ftype, theta_hydrostatic_mode + use control_mod, only: theta_hydrostatic_mode - integer (c_int), value, intent(in) :: nf, ftype_in + integer (c_int), value, intent(in) :: nf logical (c_bool), value, intent(in) :: thm - ftype = ftype_in theta_hydrostatic_mode = thm call gfr_init(par, elem, nf, 2, .false.) end subroutine gfr_init_f90 - subroutine gfr_finish_f90(nf) bind(c) + subroutine gfr_finish_f90() bind(c) use gllfvremap_mod, only: gfr_finish - integer (c_int), value, intent(in) :: nf - call gfr_finish() end subroutine gfr_finish_f90 - subroutine run_gfr_check_api(nerr) bind(c) + subroutine run_gfr_check_api(thm, nerr) bind(c) use thetal_test_interface, only: hvcoord use hybrid_mod, only: hybrid_t, hybrid_create + use control_mod, only: theta_hydrostatic_mode use gllfvremap_util_mod, only: gfr_check_api + logical (c_bool), value, intent(in) :: thm integer (c_int), intent(out) :: nerr - integer :: ithr type (hybrid_t) :: hybrid - - hybrid = hybrid_create(par, 0, 1) + logical :: thm_save + + hybrid = hybrid_create(par, 0, 1) + thm_save = theta_hydrostatic_mode + theta_hydrostatic_mode = thm nerr = gfr_check_api(hybrid, 1, nelemd, hvcoord, elem) + theta_hydrostatic_mode = thm_save end subroutine run_gfr_check_api subroutine limiter1_clip_and_sum_f90(n, spheremp, qmin, qmax, dp, q) bind(c) @@ -188,13 +188,12 @@ subroutine gfr_dyn_to_fv_phys_f90(nf, nt, ps, phis, T, uv, omega_p, q) bind(c) ps, phis, T, uv, omega_p, q) end subroutine gfr_dyn_to_fv_phys_f90 - subroutine gfr_fv_phys_to_dyn_f90(nf, nt, dt, T, uv, q) bind(c) + subroutine gfr_fv_phys_to_dyn_f90(nf, nt, T, uv, q) bind(c) use thetal_test_interface, only: hvcoord use hybrid_mod, only: hybrid_t, hybrid_create use gllfvremap_mod, only: gfr_fv_phys_to_dyn, gfr_get_nphys, gfr_f2g_dss integer (c_int), value, intent(in) :: nf, nt - real (c_double), value, intent(in) :: dt real (c_double), intent(in) :: T(nf*nf,nlev,nelemd), uv(nf*nf,2,nlev,nelemd), & q(nf*nf,nlev,qsize,nelemd) @@ -203,7 +202,7 @@ subroutine gfr_fv_phys_to_dyn_f90(nf, nt, dt, T, uv, q) bind(c) if (nf /= gfr_get_nphys()) print *, 'ERROR: nf vs gfr%nphys:', nf, gfr_get_nphys() hybrid = hybrid_create(par, 0, 1) - call gfr_fv_phys_to_dyn(hybrid, nt, dt, hvcoord, elem, 1, nelemd, T, uv, q) + call gfr_fv_phys_to_dyn(hybrid, nt, hvcoord, elem, 1, nelemd, T, uv, q) call gfr_f2g_dss(hybrid, elem, 1, nelemd) end subroutine gfr_fv_phys_to_dyn_f90 diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index d3284a3ef902..7918b17e3464 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -40,9 +40,9 @@ extern "C" { void init_geometry_f90(); void run_gfr_test(int* nerr); - void run_gfr_check_api(int* nerr); + void run_gfr_check_api(bool theta_hydrostatic_mode, int* nerr); - void gfr_init_f90(int nf, int ftype, bool theta_hydrostatic_mode); + void gfr_init_f90(int nf, bool theta_hydrostatic_mode); void gfr_init_hxx(); void gfr_finish_f90(); @@ -51,7 +51,7 @@ extern "C" { void gfr_dyn_to_fv_phys_f90(int nf, int nt, Real* ps, Real* phis, Real* T, Real* uv, Real* omega_p, Real* q); - void gfr_fv_phys_to_dyn_f90(int nf, int nt, Real dt, Real* T, Real* uv, Real* q); + void gfr_fv_phys_to_dyn_f90(int nf, int nt, Real* T, Real* uv, Real* q); void cmp_dyn_data_f90(int nlev_align, int nq, Real* ft, Real* fm, Real* q, Real* fq, int* nerr); } // extern "C" @@ -73,6 +73,48 @@ decltype(Kokkos::create_mirror_view(V())) cmvdc (const V& v) { return h; } +// Make a,b a little better behaved so levels don't get too thin. +static void clean (HybridVCoord& h) { + static const int n = NUM_INTERFACE_LEV, nh = (n+1)/2, nh0 = n-nh; + const auto ai = cmvdc(h.hybrid_ai); + const auto bi = cmvdc(h.hybrid_bi); + const auto amp = cmvdc(h.hybrid_am); + const auto bmp = cmvdc(h.hybrid_bm); + const CA1d am(reinterpret_cast(amp.data()), n-1); + const CA1d bm(reinterpret_cast(bmp.data()), n-1); + + for (int i = 0; i < n; ++i) bi(i) = 0; + for (int i = 0; i < nh; ++i) { + assert(nh0+i < n); + const Real a = Real(i)/(nh-1); + bi(nh0+i) = (1-a)*0.02 + a*1; + } + assert(b(n-1) == 1); + + Real etai[n]; + for (int i = 0; i < n; ++i) { + const Real a = Real(i)/(n-1); + etai[i] = (1-a)*0.0001 + a*1; + } + + for (int i = 0; i < n; ++i) { + ai(i) = etai[i] - bi(i); + assert(ai(i) >= 0); + } + + for (int i = 0; i < n-1; ++i) am(i) = (ai(i) + ai(i+1))/2; + for (int i = 0; i < n-1; ++i) bm(i) = (bi(i) + bi(i+1))/2; + + deep_copy(h.hybrid_ai, ai); + deep_copy(h.hybrid_bi, bi); + deep_copy(h.hybrid_am, amp); + deep_copy(h.hybrid_bm, bmp); + + h.hybrid_ai0 = ai(0); + h.compute_deltas(); + h.compute_eta(); +} + class Random { using rngalg = std::mt19937_64; using rpdf = std::uniform_real_distribution; @@ -134,7 +176,9 @@ struct Session { auto& c = Context::singleton(); c.create().random_init(seed); - h = c.get(); + auto& h_ref = c.get(); + clean(h_ref); + h = h_ref; auto& p = c.create(); p.qsize = qsize; @@ -580,9 +624,9 @@ static void init_dyn_data (Session& s) { using g = GllFvRemapImpl; // randomize C++ data + auto& c = Context::singleton(); const auto hai = cmvdc(s.h.hybrid_ai); const auto bai = cmvdc(s.h.hybrid_bi); - auto& c = Context::singleton(); const auto state = c.get(); const auto derived = c.get(); const auto tracers = c.get(); @@ -732,13 +776,13 @@ static void test_get_temperature (Session& s) { } } -static void test_dyn_to_fv_phys (Session& s, const int nf, const int ftype, - const bool theta_hydrostatic_mode) { +static void +test_dyn_to_fv_phys (Session& s, const int nf, const bool theta_hydrostatic_mode) { using g = GllFvRemapImpl; const int nf2 = nf*nf; - gfr_init_f90(nf, ftype, theta_hydrostatic_mode); + gfr_init_f90(nf, theta_hydrostatic_mode); gfr_init_hxx(); init_dyn_data(s); @@ -787,15 +831,14 @@ static void test_dyn_to_fv_phys (Session& s, const int nf, const int ftype, gfr_finish_f90(); } -static void test_fv_phys_to_dyn (Session& s, const int nf, const int ftype, - const bool theta_hydrostatic_mode) { +static void +test_fv_phys_to_dyn (Session& s, const int nf, const bool theta_hydrostatic_mode) { using Kokkos::deep_copy; using g = GllFvRemapImpl; - const Real dt = 1800; const int nf2 = nf*nf; - gfr_init_f90(nf, ftype, theta_hydrostatic_mode); + gfr_init_f90(nf, theta_hydrostatic_mode); gfr_init_hxx(); { @@ -829,8 +872,8 @@ static void test_fv_phys_to_dyn (Session& s, const int nf, const int ftype, auto& gfr = c.get(); const int nt = 1; - gfr_fv_phys_to_dyn_f90(nf, nt+1, dt, fT.data(), fuv.data(), ffq.data()); - gfr.run_fv_phys_to_dyn(nt, dt, dT, duv, dfq); + gfr_fv_phys_to_dyn_f90(nf, nt+1, fT.data(), fuv.data(), ffq.data()); + gfr.run_fv_phys_to_dyn(nt, dT, duv, dfq); gfr.run_fv_phys_to_dyn_dss(); } @@ -879,26 +922,31 @@ TEST_CASE ("gllfvremap_testing") { } // Existing F90 gllfvremap unit tests. - int nerr; - run_gfr_test(&nerr); - REQUIRE(nerr == 0); - run_gfr_test(&nerr); - REQUIRE(nerr == 0); + { + int nerr; + run_gfr_test(&nerr); + REQUIRE(nerr == 0); + } + { + int nerr; + const bool thm = true; + Context::singleton().get().theta_hydrostatic_mode = thm; + init_dyn_data(s); + run_gfr_check_api(thm, &nerr); + REQUIRE(nerr == 0); + } // Main remap routines. for (const bool theta_hydrostatic_mode : {false, true}) { - auto& c = Context::singleton(); - for (const int nf : {2,3,4}) - for (const int ftype : {2}) { // dyn_to_fv_phys is independent of ftype - printf("ut> g2f nf %d ftype %d thm %d\n", nf, ftype, (int) theta_hydrostatic_mode); - test_dyn_to_fv_phys(s, nf, ftype, theta_hydrostatic_mode); - } + for (const int nf : {2,3,4}) { + printf("ut> g2f nf %d thm %d\n", nf, (int) theta_hydrostatic_mode); + test_dyn_to_fv_phys(s, nf, theta_hydrostatic_mode); + } - for (const int nf : {2,3,4}) - for (const int ftype : {2,0}) { - printf("ut> f2g nf %d ftype %d thm %d\n", nf, ftype, (int) theta_hydrostatic_mode); - test_fv_phys_to_dyn(s, nf, ftype, theta_hydrostatic_mode); - } + for (const int nf : {2,3,4}) { + printf("ut> f2g nf %d thm %d\n", nf, (int) theta_hydrostatic_mode); + test_fv_phys_to_dyn(s, nf, theta_hydrostatic_mode); + } } } catch (...) {} Session::delete_singleton(); From 6230e841c254ac4c9b1bf4d409337c45ca9a1c2d Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 2 Nov 2021 02:11:45 -0500 Subject: [PATCH 104/301] EAM: Adjust gllfvremap_mod call. --- components/eam/src/dynamics/se/dp_coupling.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/dynamics/se/dp_coupling.F90 b/components/eam/src/dynamics/se/dp_coupling.F90 index 172f61d67c66..b937579a87db 100644 --- a/components/eam/src/dynamics/se/dp_coupling.F90 +++ b/components/eam/src/dynamics/se/dp_coupling.F90 @@ -472,7 +472,7 @@ subroutine p_d_coupling(phys_state, phys_tend, dyn_in) call t_startf('fv_phys_to_dyn') ! Map FV physics state to dynamics grid dtime = get_step_size() - call gfr_fv_phys_to_dyn(par, dom_mt, TimeLevel%n0, dtime, hvcoord, elem, T_tmp, & + call gfr_fv_phys_to_dyn(par, dom_mt, TimeLevel%n0, hvcoord, elem, T_tmp, & uv_tmp, q_tmp) call t_stopf('fv_phys_to_dyn') From 4a392ab3bf46bc621cd1741df251c70445827605 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 2 Nov 2021 18:21:25 -0500 Subject: [PATCH 105/301] CIME: Add two new pg2 tests for ftype 0. These share a build with preexisting tests, so the only additional cost is running these short ne4pg2 tests. Also remove the shell_commands file for the current test in that group since theta-l is the default dycore. --- cime_config/tests.py | 4 +++- .../testdefs/testmods_dirs/eam/thetahy_sl_pg2/shell_commands | 2 -- .../testmods_dirs/eam/thetahy_sl_pg2_ftype0/user_nl_eam | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2/shell_commands create mode 100644 components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2_ftype0/user_nl_eam diff --git a/cime_config/tests.py b/cime_config/tests.py index 821efafdd2c9..bddb693efacf 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -298,7 +298,9 @@ "tests" : ( "SMS_Ln5.ne4pg2_oQU480.F2010.eam-thetahy_pg2", "SMS_Ln5.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2", - "ERS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2" + "ERS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2", + "SMS_Ln5.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_ftype0", + "ERS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_ftype0", ) }, "e3sm_bench_hires_g" : { diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2/shell_commands deleted file mode 100644 index 26e9dbaa2fd6..000000000000 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2/shell_commands +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -./xmlchange CAM_TARGET=theta-l diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2_ftype0/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2_ftype0/user_nl_eam new file mode 100644 index 000000000000..420a46229a16 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/thetahy_sl_pg2_ftype0/user_nl_eam @@ -0,0 +1,2 @@ +semi_lagrange_cdr_check = .true. +se_ftype = 0 From c1c8d92c50d97e17a426468a13aa2b2465600f68 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 2 Nov 2021 19:18:28 -0600 Subject: [PATCH 106/301] HOMMEXX: Handle bfb_pow issue in gllfvremap API unit test. Also clean up some -Wall -pedantic warnings. --- .../homme/src/share/gllfvremap_util_mod.F90 | 32 +++++++++++-------- .../thetal_kokkos_ut/gllfvremap_ut.cpp | 2 +- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/components/homme/src/share/gllfvremap_util_mod.F90 b/components/homme/src/share/gllfvremap_util_mod.F90 index 7392b9a3676e..77e963007b75 100644 --- a/components/homme/src/share/gllfvremap_util_mod.F90 +++ b/components/homme/src/share/gllfvremap_util_mod.F90 @@ -79,15 +79,13 @@ subroutine finish() deallocate(pg_data%ps, pg_data%zs, pg_data%T, pg_data%uv, pg_data%omega_p, pg_data%q) end subroutine finish - subroutine set_state(s, hvcoord, nt1, nt2, ntq, elem) + subroutine set_state(s, nt1, nt2, ntq, elem) ! Convenience wrapper to set_elem_state. use physical_constants, only: g use element_ops, only: set_elem_state - use hybvcoord_mod, only: hvcoord_t type (State_t), intent(in) :: s - type (hvcoord_t), intent(in) :: hvcoord integer, intent(in) :: nt1, nt2, ntq type (element_t), intent(inout) :: elem @@ -116,7 +114,7 @@ subroutine set_gll_state(hvcoord, elem, nt1, nt2) type (State_t) :: s1 type (cartesian3D_t) :: p real(kind=real_kind) :: wr(np,np,nlev,2) - integer :: i, j, k, q, d, tl + integer :: i, j, k, q, tl elem%state%Q(:,:,:,1) = zero ! moisture tracer is 0 do j = 1,np @@ -156,12 +154,12 @@ subroutine set_gll_state(hvcoord, elem, nt1, nt2) s1%z = zero s1%zi = zero ! a bit of a kludge - call set_state(s1, hvcoord, nt1, nt2, nt1, elem) + call set_state(s1, nt1, nt2, nt1, elem) call get_field(elem, 'rho', wr(:,:,:,1), hvcoord, nt1, nt1) s1%w = -elem%derived%omega_p/(wr(:,:,:,1)*g) s1%wi(:,:,:nlev) = s1%w s1%wi(:,:,nlevp) = s1%w(:,:,nlev) - call set_state(s1, hvcoord, nt1, nt2, nt1, elem) + call set_state(s1, nt1, nt2, nt1, elem) do q = 1,qsize do tl = nt1,nt2 elem%state%Qdp(:,:,:,q,tl) = & @@ -198,11 +196,11 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) character(32) :: msg type (cartesian3D_t) :: p - real(kind=real_kind) :: wg(np,np,nlev), tend(np,np,nlev), f, a, b, c, rd, & + real(kind=real_kind) :: wg(np,np,nlev), tend(np,np,nlev), f, a, b, rd, & qmin1(qsize+3), qmax1(qsize+3), qmin2, qmax2, mass1, mass2, & wg1(np,np,nlev), wg2(np,np,nlev), dt, pressure(np,np,nlev), & p_fv(np*np,nlev), wf1(np*np,nlev), wf2(np*np,nlev), wf3(np*np) - integer :: nf, nf2, nt1, nt2, ie, i, j, k, d, q, qi, tl, col, nerr + integer :: nf, nf2, nt1, nt2, ie, i, j, k, q, qi, col, nerr logical :: domass nerr = 0 @@ -458,12 +456,13 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) sum(wg*elem(ie)%state%v(:,:,qi,:,nt1)**2) else call get_temperature(elem(ie), wg1, hvcoord, nt1) + call get_field(elem(ie), 'p', wg2, hvcoord, nt1, -1) global_shared_buf(ie,1) = sum(wg*(elem(ie)%derived%FT - wg1)**2) - global_shared_buf(ie,2) = sum(wg*wg1**2) - wg1 = wg1*(p0/elem(ie)%state%dp3d(:,:,:,nt1))**kappa + global_shared_buf(ie,2) = sum(wg*wg1**2) + wg1 = wg1*(p0/wg2)**kappa global_shared_buf(ie,4) = sum(wg(:,:,1)*elem(ie)%state%dp3d(:,:,1,nt1)*wg1(:,:,1)) wg1 = elem(ie)%derived%FT - wg1 = wg1*(p0/elem(ie)%state%dp3d(:,:,:,nt1))**kappa + wg1 = wg1*(p0/wg2)**kappa global_shared_buf(ie,3) = sum(wg(:,:,1)*elem(ie)%state%dp3d(:,:,1,nt1)*wg1(:,:,1)) end if else @@ -504,7 +503,12 @@ function run(hybrid, hvcoord, elem, nets, nete, nphys, tendency) result(nerr) if (domass) then a = global_shared_sum(3) b = global_shared_sum(4) - if (abs(b - a) > 5*eps*abs(a)) then + f = 5 +#ifdef HOMMEXX_BFB_TESTING + ! Errors in bfb_pow mean mass conservation holds to only ~1e-6. + if (q == qsize+3) f = 1.0e11_real_kind +#endif + if (abs(b - a) > f*eps*abs(a)) then nerr = nerr + 1 if (hybrid%masterthread) then write(iulog, '(a,i3,es12.4,es12.4,es12.4,a)') 'gfrt> test3 q mass', & @@ -592,7 +596,7 @@ subroutine gfr_convert_topo(par, elem, nphys, intopofn, outtopoprefix) #ifndef CAM real(real_kind), allocatable :: gll_fields(:,:,:,:), pg_fields(:,:,:), latlon(:,:,:) - integer :: unit, nf2, vari, phisidx, ie, i, j, k + integer :: nf2, vari, ie, i, j, k logical :: square, augment character(len=varname_len) :: fieldnames(5) @@ -643,7 +647,7 @@ end subroutine gfr_convert_topo function gfr_pgn_to_smoothed_topo(par, elem, output_nphys, intopofn, outtopoprefix) result(stat) #ifndef CAM - use common_io_mod, only: varname_len,infilenames + use common_io_mod, only: varname_len use gllfvremap_mod, only: gfr_init, gfr_finish, gfr_fv_phys_to_dyn_topo, & gfr_dyn_to_fv_phys_topo, gfr_f_get_latlon use interpolate_driver_mod, only: read_physgrid_topo_file, write_physgrid_smoothed_phis_file, & diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 7918b17e3464..670442e7c09c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -89,7 +89,7 @@ static void clean (HybridVCoord& h) { const Real a = Real(i)/(nh-1); bi(nh0+i) = (1-a)*0.02 + a*1; } - assert(b(n-1) == 1); + assert(bi(n-1) == 1); Real etai[n]; for (int i = 0; i < n; ++i) { From 8fead6eccab8b8ef186cbb28056596cbc69603a8 Mon Sep 17 00:00:00 2001 From: "Andrew M. Bradley" Date: Tue, 2 Nov 2021 21:42:31 -0500 Subject: [PATCH 107/301] Homme: Simplify dcmip1 pg2 toy-chem diagnostic calculation. --- .../homme/src/test_src/dcmip16_wrapper.F90 | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index baf71a0910c3..719710e0bb27 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -803,6 +803,15 @@ subroutine dcmip2016_test1_pg_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl call gfr_f2g_dss(hybrid, elem, nets, nete) call gfr_pg1_reconstruct(hybrid, nt, hvcoord, elem, nets, nete) + call toy_init(rcd) + do ie = nets,nete + do i = 1,2 + wrk4(:,:,:,i) = elem(ie)%state%Q(:,:,:,i+3) + end do + call toy_rcd(wrk4, rcd) + end do + call toy_print(hybrid, tl%nstep, rcd) + if (ftype == 0) then ! Convert FQ from state to Qdp tendency. do ie = nets,nete @@ -817,20 +826,6 @@ subroutine dcmip2016_test1_pg_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl end do end if - call toy_init(rcd) - do ie = nets,nete - do k = 1,nlev - dp(:,:,k) = (hvcoord%hyai(k+1) - hvcoord%hyai(k))*hvcoord%ps0 + & - (hvcoord%hybi(k+1) - hvcoord%hybi(k))*elem(ie)%state%ps_v(:,:,nt) - end do - ! Since ftype is 0, FQ is the tendency. Convert to state. - do i = 1,2 - wrk4(:,:,:,i) = elem(ie)%state%Q(:,:,:,i+3) + dt*elem(ie)%derived%FQ(:,:,:,i+3)/dp - end do - call toy_rcd(wrk4, rcd) - end do - call toy_print(hybrid, tl%nstep, rcd) - ! DSS precl do ie = nets,nete precl(:,:,ie) = precl(:,:,ie)*elem(ie)%spheremp From 5c2ab48e786dfb93691afdd575e428e80ccde022 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 3 Nov 2021 12:35:03 -0500 Subject: [PATCH 108/301] add ftype2 test --- .../homme/test/reg_test/namelists/thetahs3.nl | 79 +++++++++++++++++++ .../test/reg_test/run_tests/test-list.cmake | 1 + .../run_tests/theta-fhs3-kokkos.cmake | 18 +++++ .../test/reg_test/run_tests/theta-fhs3.cmake | 18 +++++ 4 files changed, 116 insertions(+) create mode 100644 components/homme/test/reg_test/namelists/thetahs3.nl create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs3-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fhs3.cmake diff --git a/components/homme/test/reg_test/namelists/thetahs3.nl b/components/homme/test/reg_test/namelists/thetahs3.nl new file mode 100644 index 000000000000..1233cd1ce1a0 --- /dev/null +++ b/components/homme/test/reg_test/namelists/thetahs3.nl @@ -0,0 +1,79 @@ +&ctl_nl +vthreads = 1 +NThreads = 1 +partmethod = 4 +topology = "cube" +test_case = "held_suarez0" +u_perturb = 1 +rotate_grid = 0 +ne = ${HOMME_TEST_NE} +qsize = 1 +ndays = ${HOMME_TEST_NDAYS} +statefreq = 9999 +restartfreq = 43200 +restartfile = "./R0001" +runtype = 0 +mesh_file = '/dev/null' +tstep = 100 +dt_remap_factor = 3 +dt_tracer_factor = 6 +transport_alg = 12 +integration = "explicit" +smooth = 0 +nu = 3.4e-8 +nu_div = 3.4e-8 ! test this too? +nu_p = -1 +nu_q = -1 +nu_s = -1 +nu_top = 0 !2.5e5 +se_ftype = 2 +limiter_option = 9 +vert_remap_q_alg = 10 +hypervis_scaling = 3.0 +hypervis_order = 2 +hypervis_subcycle = 5 +hypervis_subcycle_q = 6 +hypervis_subcycle_tom = 6 +theta_hydrostatic_mode = false +theta_advect_form = 1 +tstep_type = 10 +moisture = 'notdry' +/ +&solver_nl +precon_method = "identity" +maxits = 500 +tol = 1.e-9 +/ +&filter_nl +filter_type = "taylor" +transfer_type = "bv" +filter_freq = 0 +filter_mu = 0.04D0 +p_bv = 12.0D0 +s_bv = .666666666666666666D0 +wght_fm = 0.10D0 +kcut_fm = 2 +/ +&vert_nl +vform = "ccm" +vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' +vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' +/ + +&prof_inparm +profile_outpe_num = 100 +profile_single_file = .true. +/ + +! timunits: 0= steps, 1=days, 2=hours +&analysis_nl + interp_gridtype = 2 + output_timeunits = 1,1 + output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} + output_start_time = 0,0 + output_end_time = 30000,30000 + output_varnames1 = 'ps','zeta','u','v','T' + output_varnames2 = 'Q','Q2','Q3','Q4' + io_stride = 8 + output_type = 'netcdf' +/ diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index b847937f1551..884e06c24e11 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -95,6 +95,7 @@ IF (BUILD_HOMME_THETA_KOKKOS) theta-f1-tt10-hvs1-hvst0-r2-qz10-nutopoff-GB-sl theta-fhs1 theta-fhs2 + theta-fhs3 ) #all tests that will be used for cxx-vs-F bfb testing diff --git a/components/homme/test/reg_test/run_tests/theta-fhs3-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fhs3-kokkos.cmake new file mode 100644 index 000000000000..ba19805cc7c5 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs3-kokkos.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs3-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs3.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fhs3.cmake b/components/homme/test/reg_test/run_tests/theta-fhs3.cmake new file mode 100644 index 000000000000..769fa7587c1a --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fhs3.cmake @@ -0,0 +1,18 @@ + +SET(TEST_NAME theta-fhs3) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetahs3.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + held_suarez01.nc + held_suarez02.nc) + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) From a9cb1c93fcedae71e433032d1b98bcc33af613a1 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 3 Nov 2021 12:35:56 -0500 Subject: [PATCH 109/301] remove old tests --- .../homme/test/reg_test/namelists/thetaHS.nl | 76 ------------------- .../run_tests/theta-f1-tt10-HS-kokkos.cmake | 23 ------ .../reg_test/run_tests/theta-f1-tt10-HS.cmake | 23 ------ 3 files changed, 122 deletions(-) delete mode 100644 components/homme/test/reg_test/namelists/thetaHS.nl delete mode 100644 components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake delete mode 100644 components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake diff --git a/components/homme/test/reg_test/namelists/thetaHS.nl b/components/homme/test/reg_test/namelists/thetaHS.nl deleted file mode 100644 index b3780f72b243..000000000000 --- a/components/homme/test/reg_test/namelists/thetaHS.nl +++ /dev/null @@ -1,76 +0,0 @@ -&ctl_nl -vthreads = 1 -NThreads = 1 -partmethod = 4 -topology = "cube" -test_case = "held_suarez0" -u_perturb = 1 -rotate_grid = 0 -ne = ${HOMME_TEST_NE} -qsize = 1 -ndays = ${HOMME_TEST_NDAYS} -statefreq = 9999 -restartfreq = 43200 -restartfile = "./R0001" -runtype = 0 -mesh_file = '/dev/null' -tstep = 100 -rsplit = 1 !3 -qsplit = 1 -integration = "explicit" -smooth = 0 -nu = 3.4e-8 -nu_div = 3.4e-8 ! test this too? -nu_p = -1 -nu_q = -1 -nu_s = -1 -nu_top = 0 !2.5e5 -se_ftype = 0 -limiter_option = 9 -vert_remap_q_alg = 1 -hypervis_scaling = 3.0 -hypervis_order = 2 -hypervis_subcycle = 5 -hypervis_subcycle_tom = 6 -theta_hydrostatic_mode = false -theta_advect_form = 1 -tstep_type = 10 -/ -&solver_nl -precon_method = "identity" -maxits = 500 -tol = 1.e-9 -/ -&filter_nl -filter_type = "taylor" -transfer_type = "bv" -filter_freq = 0 -filter_mu = 0.04D0 -p_bv = 12.0D0 -s_bv = .666666666666666666D0 -wght_fm = 0.10D0 -kcut_fm = 2 -/ -&vert_nl -vform = "ccm" -vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' -vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' -/ - -&prof_inparm -profile_outpe_num = 100 -profile_single_file = .true. -/ - -! timunits: 0= steps, 1=days, 2=hours -&analysis_nl - interp_gridtype = 2 - output_timeunits = 1,1 - output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} - output_start_time = 0,0 - output_end_time = 30000,30000 - output_varnames1 = 'ps','zeta','u','v','T' - output_varnames2 = 'Q','Q2','Q3','Q4' - io_stride = 8 - output_type = 'netcdf' -/ diff --git a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake deleted file mode 100644 index 2c9845069e8f..000000000000 --- a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS-kokkos.cmake +++ /dev/null @@ -1,23 +0,0 @@ -# The name of this test (should be the basename of this file) -#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu -#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos -#adding BB to each var to avoid unwanted substitutions - -SET(TEST_NAME theta-f1-tt10-HS-kokkos) -# The specifically compiled executable that this test uses -SET(EXEC_NAME theta-nlev128-kokkos) - -SET(NUM_CPUS 16) - -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetaHS.nl) -SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) - -# compare all of these files against baselines: -SET(NC_OUTPUT_FILES - held_suarez01.nc - held_suarez02.nc) - -#DO NOT MOD -SET (HOMME_TEST_TIME_STEP 600) -SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) -SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake b/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake deleted file mode 100644 index 9ed79f574002..000000000000 --- a/components/homme/test/reg_test/run_tests/theta-f1-tt10-HS.cmake +++ /dev/null @@ -1,23 +0,0 @@ -# The name of this test (should be the basename of this file) -#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu -#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos -#adding BB to each var to avoid unwanted substitutions - -SET(TEST_NAME theta-f1-tt10-HS) -# The specifically compiled executable that this test uses -SET(EXEC_NAME theta-nlev128) - -SET(NUM_CPUS 16) - -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/thetaHS.nl) -SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) - -# compare all of these files against baselines: -SET(NC_OUTPUT_FILES - held_suarez01.nc - held_suarez02.nc) - -#DO NOT MOD -SET (HOMME_TEST_TIME_STEP 600) -SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) -SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) From fb8ee38e0a53d5ad80123bee637114d39eab20bd Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Wed, 3 Nov 2021 12:36:57 -0500 Subject: [PATCH 110/301] Corrected coupling of ice biogeochemistry for MARBL Adds coupling of refractory DOC with sea ice humics Also adds option to couple sea ice DOC with the macromolecules or the MARBL DOC pool. Corrects coupling of DON to conserve carbon --- components/mpas-ocean/driver/ocn_comp_mct.F | 10 ++++++- .../shared/mpas_ocn_time_average_coupled.F | 7 +++++ .../src/shared/mpas_ocn_tracer_ecosys.F | 3 +- .../tracer_groups/Registry_MacroMolecules.xml | 4 +-- .../src/tracer_groups/Registry_ecosys.xml | 7 +++-- components/mpas-seaice/bld/build-namelist | 1 + .../mpas-seaice/bld/build-namelist-section | 1 + .../namelist_defaults_mpassi.xml | 1 + .../namelist_definition_mpassi.xml | 8 +++++ components/mpas-seaice/driver/ice_comp_mct.F | 18 +++++++----- components/mpas-seaice/src/Registry.xml | 8 ++++- .../mpas-seaice/src/column/ice_algae.F90 | 4 +-- .../mpas-seaice/src/column/ice_colpkg.F90 | 29 ++++++++++++++----- .../src/column/ice_zbgc_shared.F90 | 8 +++-- .../src/shared/mpas_seaice_column.F | 10 +++++-- 15 files changed, 91 insertions(+), 28 deletions(-) diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index 281e62a612ea..c39f0b2897cf 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -2128,6 +2128,7 @@ subroutine ocn_import_mct(x2o_o, errorCode)!{{{ if ( iceFluxDOCField % isActive ) then iceFluxDOC(1,i) = x2o_o % rAttr(index_x2o_Fioi_doc1, n) iceFluxDOC(2,i) = x2o_o % rAttr(index_x2o_Fioi_doc2, n) + iceFluxDOC(3,i) = x2o_o % rAttr(index_x2o_Fioi_doc3, n) endif if ( iceFluxDONField % isActive ) then iceFluxDON(i) = x2o_o % rAttr(index_x2o_Fioi_don1, n) @@ -2496,6 +2497,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ avgOceanSurfaceDMS, & avgOceanSurfaceDMSP, & avgOceanSurfaceDOCr, & + avgOceanSurfaceDOCSum, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved @@ -2579,6 +2581,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) endif @@ -2665,6 +2668,10 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_So_algae2, n) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) o2x_o % rAttr(index_o2x_So_algae3, n) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) o2x_o % rAttr(index_o2x_So_dic1, n) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) + o2x_o % rAttr(index_o2x_So_doc1, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) + o2x_o % rAttr(index_o2x_So_doc2, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) + o2x_o % rAttr(index_o2x_So_doc3, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) + o2x_o % rAttr(index_o2x_So_don1, n) = 0.0_RKIND o2x_o % rAttr(index_o2x_So_no3, n) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) o2x_o % rAttr(index_o2x_So_sio3, n) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) o2x_o % rAttr(index_o2x_So_nh4, n) = max(0.0_RKIND,avgOceanSurfaceNH4(i)) @@ -2679,7 +2686,8 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ if (config_use_MacroMoleculesTracers .and. config_use_MacroMoleculesTracers_sea_ice_coupling) then o2x_o % rAttr(index_o2x_So_doc1, n) = max(0.0_RKIND,avgOceanSurfaceDOC(1,i)) o2x_o % rAttr(index_o2x_So_doc2, n) = max(0.0_RKIND,avgOceanSurfaceDOC(2,i)) - o2x_o % rAttr(index_o2x_So_don1, n) = max(0.0_RKIND,avgOceanSurfaceDON(i)) + o2x_o % rAttr(index_o2x_So_doc3, n) = max(0.0_RKIND,avgOceanSurfaceDOC(3,i)) + o2x_o % rAttr(index_o2x_So_don1, n) = 0.0_RKIND endif ! o2x_o % rAttr(index_o2x_Faoo_fco2_ocn, n) = CO2Flux(i) ! o2x_o % rAttr(index_o2x_Faoo_fdms_ocn, n) = DMSFlux(i) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F index f135f1bb059e..211514efe173 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F @@ -74,6 +74,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgOceanSurfaceDMS, & avgOceanSurfaceDMSP, & avgOceanSurfaceDOCr, & + avgOceanSurfaceDOCSum, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved @@ -129,6 +130,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) @@ -140,6 +142,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgOceanSurfacePhytoC(:,iCell) = 0.0_RKIND avgOceanSurfaceDIC(iCell) = 0.0_RKIND + avgOceanSurfaceDOCSum(iCell) = 0.0_RKIND avgOceanSurfaceNO3(iCell) = 0.0_RKIND avgOceanSurfaceSiO3(iCell) = 0.0_RKIND avgOceanSurfaceNH4(iCell) = 0.0_RKIND @@ -228,6 +231,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel real (kind=RKIND), dimension(:), pointer :: avgOceanSurfaceDIC, & avgOceanSurfaceDON, & + avgOceanSurfaceDOCSum, & avgOceanSurfaceNO3, & avgOceanSurfaceSiO3, & avgOceanSurfaceNH4, & @@ -307,6 +311,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) @@ -335,6 +340,8 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel avgOceanSurfaceDIC(iCell) = ( avgOceanSurfaceDIC(iCell) * nAccumulatedCoupled & + ecosysTracers(dic_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) + avgOceanSurfaceDOCSum(iCell) = ( avgOceanSurfaceDOCSum(iCell) * nAccumulatedCoupled & + + ecosysTracers(doc_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) avgOceanSurfaceSiO3(iCell) = ( avgOceanSurfaceSiO3(iCell) * nAccumulatedCoupled & + ecosysTracers(sio3_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) avgOceanSurfaceNO3(iCell) = ( avgOceanSurfaceNO3(iCell) * nAccumulatedCoupled & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F index 6cf20f63b738..e8726e560f40 100755 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_ecosys.F @@ -1987,7 +1987,8 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, ecosysSurfaceFlux(dic_alt_co2_ind_MPAS,iCell) = & ecosysSurfaceFlux(dic_alt_co2_ind_MPAS,iCell) + iceFluxDIC(iCell) ecosysSurfaceFlux(doc_ind_MPAS,iCell) = & - ecosysSurfaceFlux(doc_ind_MPAS,iCell) + iceFluxDOC(1,iCell) + iceFluxDOC(2,iCell) + ecosysSurfaceFlux(doc_ind_MPAS,iCell) + iceFluxDOC(1,iCell) + iceFluxDOC(2,iCell) & + + iceFluxDOC(3,iCell) ! this is just a placeholder ecosysSurfaceFlux(donr_ind_MPAS,iCell) = & ecosysSurfaceFlux(donr_ind_MPAS,iCell) + iceFluxDOCr(iCell) diff --git a/components/mpas-ocean/src/tracer_groups/Registry_MacroMolecules.xml b/components/mpas-ocean/src/tracer_groups/Registry_MacroMolecules.xml index e4b6c3ac6809..4b783b90f517 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_MacroMolecules.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_MacroMolecules.xml @@ -121,8 +121,8 @@ - + @@ -732,8 +735,8 @@ - ".false."); } add_default($nl, 'config_use_chlorophyll'); +add_default($nl, 'config_use_macromolecules'); add_default($nl, 'config_use_modal_aerosols'); add_default($nl, 'config_use_zaerosols'); add_default($nl, 'config_skeletal_bgc_flux_type'); diff --git a/components/mpas-seaice/bld/build-namelist-section b/components/mpas-seaice/bld/build-namelist-section index a593719fa86a..e7bb257f1e0b 100644 --- a/components/mpas-seaice/bld/build-namelist-section +++ b/components/mpas-seaice/bld/build-namelist-section @@ -193,6 +193,7 @@ add_default($nl, 'config_use_skeletal_biochemistry'); add_default($nl, 'config_use_nitrate'); add_default($nl, 'config_use_carbon'); add_default($nl, 'config_use_chlorophyll'); +add_default($nl, 'config_use_macromolecules'); add_default($nl, 'config_use_ammonium'); add_default($nl, 'config_use_silicate'); add_default($nl, 'config_use_DMS'); diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index 393242529c6e..ccfa15657313 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -188,6 +188,7 @@ false false false +false false false false diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index f43393015c35..3e6a78d5d224 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -955,6 +955,14 @@ Valid values: false Default: Defined in namelist_defaults.xml + +Use ocean macromolecule to determine DOC fractions in ice-ocean coupling + +Valid values: true or false +Default: Defined in namelist_defaults.xml + + Use the ammonium tracer diff --git a/components/mpas-seaice/driver/ice_comp_mct.F b/components/mpas-seaice/driver/ice_comp_mct.F index f44d028edc1a..20a3d00fb70d 100644 --- a/components/mpas-seaice/driver/ice_comp_mct.F +++ b/components/mpas-seaice/driver/ice_comp_mct.F @@ -1866,7 +1866,8 @@ subroutine ice_import_mct(x2i_i, errorCode)!{{{ oceanDMSPConc, & oceanHumicsConc, & carbonToNitrogenRatioAlgae, & - carbonToNitrogenRatioDON + carbonToNitrogenRatioDON, & + DOCPoolFractions real (kind=RKIND), dimension(:,:), pointer :: & oceanAlgaeConc, & @@ -1960,6 +1961,8 @@ subroutine ice_import_mct(x2i_i, errorCode)!{{{ call mpas_pool_get_array(biogeochemistry, 'oceanZAerosolConc', oceanZAerosolConc) call mpas_pool_get_array(biogeochemistry, 'carbonToNitrogenRatioAlgae', carbonToNitrogenRatioAlgae) call mpas_pool_get_array(biogeochemistry, 'carbonToNitrogenRatioDON', carbonToNitrogenRatioDON) + call mpas_pool_get_array(biogeochemistry, 'DOCPoolFractions', DOCPoolFractions) + if (config_use_zaerosols) then call mpas_pool_get_array(biogeochemistry, "atmosBlackCarbonFlux", atmosBlackCarbonFlux) call mpas_pool_get_array(biogeochemistry, "atmosDustFlux", atmosDustFlux) @@ -2056,12 +2059,13 @@ subroutine ice_import_mct(x2i_i, errorCode)!{{{ if (config_use_column_biogeochemistry) then oceanAlgaeConc(1,i) = x2i_i % rAttr(index_x2i_So_algae1, n) oceanAlgaeConc(2,i) = x2i_i % rAttr(index_x2i_So_algae2, n) - oceanAlgaeConc(3,i) = x2i_i % rAttr(index_x2i_So_algae3, n) - oceanDOCConc(1,i) = x2i_i % rAttr(index_x2i_So_doc1, n) - oceanDOCConc(2,i) = x2i_i % rAttr(index_x2i_So_doc2, n) + oceanAlgaeConc(3,i) = 0.0_RKIND !x2i_i % rAttr(index_x2i_So_algae3, n) + oceanDOCConc(1,i) = x2i_i % rAttr(index_x2i_So_doc1, n) * DOCPoolFractions(1) + oceanDOCConc(2,i) = x2i_i % rAttr(index_x2i_So_doc2, n) * DOCPoolFractions(2) oceanDOCConc(3,i) = 0.0_RKIND - oceanDICConc(1,i) = x2i_i % rAttr(index_x2i_So_dic1, n) !JW not used, set to 0? - oceanDONConc(1,i) = x2i_i % rAttr(index_x2i_So_don1, n) + oceanDICConc(1,i) = x2i_i % rAttr(index_x2i_So_dic1, n) + oceanDONConc(1,i) = x2i_i % rAttr(index_x2i_So_don1, n) * DOCPoolFractions(3) & + /carbonToNitrogenRatioDON(1) oceanNitrateConc(i) = x2i_i % rAttr(index_x2i_So_no3, n) oceanSilicateConc(i) = x2i_i % rAttr(index_x2i_So_sio3, n) oceanAmmoniumConc(i) = x2i_i % rAttr(index_x2i_So_nh4, n) @@ -2573,7 +2577,7 @@ subroutine ice_export_mct(i2x_i, errorCode) !{{{ i2x_i % rAttr(index_i2x_Fioi_algae3,n) = oceanAlgaeFlux(3,i) * carbonToNitrogenRatioAlgae(3) i2x_i % rAttr(index_i2x_Fioi_doc1 ,n) = oceanDOCFlux(1,i) i2x_i % rAttr(index_i2x_Fioi_doc2 ,n) = oceanDOCFlux(2,i) - i2x_i % rAttr(index_i2x_Fioi_doc3 ,n) = oceanDOCFlux(3,i) !JW set to 0? + i2x_i % rAttr(index_i2x_Fioi_doc3 ,n) = oceanDONFlux(3,i) * carbonToNitrogenRatioDON(1) i2x_i % rAttr(index_i2x_Fioi_dic1 ,n) = oceanDICFlux(1,i) i2x_i % rAttr(index_i2x_Fioi_don1 ,n) = oceanDONFlux(1,i) i2x_i % rAttr(index_i2x_Fioi_no3 ,n) = oceanNitrateFlux(i) diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index b005131f14f9..759c3f548a14 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -161,7 +161,7 @@ description="Specifies the third dimension of the modal aerosol optical parameter matrix" /> + + diff --git a/components/mpas-seaice/src/column/ice_algae.F90 b/components/mpas-seaice/src/column/ice_algae.F90 index 1468fc9b0a81..45f443c717dc 100644 --- a/components/mpas-seaice/src/column/ice_algae.F90 +++ b/components/mpas-seaice/src/column/ice_algae.F90 @@ -2239,8 +2239,8 @@ subroutine algal_dyn (dt, & reactb(nlt_bgc_DMS) = DMS_s - DMS_r endif if (tr_bgc_C) then - if (abs(dC) > maxval(abs(reactb(:)))*1.0e-13_dbl_kind .or. & - abs(dN) > maxval(abs(reactb(:)))*1.0e-13_dbl_kind) then + if (abs(dC) > max(puny,maxval(abs(reactb(:)))*1.0e-13_dbl_kind) .or. & + abs(dN) > max(puny,maxval(abs(reactb(:)))*1.0e-13_dbl_kind)) then conserve_C = .false. write(warning, *) 'Conservation error!' call add_warning(warning) diff --git a/components/mpas-seaice/src/column/ice_colpkg.F90 b/components/mpas-seaice/src/column/ice_colpkg.F90 index 285d43984026..bde30aa9e2eb 100644 --- a/components/mpas-seaice/src/column/ice_colpkg.F90 +++ b/components/mpas-seaice/src/column/ice_colpkg.F90 @@ -533,10 +533,11 @@ subroutine colpkg_init_bgc(dt, ncat, nblyr, nilyr, ntrcr_o, cgrid, igrid, & doc, don, dic, fed, fep, zaeros, hum, & ocean_bio_all, & max_algae, max_doc, max_dic, max_don, max_fe, max_nbtrcr, max_aero, & - l_stop, stop_label) + DOCPoolFractions, use_macromolecules, l_stop, stop_label) use ice_constants_colpkg, only: c0, c1, c2, p1, p15, p5 - use ice_zbgc_shared, only: R_S2N, zbgc_frac_init, zbgc_init_frac, remap_zbgc + use ice_zbgc_shared, only: R_S2N, zbgc_frac_init, zbgc_init_frac, remap_zbgc, & + doc_pool_fractions ! column package includes use ice_colpkg_tracers, only: nt_fbri, nt_bgc_S, nt_sice, nt_zbgc_frac, & @@ -562,21 +563,23 @@ subroutine colpkg_init_bgc(dt, ncat, nblyr, nilyr, ntrcr_o, cgrid, igrid, & max_fe, & max_nbtrcr, & max_aero - + real (kind=dbl_kind), dimension (nblyr+1), intent(inout) :: & igrid ! biology vertical interface points - + real (kind=dbl_kind), dimension (nilyr+1), intent(inout) :: & - cgrid ! CICE vertical coordinate + cgrid ! CICE vertical coordinate - logical (kind=log_kind), intent(in) :: & - restart_bgc ! if .true., read bgc restart file + logical (kind=log_kind), intent(in) :: & + restart_bgc, & ! if .true., read bgc restart file + use_macromolecules ! if .true., doc ocean fractions are determined & + ! by the ocean macromolecules subroutine real (kind=dbl_kind), dimension(nilyr, ncat), intent(in) :: & sicen ! salinity on the cice grid real (kind=dbl_kind), dimension (:,:), intent(inout) :: & - trcrn ! subset of tracer array (only bgc) + trcrn ! subset of tracer array (only bgc) real (kind=dbl_kind), intent(in) :: & sss ! sea surface salinity (ppt) @@ -610,6 +613,9 @@ subroutine colpkg_init_bgc(dt, ncat, nblyr, nilyr, ntrcr_o, cgrid, igrid, & real (kind=dbl_kind), dimension (:), intent(inout) :: & ocean_bio_all ! fixed order, all values even for tracers false + real (kind=dbl_kind), dimension (:), intent(out) :: & + DOCPoolFractions ! Fraction of DOC in polysacharids, lipids, and proteins + logical (kind=log_kind), intent(inout) :: & l_stop ! if true, print diagnostics and abort on return @@ -642,6 +648,13 @@ subroutine colpkg_init_bgc(dt, ncat, nblyr, nilyr, ntrcr_o, cgrid, igrid, & zspace(nblyr+1) = p5*zspace(nblyr+1) ntrcr_bgc = ntrcr-ntrcr_o + DOCPoolFractions(:) = c1 + if (.not. use_macromolecules) then + do mm = 1,max_doc + DOCPoolFractions(mm) = doc_pool_fractions(mm) + end do + end if + call colpkg_init_OceanConcArray(max_nbtrcr, & max_algae, max_don, max_doc, & max_dic, max_aero, max_fe, & diff --git a/components/mpas-seaice/src/column/ice_zbgc_shared.F90 b/components/mpas-seaice/src/column/ice_zbgc_shared.F90 index 58083b63e2ba..130f03168736 100644 --- a/components/mpas-seaice/src/column/ice_zbgc_shared.F90 +++ b/components/mpas-seaice/src/column/ice_zbgc_shared.F90 @@ -29,12 +29,16 @@ module ice_zbgc_shared R_Fe2C , & ! algal Fe to carbon (umol/mmol) R_Fe2N ! algal Fe to N (umol/mmol) - real (kind=dbl_kind), dimension(max_don), public :: & + real (kind=dbl_kind), dimension(max_don), public :: & R_Fe2DON ! Fe to N of DON (nmol/umol) - real (kind=dbl_kind), dimension(max_doc), public :: & + real (kind=dbl_kind), dimension(max_doc), public :: & R_Fe2DOC ! Fe to C of DOC (nmol/umol) + ! polysaccharids, lipids, proteins+nucleic acids (Lonborg et al. 2020) + real (kind=dbl_kind), dimension(max_doc), parameter, public :: & + doc_pool_fractions = (/0.26_dbl_kind, 0.17_dbl_kind, 0.57_dbl_kind/) + real (kind=dbl_kind), parameter, public :: & R_gC2molC = 12.01_dbl_kind ! mg/mmol C diff --git a/components/mpas-seaice/src/shared/mpas_seaice_column.F b/components/mpas-seaice/src/shared/mpas_seaice_column.F index b0e3498b7aab..2a2ddfb2bc91 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_column.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_column.F @@ -13033,7 +13033,8 @@ subroutine init_column_biogeochemistry_profiles(domain, tracerObject) config_use_skeletal_biochemistry, & config_do_restart_zsalinity, & config_do_restart_bgc, & - config_do_restart_hbrine + config_do_restart_hbrine, & + config_use_macromolecules real(kind=RKIND), pointer :: & config_dt, & @@ -13052,7 +13053,8 @@ subroutine init_column_biogeochemistry_profiles(domain, tracerObject) biologyGrid, & ! bgrid verticalShortwaveGrid, & ! swgrid interfaceGrid, & ! icgrid - rayleighCriteriaReal + rayleighCriteriaReal, & + DOCPoolFractions real(kind=RKIND), dimension(:,:), pointer :: & oceanAlgaeConc, & @@ -13116,6 +13118,7 @@ subroutine init_column_biogeochemistry_profiles(domain, tracerObject) call MPAS_pool_get_config(domain % configs, "config_use_vertical_tracers", config_use_vertical_tracers) call MPAS_pool_get_config(domain % configs, "config_dt", config_dt) call MPAS_pool_get_config(domain % configs, "config_snow_porosity_at_ice_surface", config_snow_porosity_at_ice_surface) + call MPAS_pool_get_config(domain % configs, "config_use_macromolecules", config_use_macromolecules) abortFlag = .false. @@ -13156,6 +13159,7 @@ subroutine init_column_biogeochemistry_profiles(domain, tracerObject) call MPAS_pool_get_array(biogeochemistry, "oceanHumicsConc", oceanHumicsConc) call MPAS_pool_get_array(biogeochemistry, "oceanZAerosolConc", oceanZAerosolConc) call MPAS_pool_get_array(biogeochemistry, "newlyFormedIce", newlyFormedIce) + call MPAS_pool_get_array(biogeochemistry, "DOCPoolFractions", DOCPoolFractions) call MPAS_pool_get_subpool(block % structs, "ocean_coupling", ocean_coupling) call MPAS_pool_get_array(ocean_coupling, "seaSurfaceSalinity", seaSurfaceSalinity) @@ -13252,6 +13256,8 @@ subroutine init_column_biogeochemistry_profiles(domain, tracerObject) maxIronType, & nZBGCTracers, & maxAerosolType, & + DOCPoolFractions, & + config_use_macromolecules, & abortFlag, & abortMessage) From 2632af1f02ad80cac2d860518230583d656bb188 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 3 Nov 2021 22:40:47 -0600 Subject: [PATCH 111/301] Hommexx: change pdf for randomizing vtheta_dp The new pdf has more realistic bounds, which hopefully will make it harder to hit combinations of dp, phinh, vtheta_dp that yield errors in the EOS (such as neg pnh). --- components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp index e41646312161..631ebb705484 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/ElementsState.cpp @@ -185,10 +185,11 @@ void ElementsState::randomize(const int seed, std::mt19937_64 engine(seed); std::uniform_real_distribution random_dist(min_value, 1.0 / min_value); + std::uniform_real_distribution pdf_vtheta_dp(100.0, 1000.0); genRandArray(m_v, engine, random_dist); genRandArray(m_w_i, engine, random_dist); - genRandArray(m_vtheta_dp, engine, random_dist); + genRandArray(m_vtheta_dp, engine, pdf_vtheta_dp); // Note: to avoid errors in the equation of state, we need phi to be increasing. // Rather than using a constraint (which may call the function many times, // we simply ask that there are no duplicates, then we sort it later. From e4c1bc29e1414bdba04a6ff394e9801f8e724cea Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Wed, 3 Nov 2021 22:41:21 -0600 Subject: [PATCH 112/301] Hommexx: fix valgrind error in forcing_ut --- components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 323f632bf1d6..d57f61dd9482 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -278,6 +278,12 @@ TEST_CASE("forcing", "forcing") { SECTION ("states") { + // Set theta_hydrostatic_mode in simulation parameters, since it is + // used later in the ForcingFunctor setup. If you don't set it, the FF + // will have conditional jumps depending on uninited memory + auto& p = Context::singleton().get(); + p.theta_hydrostatic_mode = false; + // Reset state and forcing to the original random values std::cout << "Testing dynamics forcing.\n"; From 371c9c764653cb2b2bf1ca88b9c388d80b82e4bc Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Thu, 4 Nov 2021 10:51:18 -0600 Subject: [PATCH 113/301] add -ffpe-summary=none to stand-alone Makefile for gnu --- components/mpas-framework/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 3bece571a04d..4dcdc6bd3a09 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -335,11 +335,11 @@ gfortran: "CC_SERIAL = gcc" \ "CXX_SERIAL = g++" \ "FFLAGS_PROMOTION = -fdefault-real-8 -fdefault-double-8" \ - "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form" \ + "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ "LDFLAGS_OPT = -O3 -m64" \ - "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow" \ + "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -O3 -m64" \ "LDFLAGS_DEBUG = -g -m64" \ @@ -454,11 +454,11 @@ gnu-nersc: "CC_SERIAL = cc" \ "CXX_SERIAL = CC" \ "FFLAGS_PROMOTION = -fdefault-real-8 -fdefault-double-8" \ - "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form" \ + "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ "LDFLAGS_OPT = -O3 -m64" \ - "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form" \ + "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ "LDFLAGS_DEBUG = -g -m64" \ From 589c0822838139ebc85ecfe99c52ad865c015005 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Thu, 4 Nov 2021 13:43:41 -0600 Subject: [PATCH 114/301] Add -ffpe-trap=invalid,zero,overflow to gnu-nersc --- components/mpas-framework/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 4dcdc6bd3a09..4fe250e636aa 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -458,7 +458,7 @@ gnu-nersc: "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ "LDFLAGS_OPT = -O3 -m64" \ - "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none" \ + "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-trap=invalid,zero,overflow -ffpe-summary=none" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ "LDFLAGS_DEBUG = -g -m64" \ From 6db8f13ec7b74d6889264ef6eae2d27e260a3c18 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 4 Nov 2021 15:34:57 -0600 Subject: [PATCH 115/301] Hommexx: bug fix in one overload of sync_to_device We were syncing in the opposite direction. --- components/homme/src/share/cxx/utilities/SyncUtils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/share/cxx/utilities/SyncUtils.hpp b/components/homme/src/share/cxx/utilities/SyncUtils.hpp index ba85b2b6f88d..d337d211107c 100644 --- a/components/homme/src/share/cxx/utilities/SyncUtils.hpp +++ b/components/homme/src/share/cxx/utilities/SyncUtils.hpp @@ -508,7 +508,7 @@ sync_to_device(Source_T source, Dest_T dest) } } } - Kokkos::deep_copy(dest_mirror, dest); + Kokkos::deep_copy(dest, dest_mirror); } template From aa07726796ac28bd6c049bc36b8a78eb75cd5792 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 4 Nov 2021 15:39:07 -0600 Subject: [PATCH 116/301] Hommexx: fix initialization of tracers forcing I am not sure if/when we hit the specific code branch that had the wrong initialization. Still, it was buggy, and now it's fixed. Namely, the buggy initialization was passing only num_elems to the ctor of the view, but fq has 2 dyn dimension, the latter begin the number of tracers (qsize). --- .../homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index d70af7eb7d7d..d6eee1bf474c 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -226,7 +226,7 @@ void push_forcing_to_c (F90Ptr elem_derived_FM, const SimulationParams ¶ms = Context::singleton().get(); Tracers &tracers = Context::singleton().get(); if (tracers.fq.data() == nullptr) { - tracers.fq = decltype(tracers.fq)("fq", num_elems); + tracers.fq = decltype(tracers.fq)("fq", num_elems, tracers.num_tracers()); } HostViewUnmanaged fq_f90( elem_derived_FQ, num_elems); From 5e54b3c696e0a70027bea9a4fd98824cb46f0f4e Mon Sep 17 00:00:00 2001 From: mark-petersen Date: Fri, 5 Nov 2021 13:36:07 -0700 Subject: [PATCH 117/301] Add -fbounds-check -fbacktrace to gnu-nersc --- components/mpas-framework/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 4fe250e636aa..c4204614a077 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -458,7 +458,7 @@ gnu-nersc: "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ "LDFLAGS_OPT = -O3 -m64" \ - "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-trap=invalid,zero,overflow -ffpe-summary=none" \ + "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ "LDFLAGS_DEBUG = -g -m64" \ From a777d352cf64291678dbd66340a420980dd0f46d Mon Sep 17 00:00:00 2001 From: mark-petersen Date: Fri, 5 Nov 2021 13:38:17 -0700 Subject: [PATCH 118/301] Add -ffpe-summary=none to gfortran-clang --- components/mpas-framework/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index c4204614a077..e061556378c7 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -362,11 +362,11 @@ gfortran-clang: "CC_SERIAL = clang" \ "CXX_SERIAL = clang++" \ "FFLAGS_PROMOTION = -fdefault-real-8 -fdefault-double-8" \ - "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form" \ + "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ "LDFLAGS_OPT = -O3 -m64" \ - "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow" \ + "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -O3 -m64" \ "LDFLAGS_DEBUG = -g -m64" \ From 694b85d1f1bb1fdb9835eeadc29802c4ce890a4f Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Fri, 15 Oct 2021 15:15:44 -0700 Subject: [PATCH 119/301] Fixes the script to create maps for ELM Modifies various files to change CLM related variables to ELM to account for the name change of the land model. --- ..._tools.xml => namelist_defaults_tools.xml} | 0 .../namelist_files/namelist_definition.xml | 116 +++++++++--------- components/elm/bld/queryDefaultNamelist.pl | 4 +- .../elm/tools/shared/mkmapdata/mkmapdata.sh | 50 +++----- 4 files changed, 77 insertions(+), 93 deletions(-) rename components/elm/bld/namelist_files/{namelist_defaults_clm4_5_tools.xml => namelist_defaults_tools.xml} (100%) diff --git a/components/elm/bld/namelist_files/namelist_defaults_clm4_5_tools.xml b/components/elm/bld/namelist_files/namelist_defaults_tools.xml similarity index 100% rename from components/elm/bld/namelist_files/namelist_defaults_clm4_5_tools.xml rename to components/elm/bld/namelist_files/namelist_defaults_tools.xml diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 463921f5344c..36618ae80d43 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -679,20 +679,20 @@ SCRIP format grid data file + group="elmexp" valid_values="none,64bit_offset,netcdf4" > Flag to pass to the ESMF mapping utility, telling it what kind of large file support is needed for an output file generated with this grid as either the source or destination ('none', '64bit_offset' or 'netcdf4'). + group="elmexp" valid_values="SCRIP,UGRID" > Flag to pass to the ESMF mapping utility, telling it what kind of grid file this is (SCRIP or UGRID). + group="elmexp" valid_values="" > For UGRID files, flag to pass to the ESMF mapping utility, telling it the name of the dummy variable that has all of the topology information stored in its attributes. (Only used if scripgriddata_src_type = UGRID.) @@ -708,210 +708,210 @@ Filename for mksurfdata_map to remap raw data into the output surface dataset + input_pathname="abs" group="elmexp" valid_values="" > Plant Function Type dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Dataset for percent glacier land-unit for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Dataset for topography used to define urban threshold + input_pathname="abs" group="elmexp" valid_values="" > Dataset for land topography + input_pathname="abs" group="elmexp" valid_values="" > Leaf Area Index dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil texture dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil color dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil order dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil max fraction dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > High resolution land mask/fraction dataset for mksurfdata (used for glacier_mec land-units) + group="elmexp" value="global" valid_values="global,regional" > Type of grid to create for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Grid file at the output resolution for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Text file with filepaths (or list of XML elements) for vegetation fractions and harvesting for each year to run over for mksurfdata to be able to model transient land-use change + input_pathname="abs" group="elmexp" valid_values="" > High resolution topography dataset for mksurfdata (used for glacier_mec land-units) + input_pathname="abs" group="elmexp" valid_values="" > Irrigation dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Organic soil dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Lake water dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Wetland dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Urban dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Biogenic Volatile Organic Compounds (VOC) emissions dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > GDP dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Peat dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Agricultural burning dominant month dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Topography statistics dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > VIC parameters dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Inversion-derived CH4 parameters dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil phosphorus dataset for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Soil gravel content dataset + input_pathname="abs" group="elmexp" valid_values="" > Slope percentile dataset + input_pathname="abs" group="elmexp" valid_values="" > ELM-Erosion parameters dataset + group="elmexp" value=".true."> If TRUE, output variables in double precision for mksurfdata + group="elmexp" value=".false."> If TRUE, ignore other files, and set the output percentage to 100% urban and zero for other land-use types. + group="elmexp" value=".true."> If TRUE, set wetland to 0% over land (renormalizing other landcover types as needed); wetland will only be used for ocean points. + group="elmexp" value="16" valid_values="16,20" > Number of Plant Functional Types (excluding bare-soil) Plant Function Type index to override global file with for mksurfdata + group="elmexp" valid_values=""> Plant Function Type fraction to override global file with for mksurfdata Soil color index to override global file with for mksurfdata + group="elmexp" > Soil maximum fraction to override global file with for mksurfdata + group="elmexp" > Soil percent sand to override global file with for mksurfdata + group="elmexp" > Soil percent clay to override global file with for mksurfdata @@ -920,62 +920,62 @@ Soil percent clay to override global file with for mksurfdata + input_pathname="abs" group="elmexp" valid_values="" > Orography file with surface heights and land area fraction + input_pathname="abs" group="elmexp" valid_values="" > CLM grid file + input_pathname="abs" group="elmexp" valid_values="" > CESM domain file + input_pathname="abs" group="elmexp" valid_values="" > CAM file + input_pathname="abs" group="elmexp" valid_values="" > Raw topography file + input_pathname="abs" group="elmexp" valid_values="" > CAM topography file + group="elmexp" valid_values="" > Number of longitudes to use for a regional grid (for single-point set to 1) + group="elmexp" valid_values="" > Number of latitudes to use for a regional grid (for single-point set to 1) + group="elmexp" valid_values="" > Northern edge of the regional grid + group="elmexp" valid_values="" > Southern edge of the regional grid + group="elmexp" valid_values="" > Eastern edge of the regional grid + group="elmexp" valid_values="" > Western edge of the regional grid @@ -984,7 +984,7 @@ Western edge of the regional grid + input_pathname="abs" group="elmexp" valid_values="" > Historical greenhouse gas concentrations from CAM, only used by getco2_historical.ncl @@ -993,7 +993,7 @@ by getco2_historical.ncl + input_pathname="abs" group="elmexp" valid_values="" > Aerosol deposition file name (only used for aerdepregrid.ncl) @@ -1221,7 +1221,7 @@ Datm logfile name + input_pathname="abs" group="elmexp" valid_values="" > Mapping file to go from one resolution/land-mask to another resolution/land-mask diff --git a/components/elm/bld/queryDefaultNamelist.pl b/components/elm/bld/queryDefaultNamelist.pl index 848bdd48efb9..181a7ee04d41 100755 --- a/components/elm/bld/queryDefaultNamelist.pl +++ b/components/elm/bld/queryDefaultNamelist.pl @@ -75,7 +75,6 @@ sub usage { -onlyfiles Only output filenames. -options "item=value,item2=value2" Set options to query for when matching. (comma delimited, with equality to set value). - -phys "ELM-version" [or -p] ELM version to use -res "resolution" Resolution to use for files. Use "-res list" to list all valid resolutions. Use "-res any" to use any valid resolution. @@ -130,7 +129,6 @@ sub usage { "f|file=s" => \$opts{'file'}, "n|namelist=s" => \$opts{'namelist'}, "v|var=s" => \$opts{'var'}, - "p|phys=s" => \$opts{'model'}, "r|res=s" => \$opts{'hgrid'}, "config=s" => \$opts{'config'}, "cesm" => \$opts{'cesm'}, @@ -247,7 +245,7 @@ sub usage { $settings{'csmdata'} = $inputopts{csmdata}; } else { my @files = ( "$cfgdir/namelist_files/namelist_defaults.xml", - "$cfgdir/namelist_files/namelist_defaults_${model}_tools.xml", + "$cfgdir/namelist_files/namelist_defaults_tools.xml", "$cfgdir/namelist_files/namelist_defaults_drv.xml", "$cfgdir/namelist_files/namelist_defaults_drydep.xml", ); diff --git a/components/elm/tools/shared/mkmapdata/mkmapdata.sh b/components/elm/tools/shared/mkmapdata/mkmapdata.sh index 26302f79cd00..954b62cdd45c 100755 --- a/components/elm/tools/shared/mkmapdata/mkmapdata.sh +++ b/components/elm/tools/shared/mkmapdata/mkmapdata.sh @@ -13,7 +13,6 @@ # -f Input grid filename # -t Output type, supported values are [regional, global] # -r Output resolution -# -p CLM version to use (clm4_0 or clm4_5) (defaults to clm4_5) # -b use batch mode (not default) # -l list mapping files required (so can use check_input_data to get them) # -d debug usage -- display mkmapdata that will be run but don't execute them @@ -72,9 +71,6 @@ usage() { echo "[-t|--gridtype ]" echo " Model output grid type" echo " supported values are [regional,global], (default is global)" - echo "[-p|--phys ]" - echo " Model version to generate mapping files for. Currently the only" - echo " supported value is clm4_5." echo "[-i|--inputdata-path ]" echo " Full path to root of inputdata directory" echo "[-n|--ntasks ]" @@ -153,7 +149,6 @@ interactive="YES" debug="no" res="default" gridtype="global" -phys="clm4_5" verbose="no" list="no" outgrid="" @@ -187,10 +182,6 @@ while [ $# -gt 0 ]; do gridtype=$2 shift ;; - -p|--phys) - phys=$2 - shift - ;; -i|--inputdata-path) INPUTDATA_PATH=$2 shift @@ -231,7 +222,7 @@ echo "Script to create mapping files required by mksurfdata_map" #---------------------------------------------------------------------- # Set general query command used below -QUERY="$dir/../../../bld/queryDefaultNamelist.pl -silent -namelist clmexp -phys $phys " +QUERY="$dir/../../../bld/queryDefaultNamelist.pl -silent -namelist elmexp " QUERY="$QUERY -justvalue -options sim_year=2000 -csmdata $INPUTDATA_PATH" echo "query command is $QUERY" @@ -306,29 +297,24 @@ fi # Determine all input grid files and output file names #---------------------------------------------------------------------- -if [ "$phys" = "clm4_5" ]; then - grids=( \ - "0.5x0.5_AVHRR" \ - "0.5x0.5_MODIS" \ - "3x3min_LandScan2004" \ - "3x3min_MODIS" \ - "3x3min_USGS" \ - "5x5min_nomask" \ - "5x5min_IGBP-GSDP" \ - "5x5min_ISRIC-WISE" \ - "10x10min_nomask" \ - "10x10min_IGBPmergeICESatGIS" \ - "3x3min_GLOBE-Gardner" \ - "3x3min_GLOBE-Gardner-mergeGIS" \ - "0.9x1.25_GRDC" \ - "360x720cru_cruncep" \ - "1km-merge-10min_HYDRO1K-merge-nomask" \ - "0.5x0.5_GSDTG2000" \ + grids=( \ + "0.5x0.5_AVHRR" \ + "0.5x0.5_MODIS" \ + "3x3min_LandScan2004" \ + "3x3min_MODIS" \ + "3x3min_USGS" \ + "5x5min_nomask" \ + "5x5min_IGBP-GSDP" \ + "5x5min_ISRIC-WISE" \ + "10x10min_nomask" \ + "10x10min_IGBPmergeICESatGIS" \ + "3x3min_GLOBE-Gardner" \ + "3x3min_GLOBE-Gardner-mergeGIS" \ + "0.9x1.25_GRDC" \ + "360x720cru_cruncep" \ + "1km-merge-10min_HYDRO1K-merge-nomask" \ + "0.5x0.5_GSDTG2000" \ ) -else - echo "ERROR: Unknown value for phys: $phys" - exit 1 -fi # Set timestamp for names below CDATE="c"`date +%y%m%d` From 06d48c7d6be1fed8275fa7f5f0e8857d40bb8cac Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Fri, 15 Oct 2021 15:19:59 -0700 Subject: [PATCH 120/301] Moves the ELM tools to one directory up At the begining of E3SM, the land model had `clm4_0` and `clm4_5` models and the common tools were in the `shared` directory. Since, `clm4_0` has been deleted and tools in the `clm4_5` and `shared` are consolidated in a single directory. The relative path to `queryDefaultNamelist.pl` is also updated --- components/elm/tools/{clm4_5 => }/OLMT/README | 0 components/elm/tools/{clm4_5 => }/OLMT/adjust_restart.py | 0 components/elm/tools/{clm4_5 => }/OLMT/case_copy.py | 0 components/elm/tools/{clm4_5 => }/OLMT/ensemble_copy.py | 0 components/elm/tools/{clm4_5 => }/OLMT/ensemble_run.py | 0 components/elm/tools/{clm4_5 => }/OLMT/global_fullrun.py | 0 components/elm/tools/{clm4_5 => }/OLMT/makepointdata.py | 0 components/elm/tools/{clm4_5 => }/OLMT/manage_ensemble.py | 0 components/elm/tools/{clm4_5 => }/OLMT/netcdf4_functions.py | 0 components/elm/tools/{clm4_5 => }/OLMT/netcdf_functions.py | 0 components/elm/tools/{clm4_5 => }/OLMT/plotcase.py | 0 components/elm/tools/{clm4_5 => }/OLMT/pointCLM.py | 0 components/elm/tools/{clm4_5 => }/OLMT/post_process.py | 0 components/elm/tools/{clm4_5 => }/OLMT/runcase.py | 0 components/elm/tools/{clm4_5 => }/OLMT/site_fullrun.py | 0 components/elm/tools/{shared => }/PTCLM/ChangeLog | 0 components/elm/tools/{shared => }/PTCLM/KnownBugs | 0 .../{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt | 0 .../{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt | 0 .../{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt | 0 .../{shared => }/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt | 0 .../PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl | 0 components/elm/tools/{shared => }/PTCLM/PTCLMmkdata | 0 components/elm/tools/{shared => }/PTCLM/PTCLMsublist | 0 components/elm/tools/{shared => }/PTCLM/PTCLMsublist_prog.py | 0 components/elm/tools/{shared => }/PTCLM/README | 0 components/elm/tools/{shared => }/PTCLM/batchque.py | 0 components/elm/tools/{shared => }/PTCLM/buildtools | 0 .../{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM | 0 .../{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm | 0 .../{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds | 0 components/elm/tools/{shared => }/PTCLM/mydatafiles/README | 0 .../elm/tools/{shared => }/PTCLM/mydatafiles/renamemapfiles | 0 .../elm/tools/{shared => }/PTCLM/test/PTCLMtesting_prog.py | 0 components/elm/tools/{shared => }/PTCLM/test/PTCLMtestlist.py | 0 components/elm/tools/{shared => }/PTCLM/test/PTCLMtestlist.xml | 0 components/elm/tools/{shared => }/PTCLM/test/README | 0 .../elm/tools/{shared => }/PTCLM/test/compdirs/copyfiles.csh | 0 .../{shared => }/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm | 0 .../PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/global_US-UMB/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/global_US-UMB/user_nl_clm | 0 .../PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm | 0 .../PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM | 0 .../PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm | 0 .../PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM | 0 .../PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm | 0 .../PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM | 0 .../PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm | 0 .../PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm | 0 .../PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm | 0 .../PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm | 0 .../PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm | 0 .../PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm | 0 .../PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm | 0 .../PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm | 0 .../PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm | 0 .../PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm | 0 .../PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM | 0 .../PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm | 0 .../PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM | 0 .../PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm | 0 .../PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Col/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Col/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm | 0 .../PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm | 0 .../PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds | 0 .../{shared => }/PTCLM/test/compdirs/std_US-UMB/README.PTCLM | 0 .../{shared => }/PTCLM/test/compdirs/std_US-UMB/user_nl_clm | 0 .../{shared => }/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM | 0 .../PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm | 0 .../PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM | 0 .../PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm | 0 .../PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM | 0 .../PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm | 0 .../PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM | 0 .../PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm | 0 .../PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM | 0 .../PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm | 0 .../PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM | 0 .../PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm | 0 .../PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM | 0 .../PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm | 0 .../PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds | 0 .../PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM | 0 .../PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm | 0 .../PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds | 0 components/elm/tools/{shared => }/PTCLM/test/listings/help | 0 components/elm/tools/{shared => }/PTCLM/test/listings/list | 0 components/elm/tools/{shared => }/PTCLM/test/listings/sitelist | 0 components/elm/tools/{shared => }/PTCLM/test/run_PTCLM_tests | 0 components/elm/tools/{clm4_5 => }/interpinic/README | 0 components/elm/tools/{clm4_5 => }/interpinic/addmetadata | 0 .../elm/tools/{clm4_5 => }/interpinic/interpinic.runoptions | 0 components/elm/tools/{clm4_5 => }/interpinic/src/Filepath | 0 components/elm/tools/{clm4_5 => }/interpinic/src/Makefile | 0 .../elm/tools/{clm4_5 => }/interpinic/src/Makefile.common | 0 components/elm/tools/{clm4_5 => }/interpinic/src/Mkdepends | 0 components/elm/tools/{clm4_5 => }/interpinic/src/Srcfiles | 0 components/elm/tools/{clm4_5 => }/interpinic/src/fmain.F90 | 0 components/elm/tools/{clm4_5 => }/interpinic/src/interpinic.F90 | 0 .../elm/tools/{clm4_5 => }/interpinic/src/shr_const_mod.F90 | 0 .../elm/tools/{clm4_5 => }/interpinic/src/shr_infnan_mod.F90 | 0 components/elm/tools/{clm4_5 => }/interpinic/src/shr_isnan.c | 0 components/elm/tools/{clm4_5 => }/interpinic/src/shr_isnan.h | 0 .../elm/tools/{clm4_5 => }/interpinic/src/shr_kind_mod.F90 | 0 .../elm/tools/{clm4_5 => }/interpinic/src/shr_log_mod.F90 | 0 .../elm/tools/{clm4_5 => }/interpinic/src/shr_sys_mod.F90 | 0 components/elm/tools/{clm4_5 => }/land_use_translator/Makefile | 0 .../{clm4_5 => }/land_use_translator/updateannuallanduse_v2.c | 0 components/elm/tools/{shared => }/mkmapdata/README.md | 0 components/elm/tools/{shared => }/mkmapdata/createXMLEntries.pl | 0 components/elm/tools/{shared => }/mkmapdata/mkmapdata.sh | 2 +- components/elm/tools/{shared => }/mkmapdata/mknoocnmap.pl | 0 components/elm/tools/{shared => }/mkmapdata/mkunitymap.ncl | 0 components/elm/tools/{shared => }/mkmapdata/mvNimport.sh | 0 components/elm/tools/{shared => }/mkmapdata/regridbatch.sh | 0 components/elm/tools/{shared => }/mkmapdata/rmdups.ncl | 0 components/elm/tools/{shared => }/mkmapgrids/README | 0 components/elm/tools/{shared => }/mkmapgrids/mkmapgrids.csh | 0 .../elm/tools/{shared => }/mkmapgrids/mkmapgrids.namelist | 0 components/elm/tools/{shared => }/mkmapgrids/mkscripgrid.ncl | 0 .../tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/Filepath | 0 components/elm/tools/{shared => }/mkmapgrids/src/Makefile | 0 .../elm/tools/{shared => }/mkmapgrids/src/Makefile.common | 0 .../tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/Mkdepends | 0 components/elm/tools/{shared => }/mkmapgrids/src/Srcfiles | 0 components/elm/tools/{shared => }/mkmapgrids/src/domainMod.F90 | 0 components/elm/tools/{shared => }/mkmapgrids/src/mkmapgrids.F90 | 0 .../tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/nanMod.F90 | 0 .../{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_file_mod.F90 | 0 .../elm/tools/{shared => }/mkmapgrids/src/shr_kind_mod.F90 | 0 .../{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_log_mod.F90 | 0 .../{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_sys_mod.F90 | 0 components/elm/tools/{shared => }/mkprocdata_map/README | 0 .../elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_all | 0 .../{shared => }/mkprocdata_map/mkprocdata_map_functions.bash | 0 .../elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_in | 0 .../elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_wrap | 0 .../tools/{shared/mkmapgrids => mkprocdata_map}/src/Filepath | 0 components/elm/tools/{shared => }/mkprocdata_map/src/Makefile | 0 .../elm/tools/{shared => }/mkprocdata_map/src/Makefile.common | 0 .../tools/{shared/mkmapgrids => mkprocdata_map}/src/Mkdepends | 0 components/elm/tools/{shared => }/mkprocdata_map/src/Srcfiles | 0 .../elm/tools/{shared => }/mkprocdata_map/src/constMod.F90 | 0 .../elm/tools/{shared => }/mkprocdata_map/src/fileutils.F90 | 0 components/elm/tools/{shared => }/mkprocdata_map/src/fmain.F90 | 0 .../elm/tools/{shared => }/mkprocdata_map/src/gridmapMod.F90 | 0 .../tools/{shared => }/mkprocdata_map/src/mkprocdata_map.F90 | 0 .../tools/{shared/mkmapgrids => mkprocdata_map}/src/nanMod.F90 | 0 .../elm/tools/{shared => }/mkprocdata_map/src/shr_file_mod.F90 | 0 .../mksurfdata_map => mkprocdata_map}/src/shr_kind_mod.F90 | 0 components/elm/tools/{clm4_5 => }/mksurfdata_map/README | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/README.developers | 0 components/elm/tools/{clm4_5 => }/mksurfdata_map/mksurfdata.pl | 0 .../tools/{clm4_5 => }/mksurfdata_map/mksurfdata_map.namelist | 0 .../{shared/mkprocdata_map => mksurfdata_map}/src/Filepath | 0 components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile.common | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile.titan | 0 .../{shared/mkprocdata_map => mksurfdata_map}/src/Mkdepends | 0 components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Srcfiles | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/fileutils.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/src/mkCH4inversionMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkSedMod.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/src/mkVICparamsMod.F90 | 0 .../{clm4_5 => }/mksurfdata_map/src/mkagfirepkmonthMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkchecksMod.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/src/mkdiagnosticsMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkdomainMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkfileMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkgdpMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkglcmecMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkgridmapMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkharvestMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkindexmapMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mklaiMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mklanwatMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkncdio.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkpeatMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkpftMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mksoilMod.F90 | 0 .../{clm4_5 => }/mksurfdata_map/src/mksoilphosphorusMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mksurfdat.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/src/mktopostatsMod.F90 | 0 .../{clm4_5 => }/mksurfdata_map/src/mkurbanparCommonMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkurbanparMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkutilsMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvarctl.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvarpar.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvocefMod.F90 | 0 .../{shared/mkprocdata_map => mksurfdata_map}/src/nanMod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/shr_const_mod.F90 | 0 .../{shared/mkmapgrids => mksurfdata_map}/src/shr_file_mod.F90 | 0 .../mkprocdata_map => mksurfdata_map}/src/shr_kind_mod.F90 | 0 .../{shared/mkmapgrids => mksurfdata_map}/src/shr_log_mod.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/src/shr_string_mod.F90 | 0 .../{shared/mkmapgrids => mksurfdata_map}/src/shr_sys_mod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/src/shr_timer_mod.F90 | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Filepath | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Makefile | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/README | 0 .../elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Srcfiles | 0 .../mksurfdata_map/unit_testers/test_mkchecksMod.F90 | 0 .../mksurfdata_map/unit_testers/test_mkdomainMod.F90 | 0 .../mksurfdata_map/unit_testers/test_mkgridmapMod.F90 | 0 .../mksurfdata_map/unit_testers/test_mkindexmapMod.F90 | 0 .../{clm4_5 => }/mksurfdata_map/unit_testers/test_mkncdio.F90 | 0 .../{clm4_5 => }/mksurfdata_map/unit_testers/test_mkpftMod.F90 | 0 .../mksurfdata_map/unit_testers/test_mksurfdata_map.F90 | 0 .../mksurfdata_map/unit_testers/test_mkurbanparMod.F90 | 0 .../mksurfdata_map/unit_testers/test_mkutilsMod.F90 | 0 .../tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mod.F90 | 0 components/elm/tools/{shared => }/ncl_scripts/README | 0 .../elm/tools/{shared => }/ncl_scripts/README.getregional | 0 components/elm/tools/{shared => }/ncl_scripts/cprnc.ncl | 0 components/elm/tools/{shared => }/ncl_scripts/cprnc.pl | 0 .../elm/tools/{shared => }/ncl_scripts/getco2_historical.ncl | 0 .../elm/tools/{shared => }/ncl_scripts/getregional_datasets.ncl | 0 .../elm/tools/{shared => }/ncl_scripts/getregional_datasets.pl | 0 components/elm/tools/{shared => }/ncl_scripts/sample_inlist | 0 components/elm/tools/{shared => }/ncl_scripts/sample_outlist | 0 .../elm/tools/{clm4_5 => }/refactorTools/associate/README | 0 .../{clm4_5 => }/refactorTools/associate/refactorAssociate.pl | 0 components/elm/tools/{clm4_5 => }/refactorTools/clmType/README | 0 .../tools/{clm4_5 => }/refactorTools/clmType/renameClmType.pl | 0 312 files changed, 1 insertion(+), 1 deletion(-) rename components/elm/tools/{clm4_5 => }/OLMT/README (100%) rename components/elm/tools/{clm4_5 => }/OLMT/adjust_restart.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/case_copy.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/ensemble_copy.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/ensemble_run.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/global_fullrun.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/makepointdata.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/manage_ensemble.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/netcdf4_functions.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/netcdf_functions.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/plotcase.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/pointCLM.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/post_process.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/runcase.py (100%) rename components/elm/tools/{clm4_5 => }/OLMT/site_fullrun.py (100%) rename components/elm/tools/{shared => }/PTCLM/ChangeLog (100%) rename components/elm/tools/{shared => }/PTCLM/KnownBugs (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLMmkdata (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLMsublist (100%) rename components/elm/tools/{shared => }/PTCLM/PTCLMsublist_prog.py (100%) rename components/elm/tools/{shared => }/PTCLM/README (100%) rename components/elm/tools/{shared => }/PTCLM/batchque.py (100%) rename components/elm/tools/{shared => }/PTCLM/buildtools (100%) rename components/elm/tools/{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/mydatafiles/README (100%) rename components/elm/tools/{shared => }/PTCLM/mydatafiles/renamemapfiles (100%) rename components/elm/tools/{shared => }/PTCLM/test/PTCLMtesting_prog.py (100%) rename components/elm/tools/{shared => }/PTCLM/test/PTCLMtestlist.py (100%) rename components/elm/tools/{shared => }/PTCLM/test/PTCLMtestlist.xml (100%) rename components/elm/tools/{shared => }/PTCLM/test/README (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/copyfiles.csh (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/global_US-UMB/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/global_US-UMB/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Col/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Col/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_US-UMB/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_US-UMB/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm (100%) rename components/elm/tools/{shared => }/PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds (100%) rename components/elm/tools/{shared => }/PTCLM/test/listings/help (100%) rename components/elm/tools/{shared => }/PTCLM/test/listings/list (100%) rename components/elm/tools/{shared => }/PTCLM/test/listings/sitelist (100%) rename components/elm/tools/{shared => }/PTCLM/test/run_PTCLM_tests (100%) rename components/elm/tools/{clm4_5 => }/interpinic/README (100%) rename components/elm/tools/{clm4_5 => }/interpinic/addmetadata (100%) rename components/elm/tools/{clm4_5 => }/interpinic/interpinic.runoptions (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/Filepath (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/Makefile (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/Makefile.common (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/Mkdepends (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/Srcfiles (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/fmain.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/interpinic.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_const_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_infnan_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_isnan.c (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_isnan.h (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_kind_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_log_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/interpinic/src/shr_sys_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/land_use_translator/Makefile (100%) rename components/elm/tools/{clm4_5 => }/land_use_translator/updateannuallanduse_v2.c (100%) rename components/elm/tools/{shared => }/mkmapdata/README.md (100%) rename components/elm/tools/{shared => }/mkmapdata/createXMLEntries.pl (100%) rename components/elm/tools/{shared => }/mkmapdata/mkmapdata.sh (99%) rename components/elm/tools/{shared => }/mkmapdata/mknoocnmap.pl (100%) rename components/elm/tools/{shared => }/mkmapdata/mkunitymap.ncl (100%) rename components/elm/tools/{shared => }/mkmapdata/mvNimport.sh (100%) rename components/elm/tools/{shared => }/mkmapdata/regridbatch.sh (100%) rename components/elm/tools/{shared => }/mkmapdata/rmdups.ncl (100%) rename components/elm/tools/{shared => }/mkmapgrids/README (100%) rename components/elm/tools/{shared => }/mkmapgrids/mkmapgrids.csh (100%) rename components/elm/tools/{shared => }/mkmapgrids/mkmapgrids.namelist (100%) rename components/elm/tools/{shared => }/mkmapgrids/mkscripgrid.ncl (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/Filepath (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/Makefile (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/Makefile.common (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/Mkdepends (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/Srcfiles (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/domainMod.F90 (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/mkmapgrids.F90 (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/nanMod.F90 (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_file_mod.F90 (100%) rename components/elm/tools/{shared => }/mkmapgrids/src/shr_kind_mod.F90 (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_log_mod.F90 (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkmapgrids}/src/shr_sys_mod.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/README (100%) rename components/elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_all (100%) rename components/elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_functions.bash (100%) rename components/elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_in (100%) rename components/elm/tools/{shared => }/mkprocdata_map/mkprocdata_map_wrap (100%) rename components/elm/tools/{shared/mkmapgrids => mkprocdata_map}/src/Filepath (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/Makefile (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/Makefile.common (100%) rename components/elm/tools/{shared/mkmapgrids => mkprocdata_map}/src/Mkdepends (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/Srcfiles (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/constMod.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/fileutils.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/fmain.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/gridmapMod.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/mkprocdata_map.F90 (100%) rename components/elm/tools/{shared/mkmapgrids => mkprocdata_map}/src/nanMod.F90 (100%) rename components/elm/tools/{shared => }/mkprocdata_map/src/shr_file_mod.F90 (100%) rename components/elm/tools/{clm4_5/mksurfdata_map => mkprocdata_map}/src/shr_kind_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/README (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/README.developers (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/mksurfdata.pl (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/mksurfdata_map.namelist (100%) rename components/elm/tools/{shared/mkprocdata_map => mksurfdata_map}/src/Filepath (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile.common (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Makefile.titan (100%) rename components/elm/tools/{shared/mkprocdata_map => mksurfdata_map}/src/Mkdepends (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/Srcfiles (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/fileutils.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkCH4inversionMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkSedMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkVICparamsMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkagfirepkmonthMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkchecksMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkdiagnosticsMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkdomainMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkfileMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkgdpMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkglcmecMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkgridmapMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkharvestMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkindexmapMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mklaiMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mklanwatMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkncdio.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkpeatMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkpftMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mksoilMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mksoilphosphorusMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mksurfdat.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mktopostatsMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkurbanparCommonMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkurbanparMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkutilsMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvarctl.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvarpar.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/mkvocefMod.F90 (100%) rename components/elm/tools/{shared/mkprocdata_map => mksurfdata_map}/src/nanMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/shr_const_mod.F90 (100%) rename components/elm/tools/{shared/mkmapgrids => mksurfdata_map}/src/shr_file_mod.F90 (100%) rename components/elm/tools/{shared/mkprocdata_map => mksurfdata_map}/src/shr_kind_mod.F90 (100%) rename components/elm/tools/{shared/mkmapgrids => mksurfdata_map}/src/shr_log_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/shr_string_mod.F90 (100%) rename components/elm/tools/{shared/mkmapgrids => mksurfdata_map}/src/shr_sys_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/src/shr_timer_mod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Filepath (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Makefile (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/README (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/Srcfiles (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkchecksMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkdomainMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkgridmapMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkindexmapMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkncdio.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkpftMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mksurfdata_map.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkurbanparMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mkutilsMod.F90 (100%) rename components/elm/tools/{clm4_5 => }/mksurfdata_map/unit_testers/test_mod.F90 (100%) rename components/elm/tools/{shared => }/ncl_scripts/README (100%) rename components/elm/tools/{shared => }/ncl_scripts/README.getregional (100%) rename components/elm/tools/{shared => }/ncl_scripts/cprnc.ncl (100%) rename components/elm/tools/{shared => }/ncl_scripts/cprnc.pl (100%) rename components/elm/tools/{shared => }/ncl_scripts/getco2_historical.ncl (100%) rename components/elm/tools/{shared => }/ncl_scripts/getregional_datasets.ncl (100%) rename components/elm/tools/{shared => }/ncl_scripts/getregional_datasets.pl (100%) rename components/elm/tools/{shared => }/ncl_scripts/sample_inlist (100%) rename components/elm/tools/{shared => }/ncl_scripts/sample_outlist (100%) rename components/elm/tools/{clm4_5 => }/refactorTools/associate/README (100%) rename components/elm/tools/{clm4_5 => }/refactorTools/associate/refactorAssociate.pl (100%) rename components/elm/tools/{clm4_5 => }/refactorTools/clmType/README (100%) rename components/elm/tools/{clm4_5 => }/refactorTools/clmType/renameClmType.pl (100%) diff --git a/components/elm/tools/clm4_5/OLMT/README b/components/elm/tools/OLMT/README similarity index 100% rename from components/elm/tools/clm4_5/OLMT/README rename to components/elm/tools/OLMT/README diff --git a/components/elm/tools/clm4_5/OLMT/adjust_restart.py b/components/elm/tools/OLMT/adjust_restart.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/adjust_restart.py rename to components/elm/tools/OLMT/adjust_restart.py diff --git a/components/elm/tools/clm4_5/OLMT/case_copy.py b/components/elm/tools/OLMT/case_copy.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/case_copy.py rename to components/elm/tools/OLMT/case_copy.py diff --git a/components/elm/tools/clm4_5/OLMT/ensemble_copy.py b/components/elm/tools/OLMT/ensemble_copy.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/ensemble_copy.py rename to components/elm/tools/OLMT/ensemble_copy.py diff --git a/components/elm/tools/clm4_5/OLMT/ensemble_run.py b/components/elm/tools/OLMT/ensemble_run.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/ensemble_run.py rename to components/elm/tools/OLMT/ensemble_run.py diff --git a/components/elm/tools/clm4_5/OLMT/global_fullrun.py b/components/elm/tools/OLMT/global_fullrun.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/global_fullrun.py rename to components/elm/tools/OLMT/global_fullrun.py diff --git a/components/elm/tools/clm4_5/OLMT/makepointdata.py b/components/elm/tools/OLMT/makepointdata.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/makepointdata.py rename to components/elm/tools/OLMT/makepointdata.py diff --git a/components/elm/tools/clm4_5/OLMT/manage_ensemble.py b/components/elm/tools/OLMT/manage_ensemble.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/manage_ensemble.py rename to components/elm/tools/OLMT/manage_ensemble.py diff --git a/components/elm/tools/clm4_5/OLMT/netcdf4_functions.py b/components/elm/tools/OLMT/netcdf4_functions.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/netcdf4_functions.py rename to components/elm/tools/OLMT/netcdf4_functions.py diff --git a/components/elm/tools/clm4_5/OLMT/netcdf_functions.py b/components/elm/tools/OLMT/netcdf_functions.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/netcdf_functions.py rename to components/elm/tools/OLMT/netcdf_functions.py diff --git a/components/elm/tools/clm4_5/OLMT/plotcase.py b/components/elm/tools/OLMT/plotcase.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/plotcase.py rename to components/elm/tools/OLMT/plotcase.py diff --git a/components/elm/tools/clm4_5/OLMT/pointCLM.py b/components/elm/tools/OLMT/pointCLM.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/pointCLM.py rename to components/elm/tools/OLMT/pointCLM.py diff --git a/components/elm/tools/clm4_5/OLMT/post_process.py b/components/elm/tools/OLMT/post_process.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/post_process.py rename to components/elm/tools/OLMT/post_process.py diff --git a/components/elm/tools/clm4_5/OLMT/runcase.py b/components/elm/tools/OLMT/runcase.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/runcase.py rename to components/elm/tools/OLMT/runcase.py diff --git a/components/elm/tools/clm4_5/OLMT/site_fullrun.py b/components/elm/tools/OLMT/site_fullrun.py similarity index 100% rename from components/elm/tools/clm4_5/OLMT/site_fullrun.py rename to components/elm/tools/OLMT/site_fullrun.py diff --git a/components/elm/tools/shared/PTCLM/ChangeLog b/components/elm/tools/PTCLM/ChangeLog similarity index 100% rename from components/elm/tools/shared/PTCLM/ChangeLog rename to components/elm/tools/PTCLM/ChangeLog diff --git a/components/elm/tools/shared/PTCLM/KnownBugs b/components/elm/tools/PTCLM/KnownBugs similarity index 100% rename from components/elm/tools/shared/PTCLM/KnownBugs rename to components/elm/tools/PTCLM/KnownBugs diff --git a/components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt b/components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt rename to components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_pftdata.txt diff --git a/components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt b/components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt rename to components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_sitedata.txt diff --git a/components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt b/components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt rename to components/elm/tools/PTCLM/PTCLM_sitedata/PTCLMDATA_soildata.txt diff --git a/components/elm/tools/shared/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt b/components/elm/tools/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt rename to components/elm/tools/PTCLM/PTCLM_sitedata/US-Ha1_dynpftdata.txt diff --git a/components/elm/tools/shared/PTCLM/PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl b/components/elm/tools/PTCLM/PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl rename to components/elm/tools/PTCLM/PTCLM_sitedata/cnvrt_trnsyrs2_landuse_timeseries_txtfile.pl diff --git a/components/elm/tools/shared/PTCLM/PTCLMmkdata b/components/elm/tools/PTCLM/PTCLMmkdata similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLMmkdata rename to components/elm/tools/PTCLM/PTCLMmkdata diff --git a/components/elm/tools/shared/PTCLM/PTCLMsublist b/components/elm/tools/PTCLM/PTCLMsublist similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLMsublist rename to components/elm/tools/PTCLM/PTCLMsublist diff --git a/components/elm/tools/shared/PTCLM/PTCLMsublist_prog.py b/components/elm/tools/PTCLM/PTCLMsublist_prog.py similarity index 100% rename from components/elm/tools/shared/PTCLM/PTCLMsublist_prog.py rename to components/elm/tools/PTCLM/PTCLMsublist_prog.py diff --git a/components/elm/tools/shared/PTCLM/README b/components/elm/tools/PTCLM/README similarity index 100% rename from components/elm/tools/shared/PTCLM/README rename to components/elm/tools/PTCLM/README diff --git a/components/elm/tools/shared/PTCLM/batchque.py b/components/elm/tools/PTCLM/batchque.py similarity index 100% rename from components/elm/tools/shared/PTCLM/batchque.py rename to components/elm/tools/PTCLM/batchque.py diff --git a/components/elm/tools/shared/PTCLM/buildtools b/components/elm/tools/PTCLM/buildtools similarity index 100% rename from components/elm/tools/shared/PTCLM/buildtools rename to components/elm/tools/PTCLM/buildtools diff --git a/components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM b/components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM rename to components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm b/components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm rename to components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds b/components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds rename to components/elm/tools/PTCLM/mydatafiles/1x1pt_US-UMB/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/mydatafiles/README b/components/elm/tools/PTCLM/mydatafiles/README similarity index 100% rename from components/elm/tools/shared/PTCLM/mydatafiles/README rename to components/elm/tools/PTCLM/mydatafiles/README diff --git a/components/elm/tools/shared/PTCLM/mydatafiles/renamemapfiles b/components/elm/tools/PTCLM/mydatafiles/renamemapfiles similarity index 100% rename from components/elm/tools/shared/PTCLM/mydatafiles/renamemapfiles rename to components/elm/tools/PTCLM/mydatafiles/renamemapfiles diff --git a/components/elm/tools/shared/PTCLM/test/PTCLMtesting_prog.py b/components/elm/tools/PTCLM/test/PTCLMtesting_prog.py similarity index 100% rename from components/elm/tools/shared/PTCLM/test/PTCLMtesting_prog.py rename to components/elm/tools/PTCLM/test/PTCLMtesting_prog.py diff --git a/components/elm/tools/shared/PTCLM/test/PTCLMtestlist.py b/components/elm/tools/PTCLM/test/PTCLMtestlist.py similarity index 100% rename from components/elm/tools/shared/PTCLM/test/PTCLMtestlist.py rename to components/elm/tools/PTCLM/test/PTCLMtestlist.py diff --git a/components/elm/tools/shared/PTCLM/test/PTCLMtestlist.xml b/components/elm/tools/PTCLM/test/PTCLMtestlist.xml similarity index 100% rename from components/elm/tools/shared/PTCLM/test/PTCLMtestlist.xml rename to components/elm/tools/PTCLM/test/PTCLMtestlist.xml diff --git a/components/elm/tools/shared/PTCLM/test/README b/components/elm/tools/PTCLM/test/README similarity index 100% rename from components/elm/tools/shared/PTCLM/test/README rename to components/elm/tools/PTCLM/test/README diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/copyfiles.csh b/components/elm/tools/PTCLM/test/compdirs/copyfiles.csh similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/copyfiles.csh rename to components/elm/tools/PTCLM/test/compdirs/copyfiles.csh diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/crop_US-IB1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts2/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/cyc_TS-Ts3/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/global_US-UMB/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/global_US-UMB/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/global_US-UMB/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/global_US-UMB/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/global_US-UMB/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/noopt_US-CHATS/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Let/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_CA-Man/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/notowyrs_US-WCr/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ca1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Obs/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/pftgrd_CA-Ojp/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/rcp26_US-Dk2/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/rcp45_US-Dk3/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/rcp6_US-IB1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/rcp85_US-Me4/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Ojp/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/soigrd_CA-Qfo/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_1850_US-Me2/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_2000_US-MOz/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_BE-Vie/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_BR-Sa3/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_DE-Tha/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_ES-ES1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Hyy/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_FL-Kaa/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_IT-Col/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Col/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_IT-Col/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Col/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Col/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_IT-Cpz/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_LBA-Cax/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_LTER-Sev/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_RF-Bra/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_US-UMB/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_US-UMB/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_US-UMB/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_US-UMB/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_US-UMB/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-Brw/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-FPe/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm40_US-NR1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-ARM/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Ha1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_clm45_US-Var/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Bo1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-Ho1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_quiet_US-MMS/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/std_soigrd_BR-Sa1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/stdurbpt_pftgrd_CA-Oas/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/trans_20th_US-Ne3/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM b/components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM rename to components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/README.PTCLM diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm b/components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm rename to components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/user_nl_clm diff --git a/components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds b/components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds similarity index 100% rename from components/elm/tools/shared/PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds rename to components/elm/tools/PTCLM/test/compdirs/trans_clm45_US-Ha1/xmlchange_cmnds diff --git a/components/elm/tools/shared/PTCLM/test/listings/help b/components/elm/tools/PTCLM/test/listings/help similarity index 100% rename from components/elm/tools/shared/PTCLM/test/listings/help rename to components/elm/tools/PTCLM/test/listings/help diff --git a/components/elm/tools/shared/PTCLM/test/listings/list b/components/elm/tools/PTCLM/test/listings/list similarity index 100% rename from components/elm/tools/shared/PTCLM/test/listings/list rename to components/elm/tools/PTCLM/test/listings/list diff --git a/components/elm/tools/shared/PTCLM/test/listings/sitelist b/components/elm/tools/PTCLM/test/listings/sitelist similarity index 100% rename from components/elm/tools/shared/PTCLM/test/listings/sitelist rename to components/elm/tools/PTCLM/test/listings/sitelist diff --git a/components/elm/tools/shared/PTCLM/test/run_PTCLM_tests b/components/elm/tools/PTCLM/test/run_PTCLM_tests similarity index 100% rename from components/elm/tools/shared/PTCLM/test/run_PTCLM_tests rename to components/elm/tools/PTCLM/test/run_PTCLM_tests diff --git a/components/elm/tools/clm4_5/interpinic/README b/components/elm/tools/interpinic/README similarity index 100% rename from components/elm/tools/clm4_5/interpinic/README rename to components/elm/tools/interpinic/README diff --git a/components/elm/tools/clm4_5/interpinic/addmetadata b/components/elm/tools/interpinic/addmetadata similarity index 100% rename from components/elm/tools/clm4_5/interpinic/addmetadata rename to components/elm/tools/interpinic/addmetadata diff --git a/components/elm/tools/clm4_5/interpinic/interpinic.runoptions b/components/elm/tools/interpinic/interpinic.runoptions similarity index 100% rename from components/elm/tools/clm4_5/interpinic/interpinic.runoptions rename to components/elm/tools/interpinic/interpinic.runoptions diff --git a/components/elm/tools/clm4_5/interpinic/src/Filepath b/components/elm/tools/interpinic/src/Filepath similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/Filepath rename to components/elm/tools/interpinic/src/Filepath diff --git a/components/elm/tools/clm4_5/interpinic/src/Makefile b/components/elm/tools/interpinic/src/Makefile similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/Makefile rename to components/elm/tools/interpinic/src/Makefile diff --git a/components/elm/tools/clm4_5/interpinic/src/Makefile.common b/components/elm/tools/interpinic/src/Makefile.common similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/Makefile.common rename to components/elm/tools/interpinic/src/Makefile.common diff --git a/components/elm/tools/clm4_5/interpinic/src/Mkdepends b/components/elm/tools/interpinic/src/Mkdepends similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/Mkdepends rename to components/elm/tools/interpinic/src/Mkdepends diff --git a/components/elm/tools/clm4_5/interpinic/src/Srcfiles b/components/elm/tools/interpinic/src/Srcfiles similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/Srcfiles rename to components/elm/tools/interpinic/src/Srcfiles diff --git a/components/elm/tools/clm4_5/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/fmain.F90 rename to components/elm/tools/interpinic/src/fmain.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/interpinic.F90 rename to components/elm/tools/interpinic/src/interpinic.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_const_mod.F90 b/components/elm/tools/interpinic/src/shr_const_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_const_mod.F90 rename to components/elm/tools/interpinic/src/shr_const_mod.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_infnan_mod.F90 rename to components/elm/tools/interpinic/src/shr_infnan_mod.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_isnan.c b/components/elm/tools/interpinic/src/shr_isnan.c similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_isnan.c rename to components/elm/tools/interpinic/src/shr_isnan.c diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_isnan.h b/components/elm/tools/interpinic/src/shr_isnan.h similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_isnan.h rename to components/elm/tools/interpinic/src/shr_isnan.h diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_kind_mod.F90 b/components/elm/tools/interpinic/src/shr_kind_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_kind_mod.F90 rename to components/elm/tools/interpinic/src/shr_kind_mod.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_log_mod.F90 b/components/elm/tools/interpinic/src/shr_log_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_log_mod.F90 rename to components/elm/tools/interpinic/src/shr_log_mod.F90 diff --git a/components/elm/tools/clm4_5/interpinic/src/shr_sys_mod.F90 b/components/elm/tools/interpinic/src/shr_sys_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/interpinic/src/shr_sys_mod.F90 rename to components/elm/tools/interpinic/src/shr_sys_mod.F90 diff --git a/components/elm/tools/clm4_5/land_use_translator/Makefile b/components/elm/tools/land_use_translator/Makefile similarity index 100% rename from components/elm/tools/clm4_5/land_use_translator/Makefile rename to components/elm/tools/land_use_translator/Makefile diff --git a/components/elm/tools/clm4_5/land_use_translator/updateannuallanduse_v2.c b/components/elm/tools/land_use_translator/updateannuallanduse_v2.c similarity index 100% rename from components/elm/tools/clm4_5/land_use_translator/updateannuallanduse_v2.c rename to components/elm/tools/land_use_translator/updateannuallanduse_v2.c diff --git a/components/elm/tools/shared/mkmapdata/README.md b/components/elm/tools/mkmapdata/README.md similarity index 100% rename from components/elm/tools/shared/mkmapdata/README.md rename to components/elm/tools/mkmapdata/README.md diff --git a/components/elm/tools/shared/mkmapdata/createXMLEntries.pl b/components/elm/tools/mkmapdata/createXMLEntries.pl similarity index 100% rename from components/elm/tools/shared/mkmapdata/createXMLEntries.pl rename to components/elm/tools/mkmapdata/createXMLEntries.pl diff --git a/components/elm/tools/shared/mkmapdata/mkmapdata.sh b/components/elm/tools/mkmapdata/mkmapdata.sh similarity index 99% rename from components/elm/tools/shared/mkmapdata/mkmapdata.sh rename to components/elm/tools/mkmapdata/mkmapdata.sh index 954b62cdd45c..d87538ef212d 100755 --- a/components/elm/tools/shared/mkmapdata/mkmapdata.sh +++ b/components/elm/tools/mkmapdata/mkmapdata.sh @@ -222,7 +222,7 @@ echo "Script to create mapping files required by mksurfdata_map" #---------------------------------------------------------------------- # Set general query command used below -QUERY="$dir/../../../bld/queryDefaultNamelist.pl -silent -namelist elmexp " +QUERY="$dir/../../bld/queryDefaultNamelist.pl -silent -namelist elmexp " QUERY="$QUERY -justvalue -options sim_year=2000 -csmdata $INPUTDATA_PATH" echo "query command is $QUERY" diff --git a/components/elm/tools/shared/mkmapdata/mknoocnmap.pl b/components/elm/tools/mkmapdata/mknoocnmap.pl similarity index 100% rename from components/elm/tools/shared/mkmapdata/mknoocnmap.pl rename to components/elm/tools/mkmapdata/mknoocnmap.pl diff --git a/components/elm/tools/shared/mkmapdata/mkunitymap.ncl b/components/elm/tools/mkmapdata/mkunitymap.ncl similarity index 100% rename from components/elm/tools/shared/mkmapdata/mkunitymap.ncl rename to components/elm/tools/mkmapdata/mkunitymap.ncl diff --git a/components/elm/tools/shared/mkmapdata/mvNimport.sh b/components/elm/tools/mkmapdata/mvNimport.sh similarity index 100% rename from components/elm/tools/shared/mkmapdata/mvNimport.sh rename to components/elm/tools/mkmapdata/mvNimport.sh diff --git a/components/elm/tools/shared/mkmapdata/regridbatch.sh b/components/elm/tools/mkmapdata/regridbatch.sh similarity index 100% rename from components/elm/tools/shared/mkmapdata/regridbatch.sh rename to components/elm/tools/mkmapdata/regridbatch.sh diff --git a/components/elm/tools/shared/mkmapdata/rmdups.ncl b/components/elm/tools/mkmapdata/rmdups.ncl similarity index 100% rename from components/elm/tools/shared/mkmapdata/rmdups.ncl rename to components/elm/tools/mkmapdata/rmdups.ncl diff --git a/components/elm/tools/shared/mkmapgrids/README b/components/elm/tools/mkmapgrids/README similarity index 100% rename from components/elm/tools/shared/mkmapgrids/README rename to components/elm/tools/mkmapgrids/README diff --git a/components/elm/tools/shared/mkmapgrids/mkmapgrids.csh b/components/elm/tools/mkmapgrids/mkmapgrids.csh similarity index 100% rename from components/elm/tools/shared/mkmapgrids/mkmapgrids.csh rename to components/elm/tools/mkmapgrids/mkmapgrids.csh diff --git a/components/elm/tools/shared/mkmapgrids/mkmapgrids.namelist b/components/elm/tools/mkmapgrids/mkmapgrids.namelist similarity index 100% rename from components/elm/tools/shared/mkmapgrids/mkmapgrids.namelist rename to components/elm/tools/mkmapgrids/mkmapgrids.namelist diff --git a/components/elm/tools/shared/mkmapgrids/mkscripgrid.ncl b/components/elm/tools/mkmapgrids/mkscripgrid.ncl similarity index 100% rename from components/elm/tools/shared/mkmapgrids/mkscripgrid.ncl rename to components/elm/tools/mkmapgrids/mkscripgrid.ncl diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Filepath b/components/elm/tools/mkmapgrids/src/Filepath similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Filepath rename to components/elm/tools/mkmapgrids/src/Filepath diff --git a/components/elm/tools/shared/mkmapgrids/src/Makefile b/components/elm/tools/mkmapgrids/src/Makefile similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/Makefile rename to components/elm/tools/mkmapgrids/src/Makefile diff --git a/components/elm/tools/shared/mkmapgrids/src/Makefile.common b/components/elm/tools/mkmapgrids/src/Makefile.common similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/Makefile.common rename to components/elm/tools/mkmapgrids/src/Makefile.common diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Mkdepends b/components/elm/tools/mkmapgrids/src/Mkdepends similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Mkdepends rename to components/elm/tools/mkmapgrids/src/Mkdepends diff --git a/components/elm/tools/shared/mkmapgrids/src/Srcfiles b/components/elm/tools/mkmapgrids/src/Srcfiles similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/Srcfiles rename to components/elm/tools/mkmapgrids/src/Srcfiles diff --git a/components/elm/tools/shared/mkmapgrids/src/domainMod.F90 b/components/elm/tools/mkmapgrids/src/domainMod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/domainMod.F90 rename to components/elm/tools/mkmapgrids/src/domainMod.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/mkmapgrids.F90 b/components/elm/tools/mkmapgrids/src/mkmapgrids.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/mkmapgrids.F90 rename to components/elm/tools/mkmapgrids/src/mkmapgrids.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/nanMod.F90 b/components/elm/tools/mkmapgrids/src/nanMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/nanMod.F90 rename to components/elm/tools/mkmapgrids/src/nanMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_file_mod.F90 b/components/elm/tools/mkmapgrids/src/shr_file_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_file_mod.F90 rename to components/elm/tools/mkmapgrids/src/shr_file_mod.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/shr_kind_mod.F90 b/components/elm/tools/mkmapgrids/src/shr_kind_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/shr_kind_mod.F90 rename to components/elm/tools/mkmapgrids/src/shr_kind_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_log_mod.F90 b/components/elm/tools/mkmapgrids/src/shr_log_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_log_mod.F90 rename to components/elm/tools/mkmapgrids/src/shr_log_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_sys_mod.F90 b/components/elm/tools/mkmapgrids/src/shr_sys_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_sys_mod.F90 rename to components/elm/tools/mkmapgrids/src/shr_sys_mod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/README b/components/elm/tools/mkprocdata_map/README similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/README rename to components/elm/tools/mkprocdata_map/README diff --git a/components/elm/tools/shared/mkprocdata_map/mkprocdata_map_all b/components/elm/tools/mkprocdata_map/mkprocdata_map_all similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/mkprocdata_map_all rename to components/elm/tools/mkprocdata_map/mkprocdata_map_all diff --git a/components/elm/tools/shared/mkprocdata_map/mkprocdata_map_functions.bash b/components/elm/tools/mkprocdata_map/mkprocdata_map_functions.bash similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/mkprocdata_map_functions.bash rename to components/elm/tools/mkprocdata_map/mkprocdata_map_functions.bash diff --git a/components/elm/tools/shared/mkprocdata_map/mkprocdata_map_in b/components/elm/tools/mkprocdata_map/mkprocdata_map_in similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/mkprocdata_map_in rename to components/elm/tools/mkprocdata_map/mkprocdata_map_in diff --git a/components/elm/tools/shared/mkprocdata_map/mkprocdata_map_wrap b/components/elm/tools/mkprocdata_map/mkprocdata_map_wrap similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/mkprocdata_map_wrap rename to components/elm/tools/mkprocdata_map/mkprocdata_map_wrap diff --git a/components/elm/tools/shared/mkmapgrids/src/Filepath b/components/elm/tools/mkprocdata_map/src/Filepath similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/Filepath rename to components/elm/tools/mkprocdata_map/src/Filepath diff --git a/components/elm/tools/shared/mkprocdata_map/src/Makefile b/components/elm/tools/mkprocdata_map/src/Makefile similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/Makefile rename to components/elm/tools/mkprocdata_map/src/Makefile diff --git a/components/elm/tools/shared/mkprocdata_map/src/Makefile.common b/components/elm/tools/mkprocdata_map/src/Makefile.common similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/Makefile.common rename to components/elm/tools/mkprocdata_map/src/Makefile.common diff --git a/components/elm/tools/shared/mkmapgrids/src/Mkdepends b/components/elm/tools/mkprocdata_map/src/Mkdepends similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/Mkdepends rename to components/elm/tools/mkprocdata_map/src/Mkdepends diff --git a/components/elm/tools/shared/mkprocdata_map/src/Srcfiles b/components/elm/tools/mkprocdata_map/src/Srcfiles similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/Srcfiles rename to components/elm/tools/mkprocdata_map/src/Srcfiles diff --git a/components/elm/tools/shared/mkprocdata_map/src/constMod.F90 b/components/elm/tools/mkprocdata_map/src/constMod.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/constMod.F90 rename to components/elm/tools/mkprocdata_map/src/constMod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/fileutils.F90 b/components/elm/tools/mkprocdata_map/src/fileutils.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/fileutils.F90 rename to components/elm/tools/mkprocdata_map/src/fileutils.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/fmain.F90 b/components/elm/tools/mkprocdata_map/src/fmain.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/fmain.F90 rename to components/elm/tools/mkprocdata_map/src/fmain.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/gridmapMod.F90 b/components/elm/tools/mkprocdata_map/src/gridmapMod.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/gridmapMod.F90 rename to components/elm/tools/mkprocdata_map/src/gridmapMod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/mkprocdata_map.F90 b/components/elm/tools/mkprocdata_map/src/mkprocdata_map.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/mkprocdata_map.F90 rename to components/elm/tools/mkprocdata_map/src/mkprocdata_map.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/nanMod.F90 b/components/elm/tools/mkprocdata_map/src/nanMod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/nanMod.F90 rename to components/elm/tools/mkprocdata_map/src/nanMod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/shr_file_mod.F90 b/components/elm/tools/mkprocdata_map/src/shr_file_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/shr_file_mod.F90 rename to components/elm/tools/mkprocdata_map/src/shr_file_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_kind_mod.F90 b/components/elm/tools/mkprocdata_map/src/shr_kind_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_kind_mod.F90 rename to components/elm/tools/mkprocdata_map/src/shr_kind_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/README b/components/elm/tools/mksurfdata_map/README similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/README rename to components/elm/tools/mksurfdata_map/README diff --git a/components/elm/tools/clm4_5/mksurfdata_map/README.developers b/components/elm/tools/mksurfdata_map/README.developers similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/README.developers rename to components/elm/tools/mksurfdata_map/README.developers diff --git a/components/elm/tools/clm4_5/mksurfdata_map/mksurfdata.pl b/components/elm/tools/mksurfdata_map/mksurfdata.pl similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/mksurfdata.pl rename to components/elm/tools/mksurfdata_map/mksurfdata.pl diff --git a/components/elm/tools/clm4_5/mksurfdata_map/mksurfdata_map.namelist b/components/elm/tools/mksurfdata_map/mksurfdata_map.namelist similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/mksurfdata_map.namelist rename to components/elm/tools/mksurfdata_map/mksurfdata_map.namelist diff --git a/components/elm/tools/shared/mkprocdata_map/src/Filepath b/components/elm/tools/mksurfdata_map/src/Filepath similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/Filepath rename to components/elm/tools/mksurfdata_map/src/Filepath diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Makefile b/components/elm/tools/mksurfdata_map/src/Makefile similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Makefile rename to components/elm/tools/mksurfdata_map/src/Makefile diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Makefile.common b/components/elm/tools/mksurfdata_map/src/Makefile.common similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Makefile.common rename to components/elm/tools/mksurfdata_map/src/Makefile.common diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Makefile.titan b/components/elm/tools/mksurfdata_map/src/Makefile.titan similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Makefile.titan rename to components/elm/tools/mksurfdata_map/src/Makefile.titan diff --git a/components/elm/tools/shared/mkprocdata_map/src/Mkdepends b/components/elm/tools/mksurfdata_map/src/Mkdepends similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/Mkdepends rename to components/elm/tools/mksurfdata_map/src/Mkdepends diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/Srcfiles b/components/elm/tools/mksurfdata_map/src/Srcfiles similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/Srcfiles rename to components/elm/tools/mksurfdata_map/src/Srcfiles diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/fileutils.F90 b/components/elm/tools/mksurfdata_map/src/fileutils.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/fileutils.F90 rename to components/elm/tools/mksurfdata_map/src/fileutils.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkCH4inversionMod.F90 b/components/elm/tools/mksurfdata_map/src/mkCH4inversionMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkCH4inversionMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkCH4inversionMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkSedMod.F90 b/components/elm/tools/mksurfdata_map/src/mkSedMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkSedMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkSedMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkVICparamsMod.F90 b/components/elm/tools/mksurfdata_map/src/mkVICparamsMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkVICparamsMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkVICparamsMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkagfirepkmonthMod.F90 b/components/elm/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkagfirepkmonthMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkchecksMod.F90 b/components/elm/tools/mksurfdata_map/src/mkchecksMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkchecksMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkchecksMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkdiagnosticsMod.F90 b/components/elm/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkdiagnosticsMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkdomainMod.F90 b/components/elm/tools/mksurfdata_map/src/mkdomainMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkdomainMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkdomainMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkfileMod.F90 b/components/elm/tools/mksurfdata_map/src/mkfileMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkfileMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkfileMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkgdpMod.F90 b/components/elm/tools/mksurfdata_map/src/mkgdpMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkgdpMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkgdpMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkglcmecMod.F90 b/components/elm/tools/mksurfdata_map/src/mkglcmecMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkglcmecMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkglcmecMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkgridmapMod.F90 b/components/elm/tools/mksurfdata_map/src/mkgridmapMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkgridmapMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkgridmapMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkharvestMod.F90 b/components/elm/tools/mksurfdata_map/src/mkharvestMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkharvestMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkharvestMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkindexmapMod.F90 b/components/elm/tools/mksurfdata_map/src/mkindexmapMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkindexmapMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkindexmapMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mklaiMod.F90 b/components/elm/tools/mksurfdata_map/src/mklaiMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mklaiMod.F90 rename to components/elm/tools/mksurfdata_map/src/mklaiMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mklanwatMod.F90 b/components/elm/tools/mksurfdata_map/src/mklanwatMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mklanwatMod.F90 rename to components/elm/tools/mksurfdata_map/src/mklanwatMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkncdio.F90 b/components/elm/tools/mksurfdata_map/src/mkncdio.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkncdio.F90 rename to components/elm/tools/mksurfdata_map/src/mkncdio.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkpeatMod.F90 b/components/elm/tools/mksurfdata_map/src/mkpeatMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkpeatMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkpeatMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkpftMod.F90 b/components/elm/tools/mksurfdata_map/src/mkpftMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkpftMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkpftMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mksoilMod.F90 b/components/elm/tools/mksurfdata_map/src/mksoilMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mksoilMod.F90 rename to components/elm/tools/mksurfdata_map/src/mksoilMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mksoilphosphorusMod.F90 b/components/elm/tools/mksurfdata_map/src/mksoilphosphorusMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mksoilphosphorusMod.F90 rename to components/elm/tools/mksurfdata_map/src/mksoilphosphorusMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mksurfdat.F90 b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mksurfdat.F90 rename to components/elm/tools/mksurfdata_map/src/mksurfdat.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mktopostatsMod.F90 b/components/elm/tools/mksurfdata_map/src/mktopostatsMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mktopostatsMod.F90 rename to components/elm/tools/mksurfdata_map/src/mktopostatsMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkurbanparCommonMod.F90 b/components/elm/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkurbanparCommonMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkurbanparMod.F90 b/components/elm/tools/mksurfdata_map/src/mkurbanparMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkurbanparMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkurbanparMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkutilsMod.F90 b/components/elm/tools/mksurfdata_map/src/mkutilsMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkutilsMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkutilsMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkvarctl.F90 b/components/elm/tools/mksurfdata_map/src/mkvarctl.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkvarctl.F90 rename to components/elm/tools/mksurfdata_map/src/mkvarctl.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkvarpar.F90 b/components/elm/tools/mksurfdata_map/src/mkvarpar.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkvarpar.F90 rename to components/elm/tools/mksurfdata_map/src/mkvarpar.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/mkvocefMod.F90 b/components/elm/tools/mksurfdata_map/src/mkvocefMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/mkvocefMod.F90 rename to components/elm/tools/mksurfdata_map/src/mkvocefMod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/nanMod.F90 b/components/elm/tools/mksurfdata_map/src/nanMod.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/nanMod.F90 rename to components/elm/tools/mksurfdata_map/src/nanMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_const_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_const_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_const_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_const_mod.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/shr_file_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_file_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/shr_file_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_file_mod.F90 diff --git a/components/elm/tools/shared/mkprocdata_map/src/shr_kind_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_kind_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkprocdata_map/src/shr_kind_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_kind_mod.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/shr_log_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_log_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/shr_log_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_log_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_string_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_string_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_string_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_string_mod.F90 diff --git a/components/elm/tools/shared/mkmapgrids/src/shr_sys_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_sys_mod.F90 similarity index 100% rename from components/elm/tools/shared/mkmapgrids/src/shr_sys_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_sys_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/src/shr_timer_mod.F90 b/components/elm/tools/mksurfdata_map/src/shr_timer_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/src/shr_timer_mod.F90 rename to components/elm/tools/mksurfdata_map/src/shr_timer_mod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Filepath b/components/elm/tools/mksurfdata_map/unit_testers/Filepath similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Filepath rename to components/elm/tools/mksurfdata_map/unit_testers/Filepath diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Makefile b/components/elm/tools/mksurfdata_map/unit_testers/Makefile similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Makefile rename to components/elm/tools/mksurfdata_map/unit_testers/Makefile diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/README b/components/elm/tools/mksurfdata_map/unit_testers/README similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/README rename to components/elm/tools/mksurfdata_map/unit_testers/README diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Srcfiles b/components/elm/tools/mksurfdata_map/unit_testers/Srcfiles similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/Srcfiles rename to components/elm/tools/mksurfdata_map/unit_testers/Srcfiles diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkchecksMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkchecksMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkchecksMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkchecksMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkdomainMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkdomainMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkdomainMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkdomainMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkgridmapMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkgridmapMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkgridmapMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkgridmapMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkindexmapMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkindexmapMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkindexmapMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkindexmapMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkncdio.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkncdio.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkncdio.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkncdio.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkpftMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkpftMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkpftMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkpftMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mksurfdata_map.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mksurfdata_map.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mksurfdata_map.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mksurfdata_map.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkurbanparMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkurbanparMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkurbanparMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkurbanparMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkutilsMod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mkutilsMod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mkutilsMod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mkutilsMod.F90 diff --git a/components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mod.F90 b/components/elm/tools/mksurfdata_map/unit_testers/test_mod.F90 similarity index 100% rename from components/elm/tools/clm4_5/mksurfdata_map/unit_testers/test_mod.F90 rename to components/elm/tools/mksurfdata_map/unit_testers/test_mod.F90 diff --git a/components/elm/tools/shared/ncl_scripts/README b/components/elm/tools/ncl_scripts/README similarity index 100% rename from components/elm/tools/shared/ncl_scripts/README rename to components/elm/tools/ncl_scripts/README diff --git a/components/elm/tools/shared/ncl_scripts/README.getregional b/components/elm/tools/ncl_scripts/README.getregional similarity index 100% rename from components/elm/tools/shared/ncl_scripts/README.getregional rename to components/elm/tools/ncl_scripts/README.getregional diff --git a/components/elm/tools/shared/ncl_scripts/cprnc.ncl b/components/elm/tools/ncl_scripts/cprnc.ncl similarity index 100% rename from components/elm/tools/shared/ncl_scripts/cprnc.ncl rename to components/elm/tools/ncl_scripts/cprnc.ncl diff --git a/components/elm/tools/shared/ncl_scripts/cprnc.pl b/components/elm/tools/ncl_scripts/cprnc.pl similarity index 100% rename from components/elm/tools/shared/ncl_scripts/cprnc.pl rename to components/elm/tools/ncl_scripts/cprnc.pl diff --git a/components/elm/tools/shared/ncl_scripts/getco2_historical.ncl b/components/elm/tools/ncl_scripts/getco2_historical.ncl similarity index 100% rename from components/elm/tools/shared/ncl_scripts/getco2_historical.ncl rename to components/elm/tools/ncl_scripts/getco2_historical.ncl diff --git a/components/elm/tools/shared/ncl_scripts/getregional_datasets.ncl b/components/elm/tools/ncl_scripts/getregional_datasets.ncl similarity index 100% rename from components/elm/tools/shared/ncl_scripts/getregional_datasets.ncl rename to components/elm/tools/ncl_scripts/getregional_datasets.ncl diff --git a/components/elm/tools/shared/ncl_scripts/getregional_datasets.pl b/components/elm/tools/ncl_scripts/getregional_datasets.pl similarity index 100% rename from components/elm/tools/shared/ncl_scripts/getregional_datasets.pl rename to components/elm/tools/ncl_scripts/getregional_datasets.pl diff --git a/components/elm/tools/shared/ncl_scripts/sample_inlist b/components/elm/tools/ncl_scripts/sample_inlist similarity index 100% rename from components/elm/tools/shared/ncl_scripts/sample_inlist rename to components/elm/tools/ncl_scripts/sample_inlist diff --git a/components/elm/tools/shared/ncl_scripts/sample_outlist b/components/elm/tools/ncl_scripts/sample_outlist similarity index 100% rename from components/elm/tools/shared/ncl_scripts/sample_outlist rename to components/elm/tools/ncl_scripts/sample_outlist diff --git a/components/elm/tools/clm4_5/refactorTools/associate/README b/components/elm/tools/refactorTools/associate/README similarity index 100% rename from components/elm/tools/clm4_5/refactorTools/associate/README rename to components/elm/tools/refactorTools/associate/README diff --git a/components/elm/tools/clm4_5/refactorTools/associate/refactorAssociate.pl b/components/elm/tools/refactorTools/associate/refactorAssociate.pl similarity index 100% rename from components/elm/tools/clm4_5/refactorTools/associate/refactorAssociate.pl rename to components/elm/tools/refactorTools/associate/refactorAssociate.pl diff --git a/components/elm/tools/clm4_5/refactorTools/clmType/README b/components/elm/tools/refactorTools/clmType/README similarity index 100% rename from components/elm/tools/clm4_5/refactorTools/clmType/README rename to components/elm/tools/refactorTools/clmType/README diff --git a/components/elm/tools/clm4_5/refactorTools/clmType/renameClmType.pl b/components/elm/tools/refactorTools/clmType/renameClmType.pl similarity index 100% rename from components/elm/tools/clm4_5/refactorTools/clmType/renameClmType.pl rename to components/elm/tools/refactorTools/clmType/renameClmType.pl From 158c5e6703fe3ac1d564156c697901837945d871 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Mon, 8 Nov 2021 13:58:39 -0800 Subject: [PATCH 121/301] Changes to the relative path of file --- components/elm/bld/ELMBuildNamelist.pm | 2 +- .../elm/tools/mksurfdata_map/mksurfdata.pl | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/components/elm/bld/ELMBuildNamelist.pm b/components/elm/bld/ELMBuildNamelist.pm index 5855209bb601..9981214e1f6e 100755 --- a/components/elm/bld/ELMBuildNamelist.pm +++ b/components/elm/bld/ELMBuildNamelist.pm @@ -385,7 +385,7 @@ sub check_for_perl_utils { my $cfgdir = shift; # Determine E3SM root directory and perl5lib root directory - my $cesmroot = abs_path( "$cfgdir/../../../"); + my $cesmroot = abs_path( "$cfgdir/../../"); my $perl5lib_dir = "$cesmroot/cime/utils/perl5lib"; # The root diretory for the perl SetupTools.pm module diff --git a/components/elm/tools/mksurfdata_map/mksurfdata.pl b/components/elm/tools/mksurfdata_map/mksurfdata.pl index 96afbf0c2fe3..1550f7b3856d 100755 --- a/components/elm/tools/mksurfdata_map/mksurfdata.pl +++ b/components/elm/tools/mksurfdata_map/mksurfdata.pl @@ -25,7 +25,7 @@ #----------------------------------------------------------------------------------------------- # Add $scrdir to the list of paths that Perl searches for modules -my @dirs = ( $scrdir, "$scrdir/../../../../../cime/utils/perl5lib", +my @dirs = ( $scrdir, "$scrdir/../../../../cime/utils/perl5lib", ); unshift @INC, @dirs; my $result = eval "require XML::Lite"; @@ -40,7 +40,7 @@ ** Cannot find perl module \"Build/NamelistDefinition.pm\" from directories: @dirs ** EOF } -my $nldef_file = "$scrdir/../../../bld/namelist_files/namelist_definition.xml"; +my $nldef_file = "$scrdir/../../bld/namelist_files/namelist_definition.xml"; my $definition = Build::NamelistDefinition->new( $nldef_file ); @@ -394,7 +394,7 @@ ($) } else { $queryopts = "-res $res -csmdata $CSMDATA -silent -justvalue"; } - $queryfilopts = "$queryopts -onlyfiles -phys clm4_5 "; + $queryfilopts = "$queryopts -onlyfiles "; my $mkcrop = ",crop='off'"; my $setnumpft = ""; if ( defined($opts{'crop'}) ) { @@ -418,24 +418,24 @@ ($) $merge_gis = "on"; } my $mopts = "$queryopts -namelist default_settings $usrnam"; - my $mkopts = "-csmdata $CSMDATA -silent -justvalue -namelist clmexp $usrnam"; + my $mkopts = "-csmdata $CSMDATA -silent -justvalue -namelist elmexp $usrnam"; foreach my $typ ( "lak", "veg", "voc", "top", "tex", "col","ord", "fmx", "lai", "urb", "org", "glc", "utp", "wet", "gdp", "peat","abm", "topostats" , "vic", "ch4", "pho", "grvl", "slp10", "ero") { - my $lmask = `$scrdir/../../../bld/queryDefaultNamelist.pl $mopts -options type=$typ,mergeGIS=$merge_gis,hirespft=$hirespft -var lmask`; + my $lmask = `$scrdir/../../bld/queryDefaultNamelist.pl $mopts -silent -options type=$typ,mergeGIS=$merge_gis,hirespft=$hirespft -var lmask`; $lmask = trim($lmask); - my $hgrid = `$scrdir/../../../bld/queryDefaultNamelist.pl $mopts -options type=$typ,hirespft=$hirespft -var hgrid`; + my $hgrid = `$scrdir/../../bld/queryDefaultNamelist.pl $mopts -options type=$typ,hirespft=$hirespft -var hgrid`; $hgrid = trim($hgrid); - my $filnm = `$scrdir/../../../bld/queryDefaultNamelist.pl $mopts -options type=$typ -var mksrf_filename`; + my $filnm = `$scrdir/../../bld/queryDefaultNamelist.pl $mopts -options type=$typ -var mksrf_filename`; $filnm = trim($filnm); $hgrd{$typ} = $hgrid; $lmsk{$typ} = $lmask; if ( $opts{'hgrid'} eq "usrspec" ) { $map{$typ} = $opts{'usr_mapdir'}."/map_${hgrid}_${lmask}_to_${res}_nomask_aave_da_c${mapdate}\.nc"; } else { - $map{$typ} = `$scrdir/../../../bld/queryDefaultNamelist.pl $queryfilopts -namelist clmexp -options frm_hgrid=$hgrid,frm_lmask=$lmask,to_hgrid=$res,to_lmask=nomask -var map`; - } + $map{$typ} = `$scrdir/../../bld/queryDefaultNamelist.pl $queryfilopts -namelist elmexp -options frm_hgrid=$hgrid,frm_lmask=$lmask,to_hgrid=$res,to_lmask=nomask -var map`; + } $map{$typ} = trim($map{$typ}); if ( $map{$typ} !~ /[^ ]+/ ) { die "ERROR: could NOT find a mapping file for this resolution: $res and type: $typ at $hgrid and $lmask.\n"; @@ -443,7 +443,7 @@ ($) if ( ! defined($opts{'allownofile'}) && ! -f $map{$typ} ) { die "ERROR: mapping file for this resolution does NOT exist ($map{$typ}).\n"; } - $datfil{$typ} = `$scrdir/../../../bld/queryDefaultNamelist.pl $mkopts -options hgrid=$hgrid,lmask=$lmask,mergeGIS=$merge_gis$mkcrop -var $filnm`; + $datfil{$typ} = `$scrdir/../../bld/queryDefaultNamelist.pl $mkopts -options hgrid=$hgrid,lmask=$lmask,mergeGIS=$merge_gis$mkcrop -var $filnm`; $datfil{$typ} = trim($datfil{$typ}); if ( $datfil{$typ} !~ /[^ ]+/ ) { die "ERROR: could NOT find a $filnm data file for this resolution: $hgrid and type: $typ and $lmask.\n"; @@ -457,7 +457,7 @@ ($) # my $griddata = trim($map{'veg'}); if ( $griddata eq "" ) { - $griddata = `$scrdir/../../../bld/queryDefaultNamelist.pl $queryfilopts $usrnam -var fatmgrid`; + $griddata = `$scrdir/../../bld/queryDefaultNamelist.pl $queryfilopts $usrnam -var fatmgrid`; if ( $griddata eq "" ) { die "ERROR: could NOT find a grid data file for this resolution: $res.\n"; } @@ -577,7 +577,7 @@ ($) } else { $rcp_option = ",rcp=$rcp"; } - my $cmd = "$scrdir/../../../bld/queryDefaultNamelist.pl $queryfilopts $resol -options sim_year=${sim_yr0}$mkcrop$rcp_option -var mksrf_fvegtyp -namelist clmexp"; + my $cmd = "$scrdir/../../bld/queryDefaultNamelist.pl $queryfilopts $resol -options sim_year=${sim_yr0}$mkcrop$rcp_option -var mksrf_fvegtyp -namelist elmexp"; my $vegtyp = `$cmd`; chomp( $vegtyp ); if ( $vegtyp eq "" ) { @@ -606,7 +606,7 @@ ($) $fh_landuse_timeseries->open( ">$landuse_timeseries_text_file" ) or die "** can't open file: $landuse_timeseries_text_file\n"; print "Writing out landuse_timeseries text file: $landuse_timeseries_text_file\n"; for( my $yr = $sim_yr0; $yr <= $sim_yrn; $yr++ ) { - my $vegtypyr = `$scrdir/../../../bld/queryDefaultNamelist.pl $queryfilopts $resol -options sim_year=$yr,rcp=${rcp}${mkcrop} -var mksrf_fvegtyp -namelist clmexp`; + my $vegtypyr = `$scrdir/../../bld/queryDefaultNamelist.pl $queryfilopts $resol -options sim_year=$yr,rcp=${rcp}${mkcrop} -var mksrf_fvegtyp -namelist elmexp`; chomp( $vegtypyr ); printf $fh_landuse_timeseries $dynpft_format, $vegtypyr, $yr; if ( $yr % 100 == 0 ) { @@ -729,7 +729,7 @@ ($) # If urban point, overwrite urban variables from previous surface dataset to this one # if ( $urb_pt ) { - my $prvsurfdata = `$scrdir/../../../bld/queryDefaultNamelist.pl $queryopts -var fsurdat`; + my $prvsurfdata = `$scrdir/../../bld/queryDefaultNamelist.pl $queryopts -var fsurdat`; if ( $? != 0 ) { die "ERROR:: previous surface dataset file NOT found\n"; } From 798370999030cf9cef8a55dd00b55dc6ba999e45 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Mon, 8 Nov 2021 14:07:27 -0800 Subject: [PATCH 122/301] Changes clmexp to elmexp --- components/elm/tools/mksurfdata_map/mksurfdata.pl | 2 +- components/elm/tools/mksurfdata_map/src/mksurfdat.F90 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/elm/tools/mksurfdata_map/mksurfdata.pl b/components/elm/tools/mksurfdata_map/mksurfdata.pl index 1550f7b3856d..ff670e9d16ea 100755 --- a/components/elm/tools/mksurfdata_map/mksurfdata.pl +++ b/components/elm/tools/mksurfdata_map/mksurfdata.pl @@ -510,7 +510,7 @@ ($) $fh->open( ">$nl" ) or die "** can't open file: $nl\n"; print "CSMDATA is $CSMDATA \n"; print $fh <<"EOF"; -&clmexp +&elmexp nglcec = $glc_nec mksrf_fgrid = '$griddata' map_fpft = '$map{'veg'}' diff --git a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 index 84f7c2d0815f..275ec69ec77f 100644 --- a/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 +++ b/components/elm/tools/mksurfdata_map/src/mksurfdat.F90 @@ -154,7 +154,7 @@ program mksurfdat character(len=32) :: subname = 'mksrfdat' ! program name - namelist /clmexp/ & + namelist /elmexp/ & mksrf_fgrid, & mksrf_gridtype, & mksrf_fvegtyp, & @@ -319,7 +319,7 @@ program mksurfdat all_urban = .false. no_inlandwet = .true. - read(5, clmexp, iostat=ier) + read(5, elmexp, iostat=ier) if (ier /= 0) then write(6,*)'error: namelist input resulted in error code ',ier call abort() From 300d1df550e54412f65895980eccec5a8b7a6ca9 Mon Sep 17 00:00:00 2001 From: Gautam Bisht Date: Mon, 8 Nov 2021 14:13:11 -0800 Subject: [PATCH 123/301] Fixes the path --- components/elm/bld/ELMBuildNamelist.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/ELMBuildNamelist.pm b/components/elm/bld/ELMBuildNamelist.pm index 9981214e1f6e..5855209bb601 100755 --- a/components/elm/bld/ELMBuildNamelist.pm +++ b/components/elm/bld/ELMBuildNamelist.pm @@ -385,7 +385,7 @@ sub check_for_perl_utils { my $cfgdir = shift; # Determine E3SM root directory and perl5lib root directory - my $cesmroot = abs_path( "$cfgdir/../../"); + my $cesmroot = abs_path( "$cfgdir/../../../"); my $perl5lib_dir = "$cesmroot/cime/utils/perl5lib"; # The root diretory for the perl SetupTools.pm module From c171ecd78e5232198a4986c078824fb9198577ab Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 9 Nov 2021 12:58:39 -0700 Subject: [PATCH 124/301] Refactor along pylint lines --- cime_config/machines/scripts/evaluator | 118 +++++++++++++++++++++---- 1 file changed, 102 insertions(+), 16 deletions(-) diff --git a/cime_config/machines/scripts/evaluator b/cime_config/machines/scripts/evaluator index 43f8f27b6631..a9c2b9b32a42 100755 --- a/cime_config/machines/scripts/evaluator +++ b/cime_config/machines/scripts/evaluator @@ -1,11 +1,8 @@ #! /usr/bin/env python3 """ -Evaluate macros for problems or inefficiencies: -1) Repitition -2) Promotion: all children of a parent do the same thing, so do it in the parent -3) Suspicious set, probably want append -4) Using OS, COMPILER, or MACH with no matches in config_machines.xml +Intended to be like pylint except for our cmake macros +(Evaluate macros for problems or inefficiencies.) """ import argparse, sys, os, re, pathlib @@ -20,18 +17,33 @@ IF_RE = re.compile(r'^\s*if\s*[(]([^)]+)[)]\s*$') ENDIF_RE = re.compile(r'^\s*endif') EXE_RE = re.compile(r'^\s*execute_process') +WARNING_SET = { + "REP" : "Repitition: either a repeated item in an append or setting to same value as parent", + "PRO" : "Promotion: all children of a parent do the same thing, so do it in the parent", + "SET" : "Suspicious set: probably want append", + "UNU" : "Unused: Using OS, COMPILER, or MACH with no matches in config_machines.xml", + "MIS" : "Missing: Having a machine/comp in config_machines.xml with no specialization in Macros" +} + ############################################################################### def parse_command_line(args, description): ############################################################################### + warning_help = "" + for k, v in WARNING_SET.items(): + warning_help += f" {k} => {v}\n" + parser = argparse.ArgumentParser( usage="""\n{0} [] OR {0} --help +Available warnings (CODE => Decription) +{1} + \033[1mEXAMPLES:\033[0m \033[1;32m# Evaluate cwd \033[0m > {0} -""".format(os.path.basename(args[0])), + """.format(os.path.basename(args[0]), warning_help), description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter ) @@ -42,14 +54,35 @@ OR parser.add_argument("-c", "--compiler", help="Limit evaluation to certain compilers") + parser.add_argument("-d", "--disable", default=[], action="append", choices=list(WARNING_SET.keys()), + help="Disable these warning codes. Assumes all other codes are on.") + + parser.add_argument("-e", "--enable", default=[], action="append", choices=list(WARNING_SET.keys()), + help="Enable these warning codes. Assumes all other codes are off.") + args = parser.parse_args(args[1:]) assert (args.machine is None) or (args.compiler is None), "Do not set both -m and -c" + assert (args.disable == []) or (args.enable == []), "Do not set both -d and -e" + + if args.disable: + args.codes_to_check = list(set(WARNING_SET.keys()) - set(args.disable)) + elif args.enable: + args.codes_to_check = list(args.enable) + else: + args.codes_to_check = list(WARNING_SET.keys()) + + delattr(args, "disable") + delattr(args, "enable") + + if "UNU" in args.codes_to_check: + assert args.machine is None and args.compiler is None, \ + "Cannot use UNU check with -c or -m options. Please disable" return args ############################################################################### -def find_machs_comps_oss(config_machines): +def find_machs_comps_oss(config_machines, machine=None): ############################################################################### """ Return a dict mapping MACH -> (OS, COMPILERS), compilers, OSs @@ -75,6 +108,9 @@ def find_machs_comps_oss(config_machines): mach = child.attrib["MACH"].strip() assert mach not in machs, f"Repeat of mach {mach}" + if machine is not None and mach != machine: + continue + found_os = False found_comps = False for subchild in child: @@ -149,7 +185,7 @@ def get_all_macros(macro_path_pl, universal, the_os, mach, comp): return results ############################################################################### -def find_suspicious_set(macro_path_pl, machs): +def find_SET(macro_path_pl, machs): ############################################################################### warnings = [] for universal in UNIVERSALS: @@ -167,12 +203,11 @@ def find_suspicious_set(macro_path_pl, machs): mixes = sets & appends if mixes: warnings.append(f"Macro {macro_pl} began to mix sets and appends for vars: {', '.join(mixes)}") - print(warnings[-1]) return warnings ############################################################################### -def find_unused_macros(macro_path_pl, machs): +def find_UNU(macro_path_pl, machs): ############################################################################### warnings = [] unused_macro_files = set([item for item in macro_path_pl.glob("*.cmake") if item.name != "Macros.cmake"]) @@ -184,22 +219,73 @@ def find_unused_macros(macro_path_pl, machs): unused_macro_files = unused_macro_files - set(macro_pls) for unused_macro_file in unused_macro_files: - warnings.append(f"Macro {unused_macro_file} is unused") - print(warnings[-1]) + warnings.append(f"Macro {unused_macro_file} is unused by anything in config_machines.xml") + + return warnings + +############################################################################### +def find_REP(macro_path_pl, machs): +############################################################################### + warnings = [] + for universal in UNIVERSALS: + for mach, data in machs.items(): + the_os = data[0] + for comp in data[1]: + macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) + sets = {} + appends = {} + for macro_pl in macro_pls: + curr_settings, curr_appends = parse_macro(macro_pl) + for var, val in curr_settings: + if var in sets: + if sets[var] == val: + warnings.append(f"Macro {macro_pl} has redundant set of {var} to {val}") + else: + sets[var] = val + else: + sets[var] = val + + for var, vals in curr_appends: + if var in appends: + potential_repeats = set(appends[var].split()) & set(vals.split()) + if potential_repeats: + warnings.append(f"Macro {macro_pl} has redundant set of {var}, repeated items are to {', '.join(potential_repeats)}") + + appends[var] += vals + else: + appends[var] = vals return warnings ############################################################################### -def evaluate(macro_path, machine=None, compiler=None): +def find_PRO(macro_path_pl, machs): +############################################################################### + warnings = [] + warnings.append("This warning is not yet implemented") + return warnings + +############################################################################### +def find_MIS(macro_path_pl, machs): +############################################################################### + warnings = [] + warnings.append("This warning is not yet implemented") + return warnings + +############################################################################### +def evaluate(macro_path, codes_to_check, machine=None, compiler=None): ############################################################################### macro_path_pl = pathlib.Path(macro_path) config_machines_pl = macro_path_pl / ".." / "config_machines.xml" - machs, all_comps, oss = find_machs_comps_oss(config_machines_pl) + machs, all_comps, oss = find_machs_comps_oss(config_machines_pl, machine=machine) warnings = [] - warnings.extend(find_suspicious_set(macro_path_pl, machs)) - warnings.extend(find_unused_macros(macro_path_pl, machs)) + for warning_code in codes_to_check: + warnings.extend([f"{warning_code}: {item}" + for item in globals()[f"find_{warning_code}"](macro_path_pl, machs)]) + + for warning in warnings: + print(warning) return warnings == [] From f9217b2c1c7af7a068838c260cf1670e8994bb1c Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 9 Nov 2021 14:06:18 -0700 Subject: [PATCH 125/301] Implement MIS warning --- cime_config/machines/scripts/README | 1 + cime_config/machines/scripts/evaluator | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cime_config/machines/scripts/README b/cime_config/machines/scripts/README index f9f3f2bf8cdd..d2dbfcf051e2 100644 --- a/cime_config/machines/scripts/README +++ b/cime_config/machines/scripts/README @@ -1,2 +1,3 @@ This scripts should only be needed temporarily during the transition from config_compilers.xml to cmake-macros. +The evaluator script may have longer-term usefulness. \ No newline at end of file diff --git a/cime_config/machines/scripts/evaluator b/cime_config/machines/scripts/evaluator index a9c2b9b32a42..d243b9fbf949 100755 --- a/cime_config/machines/scripts/evaluator +++ b/cime_config/machines/scripts/evaluator @@ -268,7 +268,17 @@ def find_PRO(macro_path_pl, machs): def find_MIS(macro_path_pl, machs): ############################################################################### warnings = [] - warnings.append("This warning is not yet implemented") + macro_files = set([item for item in macro_path_pl.glob("*.cmake") if item.name != "Macros.cmake"]) + for mach, data in machs.items(): + the_os = data[0] + for comp in data[1]: + compiler_macro = macro_path_pl / f"{comp}.cmake" + comp_mach_macro = macro_path_pl / f"{comp}_{mach}.cmake" + if compiler_macro not in macro_files: + warnings.append(f"No macro {compiler_macro} for compiler {comp} used by machine {mach}") + if comp_mach_macro not in macro_files: + warnings.append(f"No macro {comp_mach_macro} for compiler {comp} and machine {mach}") + return warnings ############################################################################### From 1aaba5f40e9fe488e3ee26749956544aa3d842bd Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 9 Nov 2021 14:15:05 -0700 Subject: [PATCH 126/301] Reduce duplication in ibmgpu cmake macros --- .../machines/cmake_macros/ibmgpu.cmake | 70 +++++++++++++++++++ .../machines/cmake_macros/ibmgpu_ascent.cmake | 69 ------------------ .../machines/cmake_macros/ibmgpu_summit.cmake | 69 ------------------ 3 files changed, 70 insertions(+), 138 deletions(-) create mode 100644 cime_config/machines/cmake_macros/ibmgpu.cmake diff --git a/cime_config/machines/cmake_macros/ibmgpu.cmake b/cime_config/machines/cmake_macros/ibmgpu.cmake new file mode 100644 index 000000000000..b5e6bf7e8e5e --- /dev/null +++ b/cime_config/machines/cmake_macros/ibmgpu.cmake @@ -0,0 +1,70 @@ +string(APPEND CFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") +if (NOT DEBUG) + string(APPEND CFLAGS " -O3") +endif() +if (NOT DEBUG AND compile_threaded) + string(APPEND CFLAGS " -qsmp=omp") +endif() +if (DEBUG AND compile_threaded) + string(APPEND CFLAGS " -qsmp=omp:noopt") +endif() +string(APPEND CXXFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") +if (NOT DEBUG) + string(APPEND CXXFLAGS " -O2") +endif() +if (NOT DEBUG AND compile_threaded) + string(APPEND CXXFLAGS " -qsmp=omp") +endif() +if (DEBUG AND compile_threaded) + string(APPEND CXXFLAGS " -qsmp=omp:noopt") +endif() +string(APPEND CPPDEFS " -DFORTRAN_SAME -DCPRIBM") +if (COMP_NAME STREQUAL eam) + string(APPEND CPPDEFS " -DUSE_CBOOL") +endif() +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_SLASHPROC") +endif() +set(CPRE "-WF,-D") +string(APPEND FC_AUTO_R8 " -qrealsize=8") +string(APPEND FFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") +if (NOT DEBUG) + string(APPEND FFLAGS " -O2 -qstrict -Q") +endif() +if (NOT DEBUG AND compile_threaded) + string(APPEND FFLAGS " -qsmp=omp") +endif() +if (DEBUG AND compile_threaded) + string(APPEND FFLAGS " -qsmp=omp:noopt") +endif() +string(APPEND FFLAGS " -qzerosize -qfree=f90 -qxlf2003=polymorphic") +string(APPEND FFLAGS " -qspillsize=2500 -qextname=flush") +if (COMP_NAME STREQUAL cice AND compile_threaded) + string(APPEND FFLAGS " -qsmp=omp:noopt") +endif() +string(APPEND FFLAGS_NOOPT " -O0") +string(APPEND FIXEDFLAGS " -qsuffix=f=f -qfixed=132") +string(APPEND FREEFLAGS " -qsuffix=f=f90:cpp=F90") +set(HAS_F2008_CONTIGUOUS "TRUE") +string(APPEND LDFLAGS " -Wl,--relax -Wl,--allow-multiple-definition") +string(APPEND LDFLAGS " -qsmp -qoffload -lcudart -L$ENV{CUDA_DIR}/lib64") +if (MPILIB STREQUAL mpi-serial) + string(APPEND SLIBS " -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") +endif() +if (NOT MPILIB STREQUAL mpi-serial) + string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") +endif() +string(APPEND CXX_LIBS " -L/sw/summit/gcc/8.1.1/lib64 -lstdc++ -L$ENV{OLCF_XLC_ROOT}/lib -libmc++") +set(MPICC "mpicc") +set(MPICXX "mpiCC") +set(MPIFC "mpif90") +set(PIO_FILESYSTEM_HINTS "gpfs") +set(SCC "xlc_r") +set(SFC "xlf90_r") +set(SCXX "xlc++_r") +set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") +set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") +set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") +set(SUPPORTS_CXX "TRUE") +set(KOKKOS_OPTIONS "--arch=Power9 --with-serial") diff --git a/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake b/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake index e55ce6b180df..5397658dc89a 100644 --- a/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake @@ -1,73 +1,4 @@ -string(APPEND CFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND CFLAGS " -O3") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND CFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND CFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND CXXFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND CXXFLAGS " -O2") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND CXXFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND CXXFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND CPPDEFS " -DFORTRAN_SAME -DCPRIBM") -if (COMP_NAME STREQUAL eam) - string(APPEND CPPDEFS " -DUSE_CBOOL") -endif() -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_SLASHPROC") -endif() -set(CPRE "-WF,-D") -string(APPEND FC_AUTO_R8 " -qrealsize=8") -string(APPEND FFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -qstrict -Q") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp:noopt") -endif() if (DEBUG) string(APPEND FFLAGS " -qinitauto=7FF7FFFF -qflttrap=ov:zero:inv:en") endif() -string(APPEND FFLAGS " -qzerosize -qfree=f90 -qxlf2003=polymorphic") -string(APPEND FFLAGS " -qspillsize=2500 -qextname=flush") -if (COMP_NAME STREQUAL cice AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND FFLAGS_NOOPT " -O0") -string(APPEND FIXEDFLAGS " -qsuffix=f=f -qfixed=132") -string(APPEND FREEFLAGS " -qsuffix=f=f90:cpp=F90") -set(HAS_F2008_CONTIGUOUS "TRUE") -string(APPEND LDFLAGS " -Wl,--relax -Wl,--allow-multiple-definition") -string(APPEND LDFLAGS " -qsmp -qoffload -lcudart -L$ENV{CUDA_DIR}/lib64") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") -endif() -if (NOT MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") -endif() string(APPEND CXX_LIBS " -L/sw/ascent/gcc/8.1.1/lib64 -lstdc++ -L$ENV{OLCF_XLC_ROOT}/lib -libmc++") -set(MPICC "mpicc") -set(MPICXX "mpiCC") -set(MPIFC "mpif90") -set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "xlc_r") -set(SFC "xlf90_r") -set(SCXX "xlc++_r") -set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") -set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") -set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") -set(KOKKOS_OPTIONS "--arch=Power9 --with-serial") diff --git a/cime_config/machines/cmake_macros/ibmgpu_summit.cmake b/cime_config/machines/cmake_macros/ibmgpu_summit.cmake index b5e6bf7e8e5e..f3e965549d0c 100644 --- a/cime_config/machines/cmake_macros/ibmgpu_summit.cmake +++ b/cime_config/machines/cmake_macros/ibmgpu_summit.cmake @@ -1,70 +1 @@ -string(APPEND CFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND CFLAGS " -O3") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND CFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND CFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND CXXFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND CXXFLAGS " -O2") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND CXXFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND CXXFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND CPPDEFS " -DFORTRAN_SAME -DCPRIBM") -if (COMP_NAME STREQUAL eam) - string(APPEND CPPDEFS " -DUSE_CBOOL") -endif() -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_SLASHPROC") -endif() -set(CPRE "-WF,-D") -string(APPEND FC_AUTO_R8 " -qrealsize=8") -string(APPEND FFLAGS " -g -qfullpath -qmaxmem=-1 -qphsinfo") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -qstrict -Q") -endif() -if (NOT DEBUG AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp") -endif() -if (DEBUG AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND FFLAGS " -qzerosize -qfree=f90 -qxlf2003=polymorphic") -string(APPEND FFLAGS " -qspillsize=2500 -qextname=flush") -if (COMP_NAME STREQUAL cice AND compile_threaded) - string(APPEND FFLAGS " -qsmp=omp:noopt") -endif() -string(APPEND FFLAGS_NOOPT " -O0") -string(APPEND FIXEDFLAGS " -qsuffix=f=f -qfixed=132") -string(APPEND FREEFLAGS " -qsuffix=f=f90:cpp=F90") -set(HAS_F2008_CONTIGUOUS "TRUE") -string(APPEND LDFLAGS " -Wl,--relax -Wl,--allow-multiple-definition") -string(APPEND LDFLAGS " -qsmp -qoffload -lcudart -L$ENV{CUDA_DIR}/lib64") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") -endif() -if (NOT MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -lxlopt -lxl -lxlsmp -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") -endif() string(APPEND CXX_LIBS " -L/sw/summit/gcc/8.1.1/lib64 -lstdc++ -L$ENV{OLCF_XLC_ROOT}/lib -libmc++") -set(MPICC "mpicc") -set(MPICXX "mpiCC") -set(MPIFC "mpif90") -set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "xlc_r") -set(SFC "xlf90_r") -set(SCXX "xlc++_r") -set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") -set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") -set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") -set(KOKKOS_OPTIONS "--arch=Power9 --with-serial") From 75e3ff2e060904340ecd68a5b75da4c336dd514d Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 9 Nov 2021 14:50:09 -0800 Subject: [PATCH 127/301] For the compiler flag change, only do it for GNU v9 and larger. --- cime_config/machines/Depends.gnu.cmake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/Depends.gnu.cmake b/cime_config/machines/Depends.gnu.cmake index 25c4fba8ba98..f55fca3955f7 100644 --- a/cime_config/machines/Depends.gnu.cmake +++ b/cime_config/machines/Depends.gnu.cmake @@ -4,8 +4,11 @@ list(APPEND MPAS_ICE_SHORTWAVE ${CMAKE_BINARY_DIR}/core_seaice/column/ice_shortwave.f90 ) +# For optimized GNU builds that use v9 or higher, remove an optimization on one file if (NOT DEBUG) - foreach(ITEM IN LISTS MPAS_ICE_SHORTWAVE) - e3sm_add_flags("${ITEM}" "-fno-tree-pta") # avoids an error that shows up in solver with gnu9 and higher - endforeach() + if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 9) + foreach(ITEM IN LISTS MPAS_ICE_SHORTWAVE) + e3sm_add_flags("${ITEM}" "-fno-tree-pta") # avoids an error that shows up in solver with gnu9 and higher + endforeach() + endif() endif() From f715a876294d2b44cfa5976e771fcce3c80b9960 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 9 Nov 2021 17:29:51 -0800 Subject: [PATCH 128/301] fix component namelist creation bugs when NINST>1 [BFB] --- components/eam/bld/build-namelist | 2 +- components/eam/cime_config/buildnml | 5 +++-- components/elm/cime_config/buildnml | 5 +++-- components/mpas-seaice/cime_config/buildnml | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index cc37a1fcccc2..51e938926588 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -4772,7 +4772,7 @@ sub check_input_files { my $fh; if (defined $inputdata_rootdir) { - open $fh, '>', $outfile or die "check_input_files: error opening $outfile: $!"; + open $fh, '>>', $outfile or die "check_input_files: error opening $outfile: $!"; } # Look through all namelist groups diff --git a/components/eam/cime_config/buildnml b/components/eam/cime_config/buildnml index 6ecb99222f37..4719796e168a 100755 --- a/components/eam/cime_config/buildnml +++ b/components/eam/cime_config/buildnml @@ -202,8 +202,9 @@ def buildnml(case, caseroot, compname): # call build-namelist # ----------------------------------------------------- - if os.path.exists(os.path.join(casebuild, "eam.input_data_list")): - os.remove(os.path.join(casebuild, "eam.input_data_list")) + if inst_counter ==1: + if os.path.exists(os.path.join(casebuild, "eam.input_data_list")): + os.remove(os.path.join(casebuild, "eam.input_data_list")) cam_buildnml_cmd = os.path.join(srcroot, "components/eam/bld/build-namelist") cam_buildnml_cmd += " -infile {}/cesm_namelist".format(eamconf_dir) diff --git a/components/elm/cime_config/buildnml b/components/elm/cime_config/buildnml index 46512f2c305b..dd074ee6b9f7 100755 --- a/components/elm/cime_config/buildnml +++ b/components/elm/cime_config/buildnml @@ -138,8 +138,9 @@ def buildnml(case, caseroot, compname): # create elmconf/namelist # ----------------------------------------------------- - if os.path.exists("{}/Buildconf/elm.input_data_list".format(caseroot)): - os.remove("{}/Buildconf/elm.input_data_list".format(caseroot)) + if inst_counter ==1: + if os.path.exists("{}/Buildconf/elm.input_data_list".format(caseroot)): + os.remove("{}/Buildconf/elm.input_data_list".format(caseroot)) elmicfile = "" elm_startfile = "" diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index 486683fb6d21..f2d30e5ed870 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -40,7 +40,7 @@ def buildnml(case, caseroot, compname): #ninst_ice = case.get_value("NINST_ICE") ninst_ice = 1 # Change if you want multiple instances... though this isn't coded yet. ninst_ice_real = case.get_value("NINST_ICE") - ntasks_ice = case.get_value("NTASKS_ICE") + ntasks_ice = case.get_value("NTASKS_PER_INST_ICE") rundir = case.get_value("RUNDIR") run_type = case.get_value("RUN_TYPE") run_refcase = case.get_value("RUN_REFCASE") From 8c6cbb51da088ebba454a2a253b1045d0bba4071 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Wed, 10 Nov 2021 07:36:25 -0800 Subject: [PATCH 129/301] Use F2010-CICE for extra coverage tests for ena and twp rrm F2010 is being used in extra coverage test suite for enax4v1 and twpx4v1, and the tests failed to proceed. Change to use compset F2010-CICE as no mpas mesh has been configured to work with these two atm RRM meshes. [BFB] --- cime_config/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 2424643a9260..d31d5d604bcc 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -182,8 +182,8 @@ "e3sm_extra_coverage" : { "inherit" : ("e3sm_atm_extra_coverage", "e3sm_ocnice_extra_coverage"), "tests" : ( - "SMS_D_Ln5.enax4v1_enax4v1.F2010", - "SMS_D_Ln5.twpx4v1_twpx4v1.F2010", + "SMS_D_Ln5.enax4v1_enax4v1.F2010-CICE", + "SMS_D_Ln5.twpx4v1_twpx4v1.F2010-CICE", ) }, From f630cb746e5e5b64eb8c69751a8cd413360169da Mon Sep 17 00:00:00 2001 From: hyungyukang Date: Wed, 10 Nov 2021 13:18:21 -0800 Subject: [PATCH 130/301] Time step size definition in si_init Time step size definition change (parsed from config_dt -> passing in 'dt' itself) in si_init routine and small modifications of comments --- .../mpas_ocn_time_integration_si.F | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index b8c3718ee059..956f83360283 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -743,8 +743,9 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! ! The split-implicit barotropic mode solver ! - uses the preconditioned communication-avoiding - ! (single-reduction) BiCGStab method - ! (Cool and Vanroose, 2017) as a linear iterative solver + ! (single-global synchronization) BiCGStab method + ! as a linear iterative solver + ! (Kang et al., in preparation) ! ! - solves the nonlinear barotropic system but deals with ! it as the linear system by introducing the outer and @@ -766,7 +767,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! obtain SSH at time (n+1) ! Stage 2.7. The barotropic velocity update ! - ! - requires 'config_n_ts_iter=2' + ! - recommends to set 'config_n_ts_iter=2' ! :Time line of variables ! ! Start of large outer time step iteration loop @@ -787,7 +788,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! SSH and BtrVel at time (n+1) ! ! - ! Reference: Kang et al. (2021): A scalable split-implicit + ! Reference: Kang et al. (2021): A scalable semi-implicit ! barotropic mode solver for the MPAS-Ocean, JAMES ! ! @@ -3441,10 +3442,7 @@ subroutine ocn_time_integration_si_init(domain, dt)!{{{ alpha2= 1.0_RKIND - alpha1 ! Get time step size to compute coefficients for the SI solver - read(config_dt(1:2),*) ihh - read(config_dt(4:5),*) imm - read(config_dt(7:8),*) iss - dt_si = ihh * 3600.0_RKIND + imm * 60.0_RKIND + iss + dt_si = dt ! Determination of nSiLargeIter (the barotropic system ! large iteration loop) @@ -3580,6 +3578,7 @@ subroutine ocn_time_integrator_si_preconditioner(domain, dt)!{{{ nPrecVec = nCellsHalo2nd ! length of preconditioning vector nPrecMatPacked = (nPrecVec*(nPrecVec+1))/2 + ! Packed size of preconditiong matrix allocate(prec_ivmat(1:nPrecMatPacked)) prec_ivmat(:) = 0.0_RKIND @@ -3640,8 +3639,9 @@ subroutine ocn_time_integrator_si_preconditioner(domain, dt)!{{{ elseif ( trim(config_btr_si_preconditioner) == & 'block_jacobi' ) then - nPrecVec = nCells ! length of preconditioning vector + nPrecVec = nCells ! length of preconditioning vector nPrecMatPacked = (nPrecVec*(nPrecVec+1))/2 + ! Packed size of preconditiong matrix allocate(prec_ivmat(1:nPrecMatPacked)) prec_ivmat(:) = 0.0_RKIND From ecef562f36dedf28c72b1719784be333f68d78c7 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 10 Nov 2021 15:45:25 -0600 Subject: [PATCH 131/301] Change NTASKS_XXX to NTASKS_PER_INST_XXX for mpaso and mali --- components/mpas-albany-landice/cime_config/buildnml | 2 +- components/mpas-ocean/cime_config/buildnml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 2ce0027925ec..37da31bc4507 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -33,7 +33,7 @@ def buildnml(case, caseroot, compname): #ninst_glc = case.get_value("NINST_GLC") ninst_glc = 1 # Change if you want multiple instances... though this isn't coded yet. ninst_glc_real = case.get_value("NINST_GLC") - ntasks_glc = case.get_value("NTASKS_GLC") + ntasks_glc = case.get_value("NTASKS_PER_INST_GLC") rundir = case.get_value("RUNDIR") run_type = case.get_value("RUN_TYPE") run_refcase = case.get_value("RUN_REFCASE") diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 08a712f1cd2a..ce2e6a66f080 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -43,7 +43,7 @@ def buildnml(case, caseroot, compname): ninst_ocn = 1 # Change if you want multiple instances... though this isn't coded yet. ninst_ocn_real = case.get_value("NINST_OCN") nthrds_ocn = case.get_value("NTHRDS_OCN") - ntasks_ocn = case.get_value("NTASKS_OCN") + ntasks_ocn = case.get_value("NTASKS_PER_INST_OCN") rundir = case.get_value("RUNDIR") run_type = case.get_value("RUN_TYPE") run_refcase = case.get_value("RUN_REFCASE") From dea06f5d08b40004d262a23fb2d5e1499483badb Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 11 Nov 2021 14:06:30 -0700 Subject: [PATCH 132/301] Get rid of redundant sets --- cime_config/machines/cmake_macros/gnu_ascent.cmake | 6 ------ cime_config/machines/cmake_macros/gnu_badger.cmake | 5 ----- cime_config/machines/cmake_macros/gnu_cades.cmake | 6 ------ cime_config/machines/cmake_macros/gnu_eddi.cmake | 7 ------- cime_config/machines/cmake_macros/gnu_gcp.cmake | 1 - cime_config/machines/cmake_macros/gnu_grizzly.cmake | 5 ----- cime_config/machines/cmake_macros/gnu_modex.cmake | 1 - cime_config/machines/cmake_macros/gnu_summit.cmake | 6 ------ cime_config/machines/cmake_macros/gnugpu_ascent.cmake | 1 - cime_config/machines/cmake_macros/gnugpu_summit.cmake | 1 - cime_config/machines/cmake_macros/intel_badger.cmake | 9 --------- cime_config/machines/cmake_macros/intel_grizzly.cmake | 9 --------- cime_config/machines/cmake_macros/intel_itasca.cmake | 5 ----- cime_config/machines/cmake_macros/intel_stampede2.cmake | 6 ------ cime_config/machines/cmake_macros/pgi_ascent.cmake | 4 ---- cime_config/machines/cmake_macros/pgi_summit.cmake | 4 ---- cime_config/machines/cmake_macros/pgigpu_ascent.cmake | 3 --- cime_config/machines/cmake_macros/pgigpu_summit.cmake | 3 --- 18 files changed, 82 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_ascent.cmake b/cime_config/machines/cmake_macros/gnu_ascent.cmake index b8b8f360449e..4ed4fbd1657a 100644 --- a/cime_config/machines/cmake_macros/gnu_ascent.cmake +++ b/cime_config/machines/cmake_macros/gnu_ascent.cmake @@ -14,14 +14,8 @@ if (NOT MPILIB STREQUAL mpi-serial) string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") endif() string(APPEND CXX_LIBS " -lstdc++") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") -set(SCC "gcc") -set(SCXX "g++") -set(SFC "gfortran") set(PIO_FILESYSTEM_HINTS "gpfs") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/gnu_badger.cmake b/cime_config/machines/cmake_macros/gnu_badger.cmake index 11a986f03042..55439f8d6f00 100644 --- a/cime_config/machines/cmake_macros/gnu_badger.cmake +++ b/cime_config/machines/cmake_macros/gnu_badger.cmake @@ -1,10 +1,5 @@ set(PIO_FILESYSTEM_HINTS "lustre") -set(MPICC "mpicc") -set(MPIFC "mpif90") set(MPICXX "mpic++") -set(SFC "gfortran") -set(SCC "gcc") -set(SCXX "g++") execute_process(COMMAND nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") execute_process(COMMAND nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/gnu_cades.cmake b/cime_config/machines/cmake_macros/gnu_cades.cmake index 8855080770c9..ff449c1204a2 100644 --- a/cime_config/machines/cmake_macros/gnu_cades.cmake +++ b/cime_config/machines/cmake_macros/gnu_cades.cmake @@ -5,7 +5,6 @@ if (COMP_NAME STREQUAL cism) string(APPEND CMAKE_OPTS " -D CISM_GNU=ON") endif() string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16") -set(CXX_LINKER "FORTRAN") string(APPEND FC_AUTO_R8 " -fdefault-real-8") string(APPEND FFLAGS " -O -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fno-range-check") if (compile_threaded) @@ -24,10 +23,5 @@ if (compile_threaded) string(APPEND LDFLAGS " -fopenmp") endif() string(APPEND SLIBS " -L${NETCDF_PATH}/lib -Wl,-rpath=${NETCDF_PATH}/lib -lnetcdff -lnetcdf") -set(MPICC "mpicc") set(MPICXX "mpic++") -set(MPIFC "mpif90") -set(SCC "gcc") set(SCXX "gcpp") -set(SFC "gfortran") -set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/gnu_eddi.cmake b/cime_config/machines/cmake_macros/gnu_eddi.cmake index c532cd7c6500..87c041445a99 100644 --- a/cime_config/machines/cmake_macros/gnu_eddi.cmake +++ b/cime_config/machines/cmake_macros/gnu_eddi.cmake @@ -1,4 +1,3 @@ -set(SUPPORTS_CXX "TRUE") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY -DHAVE_BACKTRACE") endif() @@ -10,9 +9,3 @@ if (DEBUG) string(APPEND FFLAGS " -g -fbacktrace -fbounds-check -ffpe-trap=invalid,zero,overflow") endif() string(APPEND SLIBS " -L$ENV{NETCDF_HOME}/lib/ -lnetcdff -lnetcdf -lcurl -llapack -lblas") -if (MPILIB STREQUAL mpi-serial) - set(SCC "gcc") -endif() -if (MPILIB STREQUAL mpi-serial) - set(SFC "gfortran") -endif() diff --git a/cime_config/machines/cmake_macros/gnu_gcp.cmake b/cime_config/machines/cmake_macros/gnu_gcp.cmake index 034d6a8e0404..aff5e4278ce5 100644 --- a/cime_config/machines/cmake_macros/gnu_gcp.cmake +++ b/cime_config/machines/cmake_macros/gnu_gcp.cmake @@ -1,4 +1,3 @@ -set(SUPPORTS_CXX "TRUE") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY -DHAVE_BACKTRACE -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/gnu_grizzly.cmake b/cime_config/machines/cmake_macros/gnu_grizzly.cmake index 11a986f03042..55439f8d6f00 100644 --- a/cime_config/machines/cmake_macros/gnu_grizzly.cmake +++ b/cime_config/machines/cmake_macros/gnu_grizzly.cmake @@ -1,10 +1,5 @@ set(PIO_FILESYSTEM_HINTS "lustre") -set(MPICC "mpicc") -set(MPIFC "mpif90") set(MPICXX "mpic++") -set(SFC "gfortran") -set(SCC "gcc") -set(SCXX "g++") execute_process(COMMAND nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") execute_process(COMMAND nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/gnu_modex.cmake b/cime_config/machines/cmake_macros/gnu_modex.cmake index bec6c2421fa1..2bf9e326aa2f 100644 --- a/cime_config/machines/cmake_macros/gnu_modex.cmake +++ b/cime_config/machines/cmake_macros/gnu_modex.cmake @@ -1,4 +1,3 @@ -set(SUPPORTS_CXX "TRUE") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY -DHAVE_BACKTRACE") endif() diff --git a/cime_config/machines/cmake_macros/gnu_summit.cmake b/cime_config/machines/cmake_macros/gnu_summit.cmake index b8b8f360449e..4ed4fbd1657a 100644 --- a/cime_config/machines/cmake_macros/gnu_summit.cmake +++ b/cime_config/machines/cmake_macros/gnu_summit.cmake @@ -14,14 +14,8 @@ if (NOT MPILIB STREQUAL mpi-serial) string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") endif() string(APPEND CXX_LIBS " -lstdc++") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") -set(SCC "gcc") -set(SCXX "g++") -set(SFC "gfortran") set(PIO_FILESYSTEM_HINTS "gpfs") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake index a8b89d691827..90bfc9adeeeb 100644 --- a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake @@ -19,6 +19,5 @@ set(PIO_FILESYSTEM_HINTS "gpfs") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") string(APPEND CUDA_FLAGS " -O3 -arch sm_70 --use_fast_math") set(USE_CUDA "TRUE") diff --git a/cime_config/machines/cmake_macros/gnugpu_summit.cmake b/cime_config/machines/cmake_macros/gnugpu_summit.cmake index f47ff86f225e..c9c2d3d12ab2 100644 --- a/cime_config/machines/cmake_macros/gnugpu_summit.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_summit.cmake @@ -20,6 +20,5 @@ set(PIO_FILESYSTEM_HINTS "gpfs") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") -set(SUPPORTS_CXX "TRUE") string(APPEND CUDA_FLAGS " -O3 -arch sm_70 --use_fast_math") set(USE_CUDA "TRUE") diff --git a/cime_config/machines/cmake_macros/intel_badger.cmake b/cime_config/machines/cmake_macros/intel_badger.cmake index 5fa22043397c..4afb1e59ee8c 100644 --- a/cime_config/machines/cmake_macros/intel_badger.cmake +++ b/cime_config/machines/cmake_macros/intel_badger.cmake @@ -8,18 +8,9 @@ endif() if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() -if (NOT MPILIB STREQUAL impi) - set(MPICC "mpicc") -endif() if (NOT MPILIB STREQUAL impi) set(MPICXX "mpic++") endif() -if (NOT MPILIB STREQUAL impi) - set(MPIFC "mpif90") -endif() -set(SFC "ifort") -set(SCC "icc") -set(SCXX "icpc") execute_process(COMMAND $ENV{NETCDF_ROOT}/bin/nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") execute_process(COMMAND $ENV{NETCDF_ROOT}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/intel_grizzly.cmake b/cime_config/machines/cmake_macros/intel_grizzly.cmake index 5fa22043397c..4afb1e59ee8c 100644 --- a/cime_config/machines/cmake_macros/intel_grizzly.cmake +++ b/cime_config/machines/cmake_macros/intel_grizzly.cmake @@ -8,18 +8,9 @@ endif() if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() -if (NOT MPILIB STREQUAL impi) - set(MPICC "mpicc") -endif() if (NOT MPILIB STREQUAL impi) set(MPICXX "mpic++") endif() -if (NOT MPILIB STREQUAL impi) - set(MPIFC "mpif90") -endif() -set(SFC "ifort") -set(SCC "icc") -set(SCXX "icpc") execute_process(COMMAND $ENV{NETCDF_ROOT}/bin/nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") execute_process(COMMAND $ENV{NETCDF_ROOT}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/intel_itasca.cmake b/cime_config/machines/cmake_macros/intel_itasca.cmake index 3fe4c1aaf552..734d5d0d6ae5 100644 --- a/cime_config/machines/cmake_macros/intel_itasca.cmake +++ b/cime_config/machines/cmake_macros/intel_itasca.cmake @@ -5,7 +5,6 @@ endif() string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16") string(APPEND CPPDEFS " -DCPRINTEL") string(APPEND CXX_LDFLAGS " -cxxlib") -set(CXX_LINKER "FORTRAN") string(APPEND FC_AUTO_R8 " -r8") string(APPEND FFLAGS " -fp-model source -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") if (compile_threaded) @@ -25,8 +24,4 @@ endif() set(MPICC "mpiicc") set(MPICXX "mpiicpc") set(MPIFC "mpiifort") -set(SCC "icc") -set(SCXX "icpc") -set(SFC "ifort") string(APPEND SLIBS " -L/soft/netcdf/fortran-4.4-intel-sp1-update3-parallel/lib -lnetcdff -L/soft/hdf5/hdf5-1.8.13-intel-2013-sp1-update3-impi-5.0.0.028/lib -openmp -fPIC -lnetcdf -lnetcdf -L/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -lm") -set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/intel_stampede2.cmake b/cime_config/machines/cmake_macros/intel_stampede2.cmake index 0e8b56f8ce1c..8a289931ea42 100644 --- a/cime_config/machines/cmake_macros/intel_stampede2.cmake +++ b/cime_config/machines/cmake_macros/intel_stampede2.cmake @@ -10,9 +10,6 @@ if (NOT DEBUG) endif() string(APPEND FFLAGS " -xCORE-AVX2") set(HDF5_PATH "$ENV{TACC_HDF5_DIR}") -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") set(MPI_LIB_NAME "impi") if (MPILIB STREQUAL impi) set(NETCDF_PATH "$ENV{TACC_NETCDF_DIR}") @@ -24,9 +21,6 @@ set(PETSC_PATH "$ENV{PETSC_DIR}") if (MPILIB STREQUAL impi) set(PNETCDF_PATH "$ENV{TACC_PNETCDF_DIR}") endif() -set(SCC "icc") -set(SCXX "icpc") -set(SFC "ifort") if (MPILIB STREQUAL impi) string(APPEND SLIBS " -L${NETCDF_PATH} -lnetcdff -Wl,--as-needed,-L${NETCDF_PATH}/lib -lnetcdff -lnetcdf") endif() diff --git a/cime_config/machines/cmake_macros/pgi_ascent.cmake b/cime_config/machines/cmake_macros/pgi_ascent.cmake index 2390694d4ed5..3c60b57b70ad 100644 --- a/cime_config/machines/cmake_macros/pgi_ascent.cmake +++ b/cime_config/machines/cmake_macros/pgi_ascent.cmake @@ -10,12 +10,8 @@ endif() string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") set(CXX_LINKER "FORTRAN") string(APPEND CXX_LIBS " -lstdc++") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "pgcc") -set(SCXX "pgc++") set(SFC "pgfortran") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") diff --git a/cime_config/machines/cmake_macros/pgi_summit.cmake b/cime_config/machines/cmake_macros/pgi_summit.cmake index 2390694d4ed5..3c60b57b70ad 100644 --- a/cime_config/machines/cmake_macros/pgi_summit.cmake +++ b/cime_config/machines/cmake_macros/pgi_summit.cmake @@ -10,12 +10,8 @@ endif() string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") set(CXX_LINKER "FORTRAN") string(APPEND CXX_LIBS " -lstdc++") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "pgcc") -set(SCXX "pgc++") set(SFC "pgfortran") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") diff --git a/cime_config/machines/cmake_macros/pgigpu_ascent.cmake b/cime_config/machines/cmake_macros/pgigpu_ascent.cmake index 366c92ec2595..f3ba8dc978ed 100644 --- a/cime_config/machines/cmake_macros/pgigpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_ascent.cmake @@ -12,11 +12,8 @@ string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib - set(CXX_LINKER "FORTRAN") string(APPEND CXX_LIBS " -lstdc++") set(KOKKOS_OPTIONS "--arch=Power9,Volta70 --with-cuda=$ENV{CUDA_DIR} --with-cuda-options=enable_lambda") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "pgcc") set(SFC "pgfortran") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") diff --git a/cime_config/machines/cmake_macros/pgigpu_summit.cmake b/cime_config/machines/cmake_macros/pgigpu_summit.cmake index 366c92ec2595..f3ba8dc978ed 100644 --- a/cime_config/machines/cmake_macros/pgigpu_summit.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_summit.cmake @@ -12,11 +12,8 @@ string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib - set(CXX_LINKER "FORTRAN") string(APPEND CXX_LIBS " -lstdc++") set(KOKKOS_OPTIONS "--arch=Power9,Volta70 --with-cuda=$ENV{CUDA_DIR} --with-cuda-options=enable_lambda") -set(MPICC "mpicc") set(MPICXX "mpiCC") -set(MPIFC "mpif90") set(PIO_FILESYSTEM_HINTS "gpfs") -set(SCC "pgcc") set(SFC "pgfortran") set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}") set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}") From ee9eabe95ba8d69e4f7667d006bf1fa417325a24 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 11 Nov 2021 15:18:22 -0700 Subject: [PATCH 133/301] Remove redundant flag appends --- .../machines/cmake_macros/gnu_cades.cmake | 21 +------------------ .../machines/cmake_macros/gnu_eddi.cmake | 2 +- .../machines/cmake_macros/gnu_gcp.cmake | 4 ++-- .../machines/cmake_macros/gnu_modex.cmake | 2 +- .../machines/cmake_macros/ibmgpu_ascent.cmake | 2 +- .../machines/cmake_macros/ibmgpu_summit.cmake | 1 - cime_config/machines/cmake_macros/intel.cmake | 2 +- .../machines/cmake_macros/intel_anvil.cmake | 2 +- .../machines/cmake_macros/intel_bebop.cmake | 2 +- .../machines/cmake_macros/intel_cascade.cmake | 2 +- .../machines/cmake_macros/intel_compy.cmake | 8 +------ .../cmake_macros/intel_constance.cmake | 8 +------ .../cmake_macros/intel_cori-haswell.cmake | 15 +++---------- .../cmake_macros/intel_cori-knl.cmake | 15 +++---------- .../machines/cmake_macros/intel_ghost.cmake | 6 ------ .../machines/cmake_macros/intel_itasca.cmake | 16 ++------------ .../machines/cmake_macros/intel_mappy.cmake | 6 ------ .../machines/cmake_macros/intel_melvin.cmake | 6 ------ .../machines/cmake_macros/intel_quartz.cmake | 8 +------ .../cmake_macros/intel_sandiatoss3.cmake | 6 ------ .../machines/cmake_macros/intel_sooty.cmake | 8 +------ .../cmake_macros/intel_stampede2.cmake | 4 ++-- .../machines/cmake_macros/intel_syrah.cmake | 8 +------ .../machines/cmake_macros/intel_theta.cmake | 4 ++-- cime_config/machines/cmake_macros/nag.cmake | 3 --- .../machines/cmake_macros/nag_constance.cmake | 4 ++-- cime_config/machines/cmake_macros/pgi.cmake | 2 +- .../machines/cmake_macros/pgi_compy.cmake | 2 +- .../machines/cmake_macros/pgi_constance.cmake | 4 ++-- .../machines/cmake_macros/pgigpu.cmake | 2 +- 30 files changed, 34 insertions(+), 141 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_cades.cmake b/cime_config/machines/cmake_macros/gnu_cades.cmake index ff449c1204a2..ed34a30cb5f1 100644 --- a/cime_config/machines/cmake_macros/gnu_cades.cmake +++ b/cime_config/machines/cmake_macros/gnu_cades.cmake @@ -1,27 +1,8 @@ -if (compile_threaded) - string(APPEND CFLAGS " -fopenmp") -endif() -if (COMP_NAME STREQUAL cism) - string(APPEND CMAKE_OPTS " -D CISM_GNU=ON") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16") -string(APPEND FC_AUTO_R8 " -fdefault-real-8") -string(APPEND FFLAGS " -O -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fno-range-check") -if (compile_threaded) - string(APPEND FFLAGS " -fopenmp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -g -Wall") -endif() -string(APPEND FIXEDFLAGS " -ffixed-form") -string(APPEND FREEFLAGS " -ffree-form") +string(APPEND FFLAGS " -O -fno-range-check") set(HDF5_PATH "/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/hdf5-parallel/1.8.17/centos7.2_gnu5.3.0") set(NETCDF_PATH "/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/netcdf-hdf5parallel/4.3.3.1/centos7.2_gnu5.3.0") set(PNETCDF_PATH "/software/dev_tools/swtree/cs400_centos7.2_pe2016-08/pnetcdf/1.9.0/centos7.2_gnu5.3.0") set(LAPACK_LIBDIR "/software/tools/compilers/intel_2017/mkl/lib/intel64") -if (compile_threaded) - string(APPEND LDFLAGS " -fopenmp") -endif() string(APPEND SLIBS " -L${NETCDF_PATH}/lib -Wl,-rpath=${NETCDF_PATH}/lib -lnetcdff -lnetcdf") set(MPICXX "mpic++") set(SCXX "gcpp") diff --git a/cime_config/machines/cmake_macros/gnu_eddi.cmake b/cime_config/machines/cmake_macros/gnu_eddi.cmake index 87c041445a99..d0fbe69155b4 100644 --- a/cime_config/machines/cmake_macros/gnu_eddi.cmake +++ b/cime_config/machines/cmake_macros/gnu_eddi.cmake @@ -6,6 +6,6 @@ if (NOT DEBUG) string(APPEND FFLAGS " -fno-unsafe-math-optimizations") endif() if (DEBUG) - string(APPEND FFLAGS " -g -fbacktrace -fbounds-check -ffpe-trap=invalid,zero,overflow") + string(APPEND FFLAGS " -fbounds-check -ffpe-trap=invalid,zero,overflow") endif() string(APPEND SLIBS " -L$ENV{NETCDF_HOME}/lib/ -lnetcdff -lnetcdf -lcurl -llapack -lblas") diff --git a/cime_config/machines/cmake_macros/gnu_gcp.cmake b/cime_config/machines/cmake_macros/gnu_gcp.cmake index aff5e4278ce5..537fb69b4c67 100644 --- a/cime_config/machines/cmake_macros/gnu_gcp.cmake +++ b/cime_config/machines/cmake_macros/gnu_gcp.cmake @@ -6,11 +6,11 @@ if (NOT DEBUG) string(APPEND FFLAGS " -fno-unsafe-math-optimizations") endif() if (DEBUG) - string(APPEND FFLAGS " -g -fbacktrace -fbounds-check -ffpe-trap=invalid,zero,overflow") + string(APPEND FFLAGS " -fbounds-check -ffpe-trap=invalid,zero,overflow") endif() string(APPEND SLIBS " -L$ENV{HDF5_PATH}/lib -lhdf5_fortran -lhdf5 -lhdf5_hl -lhdf5hl_fortran") string(APPEND SLIBS " -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff") -string(APPEND SLIBS " -L$ENV{CURL_PATH}/lib -lcurl -lblas -llapack") +string(APPEND SLIBS " -L$ENV{CURL_PATH}/lib -lcurl") string(APPEND SLIBS " -L$ENV{LAPACK_PATH}/lib64 -lblas -llapack") string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf") set(HDF5_PATH "$ENV{HDF5_PATH}") diff --git a/cime_config/machines/cmake_macros/gnu_modex.cmake b/cime_config/machines/cmake_macros/gnu_modex.cmake index 2bf9e326aa2f..a498aa767fcc 100644 --- a/cime_config/machines/cmake_macros/gnu_modex.cmake +++ b/cime_config/machines/cmake_macros/gnu_modex.cmake @@ -5,7 +5,7 @@ if (NOT DEBUG) string(APPEND FFLAGS " -fno-unsafe-math-optimizations") endif() if (DEBUG) - string(APPEND FFLAGS " -g -fbacktrace -fbounds-check -ffpe-trap=invalid,zero,overflow") + string(APPEND FFLAGS " -fbounds-check -ffpe-trap=invalid,zero,overflow") endif() string(APPEND SLIBS " -L$ENV{HDF5_HOME}/lib -lhdf5_fortran -lhdf5 -lhdf5_hl -lhdf5hl_fortran") string(APPEND SLIBS " -L$ENV{NETCDF_PATH}/lib/ -lnetcdff -lnetcdf -lcurl -lblas -llapack") diff --git a/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake b/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake index 5397658dc89a..53a9f69b2c75 100644 --- a/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/ibmgpu_ascent.cmake @@ -1,4 +1,4 @@ if (DEBUG) string(APPEND FFLAGS " -qinitauto=7FF7FFFF -qflttrap=ov:zero:inv:en") endif() -string(APPEND CXX_LIBS " -L/sw/ascent/gcc/8.1.1/lib64 -lstdc++ -L$ENV{OLCF_XLC_ROOT}/lib -libmc++") +string(APPEND CXX_LIBS " -L/sw/ascent/gcc/8.1.1/lib64") diff --git a/cime_config/machines/cmake_macros/ibmgpu_summit.cmake b/cime_config/machines/cmake_macros/ibmgpu_summit.cmake index f3e965549d0c..e69de29bb2d1 100644 --- a/cime_config/machines/cmake_macros/ibmgpu_summit.cmake +++ b/cime_config/machines/cmake_macros/ibmgpu_summit.cmake @@ -1 +0,0 @@ -string(APPEND CXX_LIBS " -L/sw/summit/gcc/8.1.1/lib64 -lstdc++ -L$ENV{OLCF_XLC_ROOT}/lib -libmc++") diff --git a/cime_config/machines/cmake_macros/intel.cmake b/cime_config/machines/cmake_macros/intel.cmake index 8e02e70633b5..5b6f8ed751ca 100644 --- a/cime_config/machines/cmake_macros/intel.cmake +++ b/cime_config/machines/cmake_macros/intel.cmake @@ -1,4 +1,4 @@ -string(APPEND CFLAGS " -O2 -fp-model precise -std=gnu99") +string(APPEND CFLAGS " -fp-model precise -std=gnu99") if (compile_threaded) string(APPEND CFLAGS " -qopenmp") endif() diff --git a/cime_config/machines/cmake_macros/intel_anvil.cmake b/cime_config/machines/cmake_macros/intel_anvil.cmake index b10916dd7eb5..0bcb502c9fa7 100644 --- a/cime_config/machines/cmake_macros/intel_anvil.cmake +++ b/cime_config/machines/cmake_macros/intel_anvil.cmake @@ -8,7 +8,7 @@ if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") + string(APPEND FFLAGS " -qno-opt-dynamic-align") endif() execute_process(COMMAND $ENV{NETCDF_FORTRAN_PATH}/bin/nf-config --fflags OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND FFLAGS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0}") diff --git a/cime_config/machines/cmake_macros/intel_bebop.cmake b/cime_config/machines/cmake_macros/intel_bebop.cmake index 23f214dcd793..93415d4b7ff6 100644 --- a/cime_config/machines/cmake_macros/intel_bebop.cmake +++ b/cime_config/machines/cmake_macros/intel_bebop.cmake @@ -4,7 +4,7 @@ if (COMP_NAME STREQUAL gptl) endif() string(APPEND CXX_LIBS " -lstdc++") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") + string(APPEND FFLAGS " -qno-opt-dynamic-align") endif() if (MPILIB STREQUAL impi) set(MPICC "mpiicc") diff --git a/cime_config/machines/cmake_macros/intel_cascade.cmake b/cime_config/machines/cmake_macros/intel_cascade.cmake index 69126268f60a..ba947116b98a 100644 --- a/cime_config/machines/cmake_macros/intel_cascade.cmake +++ b/cime_config/machines/cmake_macros/intel_cascade.cmake @@ -1,7 +1,7 @@ string(APPEND CONFIG_ARGS " --enable-filesystem-hints=lustre") string(APPEND CPPDEFS " -DLINUX") if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv") + string(APPEND FFLAGS " -check all -ftrapuv") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/intel_compy.cmake b/cime_config/machines/cmake_macros/intel_compy.cmake index fd386eeb4e8b..077e3a3b515a 100644 --- a/cime_config/machines/cmake_macros/intel_compy.cmake +++ b/cime_config/machines/cmake_macros/intel_compy.cmake @@ -1,15 +1,9 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() if (COMP_NAME STREQUAL gptl) string(APPEND CFLAGS " -DHAVE_SLASHPROC") endif() string(APPEND CPPDEFS " -DLINUX") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv -init=snan") + string(APPEND FFLAGS " -checkd all -ftrapuv -init=snan") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PNETCDF_PATH "$ENV{PNETCDF_HOME}") diff --git a/cime_config/machines/cmake_macros/intel_constance.cmake b/cime_config/machines/cmake_macros/intel_constance.cmake index e0854ab76606..431c1b93d63f 100644 --- a/cime_config/machines/cmake_macros/intel_constance.cmake +++ b/cime_config/machines/cmake_macros/intel_constance.cmake @@ -1,12 +1,6 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CPPDEFS " -DLINUX") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv") + string(APPEND FFLAGS " -check all -ftrapuv") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake index 89012b1f73f3..12339b5302ac 100644 --- a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake @@ -1,18 +1,9 @@ set(ALBANY_PATH "/global/cfs/cdirs/e3sm/software/albany-trilinos/albany-install-2020-08-07") -string(APPEND FFLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model consistent -fimf-use-svml") +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") -endif() -string(APPEND CXXFLAGS " -std=c++14 -fp-model consistent") -if (compile_threaded) - string(APPEND CXXFLAGS " -qopenmp") -endif() -if (DEBUG) - string(APPEND CXXFLAGS " -O0 -g") -endif() -if (NOT DEBUG) - string(APPEND CXXFLAGS " -O2") + string(APPEND FFLAGS " -qno-opt-dynamic-align") endif() +string(APPEND CXXFLAGS " -fp-model consistent") set(PETSC_PATH "$ENV{PETSC_DIR}") set(SCC "icc") set(SCXX "icpc") diff --git a/cime_config/machines/cmake_macros/intel_cori-knl.cmake b/cime_config/machines/cmake_macros/intel_cori-knl.cmake index a82be9c82bed..8e88c517a5c6 100644 --- a/cime_config/machines/cmake_macros/intel_cori-knl.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-knl.cmake @@ -4,24 +4,15 @@ if (MPILIB STREQUAL impi) string(APPEND CFLAGS " -axMIC-AVX512 -xCORE-AVX2") endif() string(APPEND CPPDEFS " -DARCH_MIC_KNL") -string(APPEND FFLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model consistent -fimf-use-svml") +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") + string(APPEND FFLAGS " -qno-opt-dynamic-align") endif() if (MPILIB STREQUAL impi) string(APPEND FFLAGS " -xMIC-AVX512") endif() string(APPEND FFLAGS " -DHAVE_ERF_INTRINSICS") -string(APPEND CXXFLAGS " -std=c++14 -fp-model consistent") -if (compile_threaded) - string(APPEND CXXFLAGS " -qopenmp") -endif() -if (DEBUG) - string(APPEND CXXFLAGS " -O0 -g") -endif() -if (NOT DEBUG) - string(APPEND CXXFLAGS " -O2") -endif() +string(APPEND CXXFLAGS " -fp-model consistent") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") endif() diff --git a/cime_config/machines/cmake_macros/intel_ghost.cmake b/cime_config/machines/cmake_macros/intel_ghost.cmake index d000931ba77f..3bf190ad60c6 100644 --- a/cime_config/machines/cmake_macros/intel_ghost.cmake +++ b/cime_config/machines/cmake_macros/intel_ghost.cmake @@ -1,11 +1,5 @@ set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install") -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (MPILIB STREQUAL openmpi) set(MPI_PATH "/opt/openmpi-1.8-intel") endif() diff --git a/cime_config/machines/cmake_macros/intel_itasca.cmake b/cime_config/machines/cmake_macros/intel_itasca.cmake index 734d5d0d6ae5..d309de55d937 100644 --- a/cime_config/machines/cmake_macros/intel_itasca.cmake +++ b/cime_config/machines/cmake_macros/intel_itasca.cmake @@ -1,23 +1,11 @@ -string(APPEND CFLAGS " -O2 -fp-model precise -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") +string(APPEND CFLAGS " -O2 -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") if (compile_threaded) string(APPEND CFLAGS " -openmp") endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16") -string(APPEND CPPDEFS " -DCPRINTEL") -string(APPEND CXX_LDFLAGS " -cxxlib") -string(APPEND FC_AUTO_R8 " -r8") -string(APPEND FFLAGS " -fp-model source -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") +string(APPEND FFLAGS " -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") if (compile_threaded) string(APPEND FFLAGS " -openmp") endif() -if (DEBUG) - string(APPEND FFLAGS " -O0 -g -check uninit -check bounds -check pointers -fpe0") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() -string(APPEND FIXEDFLAGS " -fixed -132") -string(APPEND FREEFLAGS " -free") if (compile_threaded) string(APPEND LDFLAGS " -openmp") endif() diff --git a/cime_config/machines/cmake_macros/intel_mappy.cmake b/cime_config/machines/cmake_macros/intel_mappy.cmake index f8915561161a..64dd290052df 100644 --- a/cime_config/machines/cmake_macros/intel_mappy.cmake +++ b/cime_config/machines/cmake_macros/intel_mappy.cmake @@ -2,13 +2,7 @@ set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CXX_LIBS " -lstdc++ -lmpi_cxx") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() set(NETCDF_PATH "$ENV{NETCDFROOT}") set(PNETCDF_PATH "$ENV{PNETCDFROOT}") execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/intel_melvin.cmake b/cime_config/machines/cmake_macros/intel_melvin.cmake index 6f345b0800a5..dd58c586d01c 100644 --- a/cime_config/machines/cmake_macros/intel_melvin.cmake +++ b/cime_config/machines/cmake_macros/intel_melvin.cmake @@ -2,13 +2,7 @@ set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CXX_LIBS " -lstdc++ -lmpi_cxx") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() set(NETCDF_PATH "$ENV{NETCDFROOT}") set(PNETCDF_PATH "$ENV{PNETCDFROOT}") execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/intel_quartz.cmake b/cime_config/machines/cmake_macros/intel_quartz.cmake index aa851129bb25..a899ab38f87a 100644 --- a/cime_config/machines/cmake_macros/intel_quartz.cmake +++ b/cime_config/machines/cmake_macros/intel_quartz.cmake @@ -1,12 +1,6 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv") + string(APPEND FFLAGS " -check all -ftrapuv") endif() string(APPEND SLIBS " -llapack -lblas") set(MPI_LIB_NAME "mpich") diff --git a/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake b/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake index 317bd3a6d69d..60df91218402 100644 --- a/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake +++ b/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake @@ -2,13 +2,7 @@ set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (MPILIB STREQUAL openmpi) set(MPI_PATH "$ENV{MPIHOME}") endif() diff --git a/cime_config/machines/cmake_macros/intel_sooty.cmake b/cime_config/machines/cmake_macros/intel_sooty.cmake index 964bbec837e1..8773085262ee 100644 --- a/cime_config/machines/cmake_macros/intel_sooty.cmake +++ b/cime_config/machines/cmake_macros/intel_sooty.cmake @@ -1,12 +1,6 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CPPDEFS " -DLINUX") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv -init=snan") + string(APPEND FFLAGS " -check all -ftrapuv -init=snan") endif() set(NETCDF_PATH "$ENV{NETCDF_PATH}") set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/intel_stampede2.cmake b/cime_config/machines/cmake_macros/intel_stampede2.cmake index 8a289931ea42..355213f8ef48 100644 --- a/cime_config/machines/cmake_macros/intel_stampede2.cmake +++ b/cime_config/machines/cmake_macros/intel_stampede2.cmake @@ -4,9 +4,9 @@ if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() string(APPEND CPPDEFS " -DARCH_MIC_KNL") -string(APPEND FFLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model consistent -fimf-use-svml") +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") + string(APPEND FFLAGS " -qno-opt-dynamic-align") endif() string(APPEND FFLAGS " -xCORE-AVX2") set(HDF5_PATH "$ENV{TACC_HDF5_DIR}") diff --git a/cime_config/machines/cmake_macros/intel_syrah.cmake b/cime_config/machines/cmake_macros/intel_syrah.cmake index aa851129bb25..a899ab38f87a 100644 --- a/cime_config/machines/cmake_macros/intel_syrah.cmake +++ b/cime_config/machines/cmake_macros/intel_syrah.cmake @@ -1,12 +1,6 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O2") -endif() string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") -if (NOT DEBUG) - string(APPEND FFLAGS " -O2") -endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv") + string(APPEND FFLAGS " -check all -ftrapuv") endif() string(APPEND SLIBS " -llapack -lblas") set(MPI_LIB_NAME "mpich") diff --git a/cime_config/machines/cmake_macros/intel_theta.cmake b/cime_config/machines/cmake_macros/intel_theta.cmake index 23ea051aafc7..cee438757107 100644 --- a/cime_config/machines/cmake_macros/intel_theta.cmake +++ b/cime_config/machines/cmake_macros/intel_theta.cmake @@ -1,8 +1,8 @@ string(APPEND CONFIG_ARGS " --host=cray") string(APPEND CPPDEFS " -DARCH_MIC_KNL") -string(APPEND FFLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model consistent -fimf-use-svml=true") +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml=true") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align -fp-speculation=off") + string(APPEND FFLAGS " -qno-opt-dynamic-align -fp-speculation=off") endif() string(APPEND FFLAGS " -DHAVE_ERF_INTRINSICS") set(SCC "icc") diff --git a/cime_config/machines/cmake_macros/nag.cmake b/cime_config/machines/cmake_macros/nag.cmake index 4a14dac99db4..9f9a8d77aa9a 100644 --- a/cime_config/machines/cmake_macros/nag.cmake +++ b/cime_config/machines/cmake_macros/nag.cmake @@ -8,9 +8,6 @@ string(APPEND FFLAGS " -wmismatch=mpi_send,mpi_recv,mpi_bcast,mpi_allreduce,mpi_ if (NOT DEBUG) string(APPEND FFLAGS " -ieee=full -O2") endif() -if (DEBUG) - string(APPEND FFLAGS " -g -time -f2003 -ieee=stop") -endif() if (DEBUG) string(APPEND FFLAGS " -C=all -g -time -f2003 -ieee=stop") endif() diff --git a/cime_config/machines/cmake_macros/nag_constance.cmake b/cime_config/machines/cmake_macros/nag_constance.cmake index 27dd36ae2c7f..8db30816e101 100644 --- a/cime_config/machines/cmake_macros/nag_constance.cmake +++ b/cime_config/machines/cmake_macros/nag_constance.cmake @@ -3,10 +3,10 @@ if (NOT DEBUG) endif() string(APPEND CPPDEFS " -DLINUX") if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -kind=byte") + string(APPEND FFLAGS " -kind=byte") endif() if (DEBUG) - string(APPEND FFLAGS " -C=all -g -O0 -v") + string(APPEND FFLAGS " -O0 -v") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/pgi.cmake b/cime_config/machines/cmake_macros/pgi.cmake index 43ce50ef8c59..a7bdf9244bb6 100644 --- a/cime_config/machines/cmake_macros/pgi.cmake +++ b/cime_config/machines/cmake_macros/pgi.cmake @@ -14,7 +14,7 @@ if (compile_threaded) string(APPEND FFLAGS " -mp") endif() if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") + string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds") endif() if (COMP_NAME STREQUAL datm) string(APPEND FFLAGS " -Mnovect") diff --git a/cime_config/machines/cmake_macros/pgi_compy.cmake b/cime_config/machines/cmake_macros/pgi_compy.cmake index f2e75b503cf6..d374760eb66a 100644 --- a/cime_config/machines/cmake_macros/pgi_compy.cmake +++ b/cime_config/machines/cmake_macros/pgi_compy.cmake @@ -11,7 +11,7 @@ if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (DEBUG) - string(APPEND FFLAGS " -C -Mbounds -traceback -Mchkfpstk -Mchkstk -Mdalign -Mdepchk -Mextend -Miomutex -Mrecursive -Ktrap=fp -O0 -g -byteswapio -Meh_frame") + string(APPEND FFLAGS " -C -traceback -Mchkfpstk -Mchkstk -Mdalign -Mdepchk -Miomutex -Mrecursive -Meh_frame") endif() if (COMP_NAME STREQUAL eam) string(APPEND FFLAGS " -Mnovect") diff --git a/cime_config/machines/cmake_macros/pgi_constance.cmake b/cime_config/machines/cmake_macros/pgi_constance.cmake index bbd49b9fb255..5acc9653567f 100644 --- a/cime_config/machines/cmake_macros/pgi_constance.cmake +++ b/cime_config/machines/cmake_macros/pgi_constance.cmake @@ -6,10 +6,10 @@ if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (DEBUG) - string(APPEND FFLAGS " -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv") + string(APPEND FFLAGS " -traceback -fpe0 -check all -check noarg_temp_created -ftrapuv") endif() if (DEBUG) - string(APPEND FFLAGS " -C -Mbounds -traceback -Mchkfpstk -Mchkstk -Mdalign -Mdepchk -Mextend -Miomutex -Mrecursive -Ktrap=fp -O0 -g -byteswapio -Meh_frame") + string(APPEND FFLAGS " -C -Mchkfpstk -Mchkstk -Mdalign -Mdepchk -Miomutex -Mrecursive -Meh_frame") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/pgigpu.cmake b/cime_config/machines/cmake_macros/pgigpu.cmake index d7e94a907612..bb28af9a00e3 100644 --- a/cime_config/machines/cmake_macros/pgigpu.cmake +++ b/cime_config/machines/cmake_macros/pgigpu.cmake @@ -17,7 +17,7 @@ if (COMP_NAME STREQUAL eam) string(APPEND FFLAGS " ") endif() if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") + string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds") endif() if (COMP_NAME STREQUAL datm) string(APPEND FFLAGS " -Mnovect") From ffd50f7bf8db9030bc3c606c74d98a8c5d8fab5f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 11 Nov 2021 15:33:53 -0700 Subject: [PATCH 134/301] Merge if statements --- cime_config/machines/cmake_macros/cray_theta.cmake | 2 -- cime_config/machines/cmake_macros/gnu_ascent.cmake | 2 -- cime_config/machines/cmake_macros/gnu_cori-haswell.cmake | 2 -- cime_config/machines/cmake_macros/gnu_cori-knl.cmake | 2 -- cime_config/machines/cmake_macros/gnu_jlse.cmake | 2 -- cime_config/machines/cmake_macros/gnu_summit.cmake | 2 -- cime_config/machines/cmake_macros/gnugpu_ascent.cmake | 2 -- cime_config/machines/cmake_macros/gnugpu_summit.cmake | 2 -- cime_config/machines/cmake_macros/intel_anvil.cmake | 6 ------ cime_config/machines/cmake_macros/intel_badger.cmake | 4 ---- cime_config/machines/cmake_macros/intel_bebop.cmake | 4 ---- cime_config/machines/cmake_macros/intel_chrysalis.cmake | 4 ---- cime_config/machines/cmake_macros/intel_compy.cmake | 4 ---- cime_config/machines/cmake_macros/intel_cori-knl.cmake | 6 ------ cime_config/machines/cmake_macros/intel_grizzly.cmake | 4 ---- cime_config/machines/cmake_macros/intel_itasca.cmake | 2 -- cime_config/machines/cmake_macros/intel_stampede2.cmake | 2 -- cime_config/machines/cmake_macros/pgi_ascent.cmake | 2 -- cime_config/machines/cmake_macros/pgi_compy.cmake | 4 ---- cime_config/machines/cmake_macros/pgi_constance.cmake | 2 -- cime_config/machines/cmake_macros/pgi_summit.cmake | 2 -- cime_config/machines/cmake_macros/pgigpu_blues.cmake | 2 -- cime_config/machines/cmake_macros/pgigpu_swing.cmake | 2 -- 23 files changed, 66 deletions(-) diff --git a/cime_config/machines/cmake_macros/cray_theta.cmake b/cime_config/machines/cmake_macros/cray_theta.cmake index 54ba99a09c64..a1681161cb59 100644 --- a/cime_config/machines/cmake_macros/cray_theta.cmake +++ b/cime_config/machines/cmake_macros/cray_theta.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O1") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O1") endif() if (COMP_NAME STREQUAL eam) diff --git a/cime_config/machines/cmake_macros/gnu_ascent.cmake b/cime_config/machines/cmake_macros/gnu_ascent.cmake index 4ed4fbd1657a..44d98eb8806a 100644 --- a/cime_config/machines/cmake_macros/gnu_ascent.cmake +++ b/cime_config/machines/cmake_macros/gnu_ascent.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/gnu_cori-haswell.cmake b/cime_config/machines/cmake_macros/gnu_cori-haswell.cmake index 044bd144900a..5b43091851b9 100644 --- a/cime_config/machines/cmake_macros/gnu_cori-haswell.cmake +++ b/cime_config/machines/cmake_macros/gnu_cori-haswell.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() string(APPEND CXX_LIBS " -lstdc++") diff --git a/cime_config/machines/cmake_macros/gnu_cori-knl.cmake b/cime_config/machines/cmake_macros/gnu_cori-knl.cmake index 044bd144900a..5b43091851b9 100644 --- a/cime_config/machines/cmake_macros/gnu_cori-knl.cmake +++ b/cime_config/machines/cmake_macros/gnu_cori-knl.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() string(APPEND CXX_LIBS " -lstdc++") diff --git a/cime_config/machines/cmake_macros/gnu_jlse.cmake b/cime_config/machines/cmake_macros/gnu_jlse.cmake index 6ea2fec384e0..ad9a45123623 100644 --- a/cime_config/machines/cmake_macros/gnu_jlse.cmake +++ b/cime_config/machines/cmake_macros/gnu_jlse.cmake @@ -1,8 +1,6 @@ string(APPEND CPPDEFS " -DHAVE_SLASHPROC") if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() execute_process(COMMAND $ENV{NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/gnu_summit.cmake b/cime_config/machines/cmake_macros/gnu_summit.cmake index 4ed4fbd1657a..44d98eb8806a 100644 --- a/cime_config/machines/cmake_macros/gnu_summit.cmake +++ b/cime_config/machines/cmake_macros/gnu_summit.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake index 90bfc9adeeeb..31a258d69987 100644 --- a/cime_config/machines/cmake_macros/gnugpu_ascent.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_ascent.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/gnugpu_summit.cmake b/cime_config/machines/cmake_macros/gnugpu_summit.cmake index c9c2d3d12ab2..369a9e63efed 100644 --- a/cime_config/machines/cmake_macros/gnugpu_summit.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_summit.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/intel_anvil.cmake b/cime_config/machines/cmake_macros/intel_anvil.cmake index 0bcb502c9fa7..2f942af42418 100644 --- a/cime_config/machines/cmake_macros/intel_anvil.cmake +++ b/cime_config/machines/cmake_macros/intel_anvil.cmake @@ -20,17 +20,11 @@ if (compile_threaded AND DEBUG) endif() if (compile_threaded) string(APPEND FFLAGS_NOOPT " -static-intel") -endif() -if (compile_threaded) string(APPEND LDFLAGS " -static-intel") endif() if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/intel_badger.cmake b/cime_config/machines/cmake_macros/intel_badger.cmake index 4afb1e59ee8c..8f7a2d0eea7e 100644 --- a/cime_config/machines/cmake_macros/intel_badger.cmake +++ b/cime_config/machines/cmake_macros/intel_badger.cmake @@ -1,11 +1,7 @@ set(PIO_FILESYSTEM_HINTS "lustre") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() if (NOT MPILIB STREQUAL impi) diff --git a/cime_config/machines/cmake_macros/intel_bebop.cmake b/cime_config/machines/cmake_macros/intel_bebop.cmake index 93415d4b7ff6..7221c6f7b8d5 100644 --- a/cime_config/machines/cmake_macros/intel_bebop.cmake +++ b/cime_config/machines/cmake_macros/intel_bebop.cmake @@ -8,11 +8,7 @@ if (NOT DEBUG) endif() if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() execute_process(COMMAND nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cime_config/machines/cmake_macros/intel_chrysalis.cmake b/cime_config/machines/cmake_macros/intel_chrysalis.cmake index 73f96f2cf7ec..0d9f3a32b8e5 100644 --- a/cime_config/machines/cmake_macros/intel_chrysalis.cmake +++ b/cime_config/machines/cmake_macros/intel_chrysalis.cmake @@ -26,10 +26,6 @@ set(PNETCDF_PATH "$ENV{PNETCDF_PATH}") string(APPEND LDFLAGS " -static-intel") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() diff --git a/cime_config/machines/cmake_macros/intel_compy.cmake b/cime_config/machines/cmake_macros/intel_compy.cmake index 077e3a3b515a..9e4a6870fd51 100644 --- a/cime_config/machines/cmake_macros/intel_compy.cmake +++ b/cime_config/machines/cmake_macros/intel_compy.cmake @@ -11,10 +11,6 @@ set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND SLIBS " -lpmi -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff -L$ENV{MKL_PATH}/lib/intel64/ -lmkl_rt $ENV{PNETCDF_LIBRARIES}") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() diff --git a/cime_config/machines/cmake_macros/intel_cori-knl.cmake b/cime_config/machines/cmake_macros/intel_cori-knl.cmake index 8e88c517a5c6..fc96aa86b82f 100644 --- a/cime_config/machines/cmake_macros/intel_cori-knl.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-knl.cmake @@ -15,14 +15,8 @@ string(APPEND FFLAGS " -DHAVE_ERF_INTRINSICS") string(APPEND CXXFLAGS " -fp-model consistent") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") -endif() -if (MPILIB STREQUAL impi) set(MPI_LIB_NAME "impi") endif() set(PETSC_PATH "$ENV{PETSC_DIR}") diff --git a/cime_config/machines/cmake_macros/intel_grizzly.cmake b/cime_config/machines/cmake_macros/intel_grizzly.cmake index 4afb1e59ee8c..8f7a2d0eea7e 100644 --- a/cime_config/machines/cmake_macros/intel_grizzly.cmake +++ b/cime_config/machines/cmake_macros/intel_grizzly.cmake @@ -1,11 +1,7 @@ set(PIO_FILESYSTEM_HINTS "lustre") if (MPILIB STREQUAL impi) set(MPICC "mpiicc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpiicpc") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpiifort") endif() if (NOT MPILIB STREQUAL impi) diff --git a/cime_config/machines/cmake_macros/intel_itasca.cmake b/cime_config/machines/cmake_macros/intel_itasca.cmake index d309de55d937..e6838280e61d 100644 --- a/cime_config/machines/cmake_macros/intel_itasca.cmake +++ b/cime_config/machines/cmake_macros/intel_itasca.cmake @@ -5,8 +5,6 @@ endif() string(APPEND FFLAGS " -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include") if (compile_threaded) string(APPEND FFLAGS " -openmp") -endif() -if (compile_threaded) string(APPEND LDFLAGS " -openmp") endif() set(MPICC "mpiicc") diff --git a/cime_config/machines/cmake_macros/intel_stampede2.cmake b/cime_config/machines/cmake_macros/intel_stampede2.cmake index 355213f8ef48..8cc5f38147e3 100644 --- a/cime_config/machines/cmake_macros/intel_stampede2.cmake +++ b/cime_config/machines/cmake_macros/intel_stampede2.cmake @@ -20,8 +20,6 @@ endif() set(PETSC_PATH "$ENV{PETSC_DIR}") if (MPILIB STREQUAL impi) set(PNETCDF_PATH "$ENV{TACC_PNETCDF_DIR}") -endif() -if (MPILIB STREQUAL impi) string(APPEND SLIBS " -L${NETCDF_PATH} -lnetcdff -Wl,--as-needed,-L${NETCDF_PATH}/lib -lnetcdff -lnetcdf") endif() if (MPILIB STREQUAL mpi-serial) diff --git a/cime_config/machines/cmake_macros/pgi_ascent.cmake b/cime_config/machines/cmake_macros/pgi_ascent.cmake index 3c60b57b70ad..090a1b86f250 100644 --- a/cime_config/machines/cmake_macros/pgi_ascent.cmake +++ b/cime_config/machines/cmake_macros/pgi_ascent.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2 -Mvect=nosimd") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2 -DSUMMITDEV_PGI") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/pgi_compy.cmake b/cime_config/machines/cmake_macros/pgi_compy.cmake index d374760eb66a..2d59aaa0dd05 100644 --- a/cime_config/machines/cmake_macros/pgi_compy.cmake +++ b/cime_config/machines/cmake_macros/pgi_compy.cmake @@ -25,11 +25,7 @@ set(PNETCDF_PATH "$ENV{PNETCDF_HOME}") string(APPEND SLIBS " -lpmi -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff -L$ENV{MKL_PATH}/lib/intel64/ -lmkl_rt $ENV{PNETCDF_LIBRARIES}") if (MPILIB STREQUAL impi) set(MPICC "mpipgcc") -endif() -if (MPILIB STREQUAL impi) set(MPICXX "mpipgcxx") -endif() -if (MPILIB STREQUAL impi) set(MPIFC "mpipgf90") endif() set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/pgi_constance.cmake b/cime_config/machines/cmake_macros/pgi_constance.cmake index 5acc9653567f..112bc32fb334 100644 --- a/cime_config/machines/cmake_macros/pgi_constance.cmake +++ b/cime_config/machines/cmake_macros/pgi_constance.cmake @@ -7,8 +7,6 @@ if (NOT DEBUG) endif() if (DEBUG) string(APPEND FFLAGS " -traceback -fpe0 -check all -check noarg_temp_created -ftrapuv") -endif() -if (DEBUG) string(APPEND FFLAGS " -C -Mchkfpstk -Mchkstk -Mdalign -Mdepchk -Miomutex -Mrecursive -Meh_frame") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") diff --git a/cime_config/machines/cmake_macros/pgi_summit.cmake b/cime_config/machines/cmake_macros/pgi_summit.cmake index 3c60b57b70ad..090a1b86f250 100644 --- a/cime_config/machines/cmake_macros/pgi_summit.cmake +++ b/cime_config/machines/cmake_macros/pgi_summit.cmake @@ -1,7 +1,5 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2 -Mvect=nosimd") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2 -DSUMMITDEV_PGI") endif() if (COMP_NAME STREQUAL gptl) diff --git a/cime_config/machines/cmake_macros/pgigpu_blues.cmake b/cime_config/machines/cmake_macros/pgigpu_blues.cmake index dfb36fce22fb..955639298c63 100644 --- a/cime_config/machines/cmake_macros/pgigpu_blues.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_blues.cmake @@ -4,8 +4,6 @@ endif() set(PIO_FILESYSTEM_HINTS "gpfs") if (NOT DEBUG) string(APPEND CFLAGS " -O2 -Mvect=nosimd") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2 -Mvect=nosimd -DSUMMITDEV_PGI") endif() string(APPEND LDFLAGS " -Minline -ta=tesla:ccall,fastmath,loadcache:L1,unroll,fma,managed,deepcopy,nonvvm -Mcuda -Minfo=accel") diff --git a/cime_config/machines/cmake_macros/pgigpu_swing.cmake b/cime_config/machines/cmake_macros/pgigpu_swing.cmake index dfb36fce22fb..955639298c63 100644 --- a/cime_config/machines/cmake_macros/pgigpu_swing.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_swing.cmake @@ -4,8 +4,6 @@ endif() set(PIO_FILESYSTEM_HINTS "gpfs") if (NOT DEBUG) string(APPEND CFLAGS " -O2 -Mvect=nosimd") -endif() -if (NOT DEBUG) string(APPEND FFLAGS " -O2 -Mvect=nosimd -DSUMMITDEV_PGI") endif() string(APPEND LDFLAGS " -Minline -ta=tesla:ccall,fastmath,loadcache:L1,unroll,fma,managed,deepcopy,nonvvm -Mcuda -Minfo=accel") From 06fb892588a29906effe8f85bf208b72b69e16ea Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 11 Nov 2021 15:34:38 -0700 Subject: [PATCH 135/301] evaluator: Lots of enhancements --- cime_config/machines/scripts/evaluator | 119 ++++++++++++++++++++----- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/cime_config/machines/scripts/evaluator b/cime_config/machines/scripts/evaluator index d243b9fbf949..02e365a27848 100755 --- a/cime_config/machines/scripts/evaluator +++ b/cime_config/machines/scripts/evaluator @@ -18,11 +18,13 @@ ENDIF_RE = re.compile(r'^\s*endif') EXE_RE = re.compile(r'^\s*execute_process') WARNING_SET = { - "REP" : "Repitition: either a repeated item in an append or setting to same value as parent", + "REP" : "Repitition: a redundant setting to same value as parent", + "APP" : "Append: a repeated token in an append", "PRO" : "Promotion: all children of a parent do the same thing, so do it in the parent", "SET" : "Suspicious set: probably want append", "UNU" : "Unused: Using OS, COMPILER, or MACH with no matches in config_machines.xml", - "MIS" : "Missing: Having a machine/comp in config_machines.xml with no specialization in Macros" + "MIS" : "Missing: Having a machine/comp in config_machines.xml with no specialization in Macros", + "IFS" : "Ifs: Conditionals can be combined" } ############################################################################### @@ -135,10 +137,11 @@ def parse_macro(macro_path): ############################################################################### """ Parse one of our cmake macros, return (settings, appends), both lists - of (varname, value) tuples. + of (conditional, varname, value) tuples. """ settings = [] appends = [] + active_conditional = None with macro_path.open("r") as fd: for line in fd.readlines(): line = line.strip() @@ -151,12 +154,18 @@ def parse_macro(macro_path): if set_m: var, val = set_m.groups() - settings.append((var, val)) + settings.append((active_conditional, var, val)) elif append_m: var, val = append_m.groups() - appends.append((var, val)) - elif if_m or endif_m or exe_m: - pass # for now + appends.append((active_conditional, var, val)) + elif if_m: + assert active_conditional is None, "Nested conditionals not supported at the moment" + active_conditional = if_m.groups()[0] + elif endif_m: + assert active_conditional is not None, "Endif with no conditional?" + active_conditional = None + elif exe_m: + pass else: assert False, f"In {macro_path}, Could not parse line: {line}" @@ -197,8 +206,8 @@ def find_SET(macro_path_pl, machs): appends = set() for macro_pl in macro_pls: curr_settings, curr_appends = parse_macro(macro_pl) - sets.update ([var for var, val in curr_settings]) - appends.update([var for var, val in curr_appends]) + sets.update ([var for _, var, _ in curr_settings]) + appends.update([var for _, var, _ in curr_appends]) mixes = sets & appends if mixes: @@ -232,28 +241,57 @@ def find_REP(macro_path_pl, machs): the_os = data[0] for comp in data[1]: macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) - sets = {} - appends = {} + sets = {} # var -> {condition -> vals} for macro_pl in macro_pls: - curr_settings, curr_appends = parse_macro(macro_pl) - for var, val in curr_settings: + curr_settings = parse_macro(macro_pl)[0] + for cond, var, val in curr_settings: if var in sets: - if sets[var] == val: - warnings.append(f"Macro {macro_pl} has redundant set of {var} to {val}") - else: - sets[var] = val + cond_dict = sets[var] + conds_to_check = [None] if cond is None else [cond, None] + for cond_to_check in conds_to_check: + if cond_to_check in cond_dict: + prev_val = cond_dict[cond_to_check] + if prev_val == val: + warnings.append(f"Macro {macro_pl} has redundant set of {var} to {val}") + + cond_dict[cond] = val + else: - sets[var] = val + sets[var] = {cond:val} - for var, vals in curr_appends: + return warnings + +############################################################################### +def find_APP(macro_path_pl, machs): +############################################################################### + warnings = [] + for universal in UNIVERSALS: + for mach, data in machs.items(): + the_os = data[0] + for comp in data[1]: + macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) + appends = {} # var -> {condition -> vals} + for macro_pl in macro_pls: + curr_appends = parse_macro(macro_pl)[1] + for cond, var, vals in curr_appends: + val_tokens = vals.split() if var in appends: - potential_repeats = set(appends[var].split()) & set(vals.split()) - if potential_repeats: - warnings.append(f"Macro {macro_pl} has redundant set of {var}, repeated items are to {', '.join(potential_repeats)}") + cond_dict = appends[var] + conds_to_check = [None] if cond is None else [cond, None] + for cond_to_check in conds_to_check: + if cond_to_check in cond_dict: + prev_vals = cond_dict[cond_to_check] + potential_repeats = set(val_tokens) & set(prev_vals) + if potential_repeats: + warnings.append(f"Macro {macro_pl} has redundant appends for {var}, repeated items are to {', '.join(potential_repeats)}") + + if cond in cond_dict: + cond_dict[cond] += val_tokens + else: + cond_dict[cond] = val_tokens - appends[var] += vals else: - appends[var] = vals + appends[var] = {cond:val_tokens} return warnings @@ -264,6 +302,39 @@ def find_PRO(macro_path_pl, machs): warnings.append("This warning is not yet implemented") return warnings +############################################################################### +def find_IFS(macro_path_pl, machs): +############################################################################### + warnings = [] + for universal in UNIVERSALS: + for mach, data in machs.items(): + the_os = data[0] + for comp in data[1]: + macro_pls = get_all_macros(macro_path_pl, universal, the_os, mach, comp) + for macro_pl in macro_pls: + with macro_pl.open("r") as fd: + last_closed_if = None + active_conditional = None + for line in fd.readlines(): + line = line.strip() + if line != "": + if_m = IF_RE.match(line) + endif_m = ENDIF_RE.match(line) + + if if_m: + if last_closed_if == if_m.groups()[0]: + warnings.append(f"Could have merged ifs for {last_closed_if} in {macro_pl}") + + assert active_conditional is None, "Nested conditionals not supported at the moment" + active_conditional = if_m.groups()[0] + elif endif_m: + last_closed_if = active_conditional + active_conditional = None + else: + last_closed_if = None + + return warnings + ############################################################################### def find_MIS(macro_path_pl, machs): ############################################################################### From 1ebe843e46b49d353cbd74eb27acc65e01d89f4a Mon Sep 17 00:00:00 2001 From: donghuix Date: Thu, 11 Nov 2021 20:22:18 -0800 Subject: [PATCH 136/301] fix bug for mosart reading lat from unstructured input file --- components/mosart/src/riverroute/RtmMod.F90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/mosart/src/riverroute/RtmMod.F90 b/components/mosart/src/riverroute/RtmMod.F90 index da1175b85eae..4dde91595900 100644 --- a/components/mosart/src/riverroute/RtmMod.F90 +++ b/components/mosart/src/riverroute/RtmMod.F90 @@ -1409,7 +1409,11 @@ subroutine Rtmini(rtm_active,flood_active) call shr_sys_abort(subname//' ERROR gdc2glo values') endif rtmCTL%lonc(nr) = rtmCTL%rlon(i) - rtmCTL%latc(nr) = rtmCTL%rlat(j) + if (isgrid2d) then + rtmCTL%latc(nr) = rtmCTL%rlat(j) + else + rtmCTL%latc(nr) = rtmCTL%rlat(i) + endif rtmCTL%outletg(nr) = idxocn(n) rtmCTL%area(nr) = area_global(n) From 45702184ab780b2e39fcb7d515b1ab95942cee45 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Fri, 12 Nov 2021 12:42:10 -0800 Subject: [PATCH 137/301] Adds compset only changes for BGC prognoctic co2 These changes are brought in from sfeng/cpl/BPRP-lnd-atm branch which is based on E3SMv1.1. We verified that all the F90 changes in sfeng/cpl/BPRP-lnd-atm branch are already present in E3SMv2 master. --- cime_config/allactive/config_compsets.xml | 15 +++++++++++++++ components/mpas-ocean/bld/build-namelist | 5 +++++ .../mpas-ocean/cime_config/config_component.xml | 3 +++ driver-mct/cime_config/config_component_e3sm.xml | 3 +++ 4 files changed, 26 insertions(+) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index fece8e1e010a..c31d6420ed0a 100644 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -99,6 +99,11 @@ 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BCRC + + BGCEXP_BPRP_CNPRDCTC_1850 + 1850_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP + + BGCEXP_BCRC_CNPRDCTC_20TR 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC @@ -119,6 +124,16 @@ SSP585SOI_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BCRC + + BGCEXP_BPRP_CNPRDCTC_20TR + 20TR_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP + + + + BGCEXP_BPRP_CNPRDCTC_SSP585_CMIP6 + SSP585_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP + + BGCEXP_BCRD_CNPRDCTC_20TR 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD diff --git a/components/mpas-ocean/bld/build-namelist b/components/mpas-ocean/bld/build-namelist index e07bc8bf8f03..1bee62303852 100755 --- a/components/mpas-ocean/bld/build-namelist +++ b/components/mpas-ocean/bld/build-namelist @@ -1002,6 +1002,11 @@ if ($ocn_co2_type eq 'constant' ) { add_default($nl, 'config_ecosys_atm_alt_co2_option', 'val'=>"diagnostic"); add_default($nl, 'config_ecosys_atm_alt_co2_use_eco', 'val'=>".false."); add_default($nl, 'config_ecosys_atm_co2_constant_value', 'val'=>$atm_co2_const_val); +} elsif ($ocn_co2_type eq 'prognostic' ) { + add_default($nl, 'config_ecosys_atm_co2_option', 'val'=>"prognostic"); + add_default($nl, 'config_ecosys_atm_alt_co2_option', 'val'=>"prognostic"); + add_default($nl, 'config_ecosys_atm_alt_co2_use_eco', 'val'=>".false."); + add_default($nl, 'config_ecosys_atm_co2_constant_value', 'val'=>$atm_co2_const_val); } else { add_default($nl, 'config_ecosys_atm_co2_option', 'val'=>"diagnostic"); add_default($nl, 'config_ecosys_atm_alt_co2_option', 'val'=>"diagnostic"); diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index 9133731b3332..71a9f91ab4f2 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -88,14 +88,17 @@ diagnostic constant constant + prognostic bcrc bcrd bdrc bdrd + prognostic bcrc bcrd bdrc bdrd + prognostic case_comp env_case.xml diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 26be9d252e86..0e44028d6c83 100644 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -262,14 +262,17 @@ CO2A CO2A_OI CO2A_OI + CO2C CO2C CO2C CO2C CO2C + CO2C_OI CO2C_OI CO2C_OI CO2C_OI CO2C_OI + CO2B run_coupling env_run.xml From e4ebf23dc21259148e5e3041c2d9218de0490c9c Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Mon, 15 Nov 2021 10:51:53 -0600 Subject: [PATCH 138/301] Added namelist flag for macromolecule coupling to testcases Option is false for all testcases. Macromolecules are currently disabled in the new MARBL configuration. Biogeochemistry regression test is nonBFB. All other testcases pass. --- .../configurations/aerosol_shortwave_physics/namelist.seaice | 1 + .../configurations/prescribed_ice/namelist.seaice | 1 + .../configurations/snicar_shortwave/namelist.seaice | 1 + .../configurations/snow_tracer_physics/namelist.seaice | 1 + .../configurations/standard_bgc/namelist.seaice | 1 + .../configurations/standard_physics/namelist.seaice | 1 + .../configurations/standard_physics_single_cell/namelist.seaice | 1 + .../testcases/advection/namelist.seaice.advection | 1 + .../testcases/error_analysis/namelist.seaice.strain | 1 + .../testcases/ridging_1D/namelist.seaice.ridging_1D | 1 + .../testcases/ridging_island/namelist.seaice.ridging_island | 1 + .../testcases/spherical_operators/strain/namelist.seaice.strain | 1 + .../namelist.seaice.strain_stress_divergence | 1 + .../stress_divergence/namelist.seaice.stress_divergence | 1 + .../testcases/square/1D_velocity_hex/namelist.seaice.square | 1 + .../testcases/square/1D_velocity_quad/namelist.seaice.square | 1 + .../testcases/square/operators_strain/namelist.seaice.strain | 1 + .../namelist.seaice.strain_stress_divergence | 1 + .../namelist.seaice.stress_divergence | 1 + .../testcases/square/square_hex_sb/namelist.seaice.square | 1 + .../testcases/square/square_quadhex/namelist.seaice.square | 1 + 21 files changed, 21 insertions(+) diff --git a/components/mpas-seaice/testing_and_setup/configurations/aerosol_shortwave_physics/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/aerosol_shortwave_physics/namelist.seaice index eafbbf39c19d..25d484bec0aa 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/aerosol_shortwave_physics/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/aerosol_shortwave_physics/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = true config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/prescribed_ice/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/prescribed_ice/namelist.seaice index 5d616fcf7987..9ed1849ab0d8 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/prescribed_ice/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/prescribed_ice/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/snicar_shortwave/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/snicar_shortwave/namelist.seaice index 6ee27aab3211..45d6fe3ac6ed 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/snicar_shortwave/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/snicar_shortwave/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/snow_tracer_physics/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/snow_tracer_physics/namelist.seaice index d54f4bef705f..327e9d3438c6 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/snow_tracer_physics/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/snow_tracer_physics/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/standard_bgc/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/standard_bgc/namelist.seaice index d22ef6ea4451..a872892c849d 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/standard_bgc/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/standard_bgc/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = true config_use_DON = true config_use_iron = true + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = true config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/standard_physics/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/standard_physics/namelist.seaice index fdeda17b5f06..8b7d697b14db 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/standard_physics/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/standard_physics/namelist.seaice @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/configurations/standard_physics_single_cell/namelist.seaice b/components/mpas-seaice/testing_and_setup/configurations/standard_physics_single_cell/namelist.seaice index 3e2f785258e1..7c287d12e533 100644 --- a/components/mpas-seaice/testing_and_setup/configurations/standard_physics_single_cell/namelist.seaice +++ b/components/mpas-seaice/testing_and_setup/configurations/standard_physics_single_cell/namelist.seaice @@ -136,6 +136,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/advection/namelist.seaice.advection b/components/mpas-seaice/testing_and_setup/testcases/advection/namelist.seaice.advection index 37818e9761dc..0017855d1aa0 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/advection/namelist.seaice.advection +++ b/components/mpas-seaice/testing_and_setup/testcases/advection/namelist.seaice.advection @@ -139,6 +139,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/error_analysis/namelist.seaice.strain b/components/mpas-seaice/testing_and_setup/testcases/error_analysis/namelist.seaice.strain index cf00e602db54..72596349c2e3 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/error_analysis/namelist.seaice.strain +++ b/components/mpas-seaice/testing_and_setup/testcases/error_analysis/namelist.seaice.strain @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/ridging_1D/namelist.seaice.ridging_1D b/components/mpas-seaice/testing_and_setup/testcases/ridging_1D/namelist.seaice.ridging_1D index e6ace1e0846d..a8c1e2d3a83a 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/ridging_1D/namelist.seaice.ridging_1D +++ b/components/mpas-seaice/testing_and_setup/testcases/ridging_1D/namelist.seaice.ridging_1D @@ -142,6 +142,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/ridging_island/namelist.seaice.ridging_island b/components/mpas-seaice/testing_and_setup/testcases/ridging_island/namelist.seaice.ridging_island index dd9368d26e2a..cdc4ef9320d3 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/ridging_island/namelist.seaice.ridging_island +++ b/components/mpas-seaice/testing_and_setup/testcases/ridging_island/namelist.seaice.ridging_island @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/namelist.seaice.strain b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/namelist.seaice.strain index 483a6cda31db..812c59f6212b 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/namelist.seaice.strain +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/namelist.seaice.strain @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/namelist.seaice.strain_stress_divergence b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/namelist.seaice.strain_stress_divergence index 483a6cda31db..812c59f6212b 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/namelist.seaice.strain_stress_divergence +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/namelist.seaice.strain_stress_divergence @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/namelist.seaice.stress_divergence b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/namelist.seaice.stress_divergence index 22f150910049..cbd51e8d23c0 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/namelist.seaice.stress_divergence +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/namelist.seaice.stress_divergence @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_hex/namelist.seaice.square b/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_hex/namelist.seaice.square index a35e22989b73..4a71f5f755b9 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_hex/namelist.seaice.square +++ b/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_hex/namelist.seaice.square @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_quad/namelist.seaice.square b/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_quad/namelist.seaice.square index b913dbf85d96..3f90e9803435 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_quad/namelist.seaice.square +++ b/components/mpas-seaice/testing_and_setup/testcases/square/1D_velocity_quad/namelist.seaice.square @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain/namelist.seaice.strain b/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain/namelist.seaice.strain index cf00e602db54..72596349c2e3 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain/namelist.seaice.strain +++ b/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain/namelist.seaice.strain @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain_stress_divergence/namelist.seaice.strain_stress_divergence b/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain_stress_divergence/namelist.seaice.strain_stress_divergence index dba12b27c4b4..1f6a253a1079 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain_stress_divergence/namelist.seaice.strain_stress_divergence +++ b/components/mpas-seaice/testing_and_setup/testcases/square/operators_strain_stress_divergence/namelist.seaice.strain_stress_divergence @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/operators_stress_divergence/namelist.seaice.stress_divergence b/components/mpas-seaice/testing_and_setup/testcases/square/operators_stress_divergence/namelist.seaice.stress_divergence index f7e8ca9ae76d..c57d7e4f1964 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/operators_stress_divergence/namelist.seaice.stress_divergence +++ b/components/mpas-seaice/testing_and_setup/testcases/square/operators_stress_divergence/namelist.seaice.stress_divergence @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/square_hex_sb/namelist.seaice.square b/components/mpas-seaice/testing_and_setup/testcases/square/square_hex_sb/namelist.seaice.square index b0611ee85c74..0ce2d10d0d26 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/square_hex_sb/namelist.seaice.square +++ b/components/mpas-seaice/testing_and_setup/testcases/square/square_hex_sb/namelist.seaice.square @@ -139,6 +139,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' diff --git a/components/mpas-seaice/testing_and_setup/testcases/square/square_quadhex/namelist.seaice.square b/components/mpas-seaice/testing_and_setup/testcases/square/square_quadhex/namelist.seaice.square index c4d1359b4fc5..9bcfae7318de 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/square/square_quadhex/namelist.seaice.square +++ b/components/mpas-seaice/testing_and_setup/testcases/square/square_quadhex/namelist.seaice.square @@ -140,6 +140,7 @@ config_use_humics = false config_use_DON = false config_use_iron = false + config_use_macromolecules = false config_use_modal_aerosols = false config_use_zaerosols = false config_skeletal_bgc_flux_type = 'Jin2006' From e63dfd2b17237d313c7b5f25c0dcbaac73d73be4 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 15 Nov 2021 12:24:54 -0800 Subject: [PATCH 139/301] Initial port to Perlmutter at NERSC. Adds gnu and gnugpu compilers for PM, as well as adding a new compiler name nvidia (and nvidiagpu). Minor cleanup changes to HOMME cmake files. --- .../cmake_macros/gnu_perlmutter.cmake | 25 +++ .../cmake_macros/gnugpu_perlmutter.cmake | 29 ++++ .../cmake_macros/nvidia_perlmutter.cmake | 27 +++ .../cmake_macros/nvidiagpu_perlmutter.cmake | 19 +++ cime_config/machines/config_batch.xml | 23 +++ cime_config/machines/config_compilers.xml | 160 ++++++++++++++++++ cime_config/machines/config_machines.xml | 127 ++++++++++++++ .../cmake/machineFiles/perlmutter-gnu.cmake | 6 +- .../machineFiles/perlmutter-nocuda-gnu.cmake | 4 +- .../perlmutter-nocuda-nvidia.cmake | 4 +- .../machineFiles/perlmutter-nvidia.cmake | 6 +- .../homme/utils/csm_share/shr_spfn_mod.F90 | 2 +- share/util/shr_infnan_mod.F90.in | 2 +- 13 files changed, 418 insertions(+), 16 deletions(-) create mode 100644 cime_config/machines/cmake_macros/gnu_perlmutter.cmake create mode 100644 cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake create mode 100644 cime_config/machines/cmake_macros/nvidia_perlmutter.cmake create mode 100644 cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake diff --git a/cime_config/machines/cmake_macros/gnu_perlmutter.cmake b/cime_config/machines/cmake_macros/gnu_perlmutter.cmake new file mode 100644 index 000000000000..89ba5a77c315 --- /dev/null +++ b/cime_config/machines/cmake_macros/gnu_perlmutter.cmake @@ -0,0 +1,25 @@ +string(APPEND CONFIG_ARGS " --host=cray") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff") +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +set(NETCDF_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_C_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_FORTRAN_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}") +set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2 -g") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2 -g") +endif() +string(APPEND CXX_LIBS " -lstdc++") +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake b/cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake new file mode 100644 index 000000000000..b1974edcf387 --- /dev/null +++ b/cime_config/machines/cmake_macros/gnugpu_perlmutter.cmake @@ -0,0 +1,29 @@ +string(APPEND CONFIG_ARGS " --host=cray") +set(USE_CUDA "TRUE") +string(APPEND CPPDEFS " -DGPU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") +string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff") +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +set(NETCDF_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_C_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_FORTRAN_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}") +set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2") +endif() +string(APPEND CXX_LIBS " -lstdc++") +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/cmake_macros/nvidia_perlmutter.cmake b/cime_config/machines/cmake_macros/nvidia_perlmutter.cmake new file mode 100644 index 000000000000..a75763777953 --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidia_perlmutter.cmake @@ -0,0 +1,27 @@ +string(APPEND CONFIG_ARGS " --host=cray") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff") +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +set(NETCDF_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_C_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(NETCDF_FORTRAN_PATH "$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX}") +set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}") +set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +if (NOT DEBUG) + string(APPEND CFLAGS " -O2") +endif() +if (NOT DEBUG) + string(APPEND CXXFLAGS " -O2") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2") +endif() +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake b/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake new file mode 100644 index 000000000000..d8b5d49988ad --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake @@ -0,0 +1,19 @@ +string(APPEND CONFIG_ARGS " --host=cray") +set(USE_CUDA "TRUE") +string(APPEND CPPDEFS " -DGPU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") +string(APPEND CUDA_FLAGS " -O2 -arch sm_80 --use_fast_math") +string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff") +string(APPEND SLIBS " -lblas -llapack") +set(CXX_LINKER "FORTRAN") +set(NETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +set(NETCDF_C_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +set(NETCDF_FORTRAN_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +set(HDF5_PATH "$ENV{CRAY_HDF5_PARALLEL_PREFIX}") +set(PNETCDF_PATH "$ENV{CRAY_PARALLEL_NETCDF_PREFIX}") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 1a59677de3b3..1b2609b749d7 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -351,6 +351,29 @@ + + + --constraint=gpu + + + --gpus-per-task=1 + --gpu-bind=map_gpu:0,1,2,3 + + + -G 0 + + + --gpus-per-task=1 + --gpu-bind=map_gpu:0,1,2,3 + + + -G 0 + + + regular + + + -n {{ total_tasks }} diff --git a/cime_config/machines/config_compilers.xml b/cime_config/machines/config_compilers.xml index 96ec79ee29d8..82ee1f97cafd 100644 --- a/cime_config/machines/config_compilers.xml +++ b/cime_config/machines/config_compilers.xml @@ -1150,6 +1150,166 @@ flags should be captured within MPAS CMake files. + + + --host=cray + + + -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY + + + -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff + -lblas -llapack + + FORTRAN + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_HDF5_PARALLEL_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + + -O2 -g + + + + + -O2 -g + + + + -lstdc++ + + cc + CC + ftn + gcc + g++ + gfortran + + + + + --host=cray + + TRUE + + -DGPU + -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY + -DTHRUST_IGNORE_CUB_VERSION_CHECK + + + -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff + -lblas -llapack + + FORTRAN + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_HDF5_PARALLEL_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + -ccbin CC -O2 -arch sm_80 --use_fast_math + + + -O2 + + + + -O2 + + + + + -lstdc++ + + cc + CC + ftn + cc + CC + ftn + + + + + --host=cray + + + + -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY + + + -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff + -lblas -llapack + + FORTRAN + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} + $ENV{CRAY_HDF5_PARALLEL_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + + -O2 + + + + -O2 + + + + -O2 + + + + + cc + CC + ftn + cc + CC + ftn + + + + + --host=cray + + TRUE + + -DGPU + -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY + -DTHRUST_IGNORE_CUB_VERSION_CHECK + + + -O2 -arch sm_80 --use_fast_math + + + + + -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff + -lblas -llapack + + FORTRAN + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + $ENV{CRAY_HDF5_PARALLEL_PREFIX} + $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + cc + CC + ftn + + /projects/ccsm/AlbanyTrilinos_20190904/albany-build/install diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 32d740654954..13ecef028bd7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -55,6 +55,133 @@ This allows using a different mpirun command to launch unit tests --> + + Perlmutter at NERSC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) and 4 nvidia A100's. + login + Linux + gnu,gnugpu,nvidia,nvidiagpu + mpich + e3sm_g + /global/cfs/cdirs/e3sm + e3sm,m3411,m3412 + $ENV{PSCRATCH}/e3sm_scratch/perlmutter + /global/cfs/cdirs/e3sm/www/$ENV{USER} + http://portal.nersc.gov/project/e3sm/$ENV{USER} + /global/cfs/cdirs/e3sm/inputdata + /global/cfs/cdirs/e3sm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /global/cfs/cdirs/e3sm/baselines/$COMPILER + /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc + 10 + e3sm_developer + 4 + nersc_slurm + e3sm + 256 + 128 + 128 + 64 + 4 + 4 + TRUE + + srun + + --label + -n {{ total_tasks }} -N {{ num_nodes }} + -c $SHELL{echo 128/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} + $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} + + + + + + /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh + + /usr/share/lmod/lmod/libexec/lmod perl + /usr/share/lmod/lmod/libexec/lmod python + module + module + + + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf + PrgEnv-gnu + PrgEnv-nvidia + cuda + cudatoolkit + craype-accel-nvidia80 + craype-accel-host + perftools-base + perftools + darshan + + + + PrgEnv-gnu/8.2.0 + gcc/9.3.0 + + + + + PrgEnv-nvidia + nvidia/21.7 + + + + cuda + + + + cuda + + + + craype-accel-host + + + + craype-accel-host + + + + cray-libsci/21.08.1.2 + craype + cray-mpich/8.1.10 + cray-hdf5-parallel/1.12.0.7 + cray-netcdf-hdf5parallel/4.7.4.7 + cray-parallel-netcdf/1.12.1.7 + cmake/3.20.5 + + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + + + 1 + 1 + + + 128M + spread + threads + FALSE + /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + + + + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* diff --git a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake index 50229c12c1a7..a9ad558677ab 100644 --- a/components/homme/cmake/machineFiles/perlmutter-gnu.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-gnu.cmake @@ -3,7 +3,6 @@ SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") #SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") #SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") -set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "") # cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") @@ -23,7 +22,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "") SET(USE_QUEUING FALSE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") #SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") @@ -36,7 +34,7 @@ SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") #SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") -SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") # need this to get around link error with fortran (-arch=sm_80) +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") #SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") #SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") @@ -44,7 +42,7 @@ SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") # need this to get around SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") -# Note: need to set OMPI_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") diff --git a/components/homme/cmake/machineFiles/perlmutter-nocuda-gnu.cmake b/components/homme/cmake/machineFiles/perlmutter-nocuda-gnu.cmake index 5fbcc6650972..c28b53b4afa0 100644 --- a/components/homme/cmake/machineFiles/perlmutter-nocuda-gnu.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-nocuda-gnu.cmake @@ -1,7 +1,5 @@ # CMake initial cache file for Perlmutter using GNU without cuda -#set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "") - # cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") @@ -20,7 +18,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "") SET(USE_QUEUING FALSE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") #SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") @@ -33,6 +30,7 @@ SET(Kokkos_ARCH_VOLTA70 OFF CACHE BOOL "") SET(Kokkos_ARCH_AMPERE80 OFF CACHE BOOL "") SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") diff --git a/components/homme/cmake/machineFiles/perlmutter-nocuda-nvidia.cmake b/components/homme/cmake/machineFiles/perlmutter-nocuda-nvidia.cmake index f8e32762a7d4..20521e65e37d 100644 --- a/components/homme/cmake/machineFiles/perlmutter-nocuda-nvidia.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-nocuda-nvidia.cmake @@ -1,7 +1,5 @@ # CMake initial cache file for Perlmutter using nvidia without cuda -#set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "") - # cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") @@ -20,7 +18,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "") SET(USE_QUEUING FALSE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") #SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") @@ -33,6 +30,7 @@ SET(Kokkos_ARCH_VOLTA70 OFF CACHE BOOL "") SET(Kokkos_ARCH_AMPERE80 OFF CACHE BOOL "") SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") diff --git a/components/homme/cmake/machineFiles/perlmutter-nvidia.cmake b/components/homme/cmake/machineFiles/perlmutter-nvidia.cmake index dbb92fb8cfd7..81f69cd1f465 100644 --- a/components/homme/cmake/machineFiles/perlmutter-nvidia.cmake +++ b/components/homme/cmake/machineFiles/perlmutter-nvidia.cmake @@ -3,7 +3,6 @@ SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") #SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") #SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") -set(HOMMEXX_VECTOR_SIZE 1 CACHE STRING "") # cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") @@ -24,7 +23,6 @@ SET(WITH_PNETCDF FALSE CACHE FILEPATH "") SET(USE_QUEUING FALSE CACHE BOOL "") SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") -SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") #SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") @@ -37,7 +35,7 @@ SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") #SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") -SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") # need this to get around link error with fortran (-arch=sm_80) +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") #SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") #SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") @@ -45,7 +43,7 @@ SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") # need this to get around SET(CMAKE_C_COMPILER "cc" CACHE STRING "") SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") -# Note: need to set OMPI_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") diff --git a/components/homme/utils/csm_share/shr_spfn_mod.F90 b/components/homme/utils/csm_share/shr_spfn_mod.F90 index 8192665a5869..ac3e29e229ac 100644 --- a/components/homme/utils/csm_share/shr_spfn_mod.F90 +++ b/components/homme/utils/csm_share/shr_spfn_mod.F90 @@ -9,7 +9,7 @@ ! HAVE_ERF_EXTERNALS: erf and erfc ! These compilers have the intrinsics. -#if defined CPRIBM || defined CPRINTEL || defined __GFORTRAN__ || defined CPRCRAY +#if defined CPRIBM || defined CPRINTEL || defined __GFORTRAN__ || defined CPRCRAY || defined CPRNVIDIA #define HAVE_GAMMA_INTRINSICS #define HAVE_ERF_INTRINSICS #endif diff --git a/share/util/shr_infnan_mod.F90.in b/share/util/shr_infnan_mod.F90.in index 992c46fc9b86..568415af3dd5 100644 --- a/share/util/shr_infnan_mod.F90.in +++ b/share/util/shr_infnan_mod.F90.in @@ -1,6 +1,6 @@ ! Flag representing compiler support of Fortran 2003's ! ieee_arithmetic intrinsic module. -#if defined CPRIBM || defined CPRPGI || defined CPRINTEL || defined CPRCRAY || defined CPRNAG +#if defined CPRIBM || defined CPRPGI || defined CPRNVIDIA || defined CPRINTEL || defined CPRCRAY || defined CPRNAG #define HAVE_IEEE_ARITHMETIC #endif From fc42c6c29a8a22e68abfa837a7d190d1a4ddfd46 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 15 Nov 2021 14:07:23 -0800 Subject: [PATCH 140/301] For Perlmutter, Update default 1-node PE layouts. --- cime_config/allactive/config_pesall.xml | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 175917099b35..f47c16e083c4 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -73,6 +73,41 @@ + + + none + + 64 + 64 + 64 + 64 + 64 + 64 + 64 + 64 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 1 node default From 9e7b8d54a0ac26501206f7ea9ced7f949b0da121 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 15 Nov 2021 14:12:10 -0800 Subject: [PATCH 141/301] remove comments --- cime_config/machines/config_machines.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 13ecef028bd7..930ac2c4d608 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -95,16 +95,10 @@ - - /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh /usr/share/lmod/8.3.1/init/csh - /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module From d311e8e20dab86ffe1743f3ef9e0e48ed85e93bb Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 15 Nov 2021 15:25:05 -0800 Subject: [PATCH 142/301] Forgot to add the initial nvidia compiler definition and do `../scripts/converter ../config_compilers.xml -c perlmutter` to generate cmake_macros. --- .../machines/cmake_macros/nvidia.cmake | 48 ++++++++ .../machines/cmake_macros/nvidiagpu.cmake | 51 +++++++++ cime_config/machines/config_compilers.xml | 106 ++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 cime_config/machines/cmake_macros/nvidia.cmake create mode 100644 cime_config/machines/cmake_macros/nvidiagpu.cmake diff --git a/cime_config/machines/cmake_macros/nvidia.cmake b/cime_config/machines/cmake_macros/nvidia.cmake new file mode 100644 index 000000000000..2d6d2b44f23d --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidia.cmake @@ -0,0 +1,48 @@ +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "nvcc") +set(SCXX "nvc++") +set(SFC "nvfortran") +string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRNVIDIA") +if (compile_threaded) + string(APPEND CFLAGS " -mp") +endif() +if (NOT DEBUG) + string(APPEND CFLAGS " -O0 -Mnofma -Mvect=nosimd") +endif() +if (DEBUG) + string(APPEND CFLAGS " -g") +endif() +string(APPEND FFLAGS " -i4 -Mstack_arrays -Mextend -byteswapio -Mflushz -Kieee -DHAVE_IEEE_ARITHMETIC -Mallocatable=03 -DNO_R16 -traceback") +if (compile_threaded) + string(APPEND FFLAGS " -mp") +endif() +if (DEBUG) + string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2") +endif() +string(APPEND CXXFLAGS " -std=c++14") +if (compile_threaded) + string(APPEND CXXFLAGS " -mp") +endif() +if (DEBUG) + string(APPEND CXXFLAGS " -g -O0 -Mnofma -Wall -Kieee -traceback") +endif() +if (NOT DEBUG) + string(APPEND CXXFLAGS " -O2") +endif() +set(CXX_LINKER "CXX") +string(APPEND CXX_LIBS " -lstdc++") +string(APPEND FC_AUTO_R8 " -r8") +string(APPEND FFLAGS_NOOPT " -O0 -Mnofma") +string(APPEND FIXEDFLAGS " -Mfixed") +string(APPEND FREEFLAGS " -Mfree") +set(HAS_F2008_CONTIGUOUS "FALSE") +string(APPEND LDFLAGS " -Wl,--allow-multiple-definition") +if (compile_threaded) + string(APPEND LDFLAGS " -mp") +endif() +set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/cmake_macros/nvidiagpu.cmake b/cime_config/machines/cmake_macros/nvidiagpu.cmake new file mode 100644 index 000000000000..45ee8068183b --- /dev/null +++ b/cime_config/machines/cmake_macros/nvidiagpu.cmake @@ -0,0 +1,51 @@ +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "nvcc") +set(SCXX "nvc++") +set(SFC "nvfortran") +string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRNVIDIA") +if (DEBUG) + string(APPEND CPPDEFS " -DYAKL_DEBUG") +endif() +if (compile_threaded) + string(APPEND CFLAGS " -mp") +endif() +if (NOT DEBUG) + string(APPEND CFLAGS " -O2") +endif() +if (DEBUG) + string(APPEND CFLAGS " -g") +endif() +string(APPEND FFLAGS " -i4 -Mstack_arrays -Mextend -byteswapio -Mflushz -Kieee -DHAVE_IEEE_ARITHMETIC -Mallocatable=03 -DNO_R16 -traceback") +if (compile_threaded) + string(APPEND FFLAGS " -mp") +endif() +if (DEBUG) + string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -O2 -DHAVE_IEEE_ARITHMETIC") +endif() +string(APPEND CXXFLAGS " -std=c++14") +if (compile_threaded) + string(APPEND CXXFLAGS " -mp") +endif() +if (DEBUG) + string(APPEND CXXFLAGS " -g -O0 -Mnofma -Wall -traceback") +endif() +if (NOT DEBUG) + string(APPEND CXXFLAGS " -O2") +endif() +set(CXX_LINKER "CXX") +string(APPEND CXX_LIBS " -lstdc++") +string(APPEND FC_AUTO_R8 " -r8") +string(APPEND FFLAGS_NOOPT " -O0 -Mnofma") +string(APPEND FIXEDFLAGS " -Mfixed") +string(APPEND FREEFLAGS " -Mfree") +set(HAS_F2008_CONTIGUOUS "FALSE") +string(APPEND LDFLAGS " -Wl,--allow-multiple-definition") +if (compile_threaded) + string(APPEND LDFLAGS " -mp") +endif() +set(SUPPORTS_CXX "TRUE") diff --git a/cime_config/machines/config_compilers.xml b/cime_config/machines/config_compilers.xml index 82ee1f97cafd..680e09d1efe4 100644 --- a/cime_config/machines/config_compilers.xml +++ b/cime_config/machines/config_compilers.xml @@ -416,6 +416,112 @@ flags should be captured within MPAS CMake files. mpif90 + + cc + CC + ftn + nvcc + nvc++ + nvfortran + + -DFORTRANUNDERSCORE -DNO_R16 -DCPRNVIDIA + + + + -mp + -O0 -Mnofma -Mvect=nosimd + -g + + + -i4 -Mstack_arrays -Mextend -byteswapio -Mflushz -Kieee -DHAVE_IEEE_ARITHMETIC -Mallocatable=03 -DNO_R16 -traceback + -mp + -O0 -g -Ktrap=fp -Mbounds -Kieee + -O2 + + + -std=c++14 + -mp + -g -O0 -Mnofma -Wall -Kieee -traceback + -O2 + + CXX + + -lstdc++ + + + -r8 + + + -O0 -Mnofma + + + -Mfixed + + + -Mfree + + FALSE + + -Wl,--allow-multiple-definition + -mp + + TRUE + + + + cc + CC + ftn + nvcc + nvc++ + nvfortran + + -DFORTRANUNDERSCORE -DNO_R16 -DCPRNVIDIA + + + -DYAKL_DEBUG + + + -mp + -O2 + -g + + + -i4 -Mstack_arrays -Mextend -byteswapio -Mflushz -Kieee -DHAVE_IEEE_ARITHMETIC -Mallocatable=03 -DNO_R16 -traceback + -mp + -O0 -g -Ktrap=fp -Mbounds -Kieee + -O2 -DHAVE_IEEE_ARITHMETIC + + + -std=c++14 + -mp + -g -O0 -Mnofma -Wall -traceback + -O2 + + CXX + + -lstdc++ + + + -r8 + + + -O0 -Mnofma + + + -Mfixed + + + -Mfree + + FALSE + + -Wl,--allow-multiple-definition + -mp + + TRUE + + -time From 26184c02c7d77c5034402c794f4043573f5f5453 Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 15 Nov 2021 17:01:15 -0800 Subject: [PATCH 143/301] Missed flag for nvidiagpu compiler. Cleaned up comments --- .../cmake_macros/nvidiagpu_perlmutter.cmake | 2 +- cime_config/machines/config_compilers.xml | 26 +------------------ cime_config/machines/config_machines.xml | 3 --- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake b/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake index d8b5d49988ad..842cfc41ff4e 100644 --- a/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake +++ b/cime_config/machines/cmake_macros/nvidiagpu_perlmutter.cmake @@ -5,7 +5,7 @@ if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY") endif() string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") -string(APPEND CUDA_FLAGS " -O2 -arch sm_80 --use_fast_math") +string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") string(APPEND SLIBS " -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff") string(APPEND SLIBS " -lblas -llapack") set(CXX_LINKER "FORTRAN") diff --git a/cime_config/machines/config_compilers.xml b/cime_config/machines/config_compilers.xml index 680e09d1efe4..ff1cc5fbab90 100644 --- a/cime_config/machines/config_compilers.xml +++ b/cime_config/machines/config_compilers.xml @@ -1274,17 +1274,11 @@ flags should be captured within MPAS CMake files. $ENV{CRAY_HDF5_PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - -O2 -g - - -O2 -g - -lstdc++ @@ -1323,13 +1317,8 @@ flags should be captured within MPAS CMake files. -O2 - -O2 - - -lstdc++ @@ -1344,7 +1333,6 @@ flags should be captured within MPAS CMake files. --host=cray - -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY @@ -1360,22 +1348,14 @@ flags should be captured within MPAS CMake files. $ENV{CRAY_HDF5_PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - -O2 - -O2 - -O2 - - - cc CC ftn @@ -1395,12 +1375,8 @@ flags should be captured within MPAS CMake files. -DTHRUST_IGNORE_CUB_VERSION_CHECK - -O2 -arch sm_80 --use_fast_math + -ccbin CC -O2 -arch sm_80 --use_fast_math - - -L$ENV{CRAY_HDF5_PARALLEL_PREFIX}/lib -lhdf5_hl -lhdf5 -L$ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} -L$ENV{CRAY_PARALLEL_NETCDF_PREFIX}/lib -lpnetcdf -lnetcdf -lnetcdff -lblas -llapack diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 930ac2c4d608..e7f819352313 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -155,7 +155,6 @@ cray-parallel-netcdf/1.12.1.7 cmake/3.20.5 - $CIME_OUTPUT_ROOT/$CASE/run @@ -166,14 +165,12 @@ 1 1 - 128M spread threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - From d5c4224973d6edcf400081347feb1cdec2479d96 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 16 Nov 2021 11:14:35 -0500 Subject: [PATCH 144/301] refactor ESMT calculation of PGF modified: components/eam/src/physics/crm/samxx/scalar_momentum.cpp modified: components/eam/src/physics/crm/samxx/scalar_momentum.h --- .../src/physics/crm/samxx/scalar_momentum.cpp | 490 ++++++++---------- .../src/physics/crm/samxx/scalar_momentum.h | 6 +- 2 files changed, 212 insertions(+), 284 deletions(-) diff --git a/components/eam/src/physics/crm/samxx/scalar_momentum.cpp b/components/eam/src/physics/crm/samxx/scalar_momentum.cpp index 1bb0fce130e8..94445824ce8a 100644 --- a/components/eam/src/physics/crm/samxx/scalar_momentum.cpp +++ b/components/eam/src/physics/crm/samxx/scalar_momentum.cpp @@ -17,150 +17,32 @@ #include "scalar_momentum.h" #ifdef MMF_ESMT -void esmt_fft_forward(real2d& arr_in, real1d& k_out, real2d& arr_out) { -/*!------------------------------------------------------------------ - * Purpose: calculate forward FFT transform - * Author: Walter Hannah - Lawrence Livermore National Lab - * adapted from SP-WRF code provided by Stefan Tulich - *------------------------------------------------------------------*/ - auto& dx = ::dx; - int lensave, ier, nh, nl; - int lot, jump, n, inc, lenr, lensav, lenwrk; - real wsave[nx+15]; - - // naming convention follows fftpack5 routines - n = nx; - lot = 1; - lensav = n+15; - inc = 1; - lenr = nx; - jump = nx; - lenwrk = lenr; - -#ifndef USE_ORIG_FFT - realHost2d arrinHost = arr_in.createHostCopy(); - realHost2d arroutHost = arr_out.createHostCopy(); - - // initialization for FFT - rfft1i(n,wsave,lensav,ier); - - // do the forward transform - //do k = 1,nzm - for (auto k = 0; k < nzm; ++k) { - real atmp[nx]; - real tmp[nx]; - for (auto i = 0; i < nx; ++i) { atmp[i] = arrinHost(k,i); } - rfft1f( n, inc, atmp, lenr, wsave, lensav, tmp, lenwrk, ier ); - for (int i=0; i fft; - parallel_for( nzm , YAKL_LAMBDA (int k) { - real atmp[nx+2]; - real tmp[nx]; - - for (int i=0; i fft; - parallel_for( nzm , YAKL_LAMBDA (int k) { - real atmp[nx+2]; - real tmp[nx]; - - for (int i=0; i(nzm+1,ncrms) , YAKL_LAMBDA (int k, int icrm) { + if (k < nzm) { + dz_loc(k,icrm) = zi(k+1,icrm)-zi(k,icrm); + } else{ + dz_loc(k,icrm) = zi(k,icrm)-zi(k-1,icrm); + } }); - //do j=1,ny - for (auto j = 0; j < ny; ++j) { - //----------------------------------------- - // Initialize stuff for averaging - //----------------------------------------- - parallel_for( nzm, YAKL_LAMBDA (int k) { - u_s_avg(k) = 0.0; - shr(k) = 0.0; - }); - //----------------------------------------- - // Calculate shear of domain average profile - // defined on scalar levels - //----------------------------------------- - //do k = 1,nzm - parallel_for( nzm, YAKL_LAMBDA (int k) { - for (auto i = 0; i < nx; ++i) { - u_s_avg(k) = u_s_avg(k) + u_s(k,j,i); - // yakl::atomicAdd( u_s_avg(k) , u_s(k,j,i) ); + //----------------------------------------- + // Initialize stuff for averaging + //----------------------------------------- + //do k=1,nzm + // for (int icrm=0; icrm(nzm,ny,ncrms) , YAKL_LAMBDA (int k, int j, int icrm) { + scalar_wind_avg(k,j,icrm) = 0.0; + shear(k,j,icrm) = 0.0; + }); - // note that w is on interface levels - w_i(k,i) = ( w(k,j,i,icrm) + w(k+1,j,i,icrm) )/2.0; - } - u_s_avg(k) = u_s_avg(k) / real(nx); - }); + //----------------------------------------- + // Calculate shear of wind profile + // averaged over the "x" dimension + // defined on scalar levels + //----------------------------------------- + //do k=1,nzm + // for (int j=0; j(nzm,ny,nx,ncrms) , YAKL_DEVICE_LAMBDA (int k, int j, int i, int icrm) { + // real tmp = scalar_wind(k,j,i,icrm) / real(nx); + yakl::atomicAdd( scalar_wind_avg(k,j,icrm) , scalar_wind(k,j,i,icrm) / real(nx) ); + // note that w is on interface levels - need to interpolate to mid-levels + w_i(k,j,i,icrm) = ( w(k,j,i,icrm) + w(k+1,j,i,icrm) )/2.0 ; + }); - //do k = 2,nzm-1 - parallel_for( nzm-1, YAKL_LAMBDA (int k) { - if ( k>0) { - shr(k) = ( u_s_avg(k+1) - u_s_avg(k-1) )/(z(k+1,icrm)-z(k-1,icrm)); - } - shr(0) = ( u_s_avg(1) - u_s_avg(0) ) / ( z(1,icrm) - z(0,icrm) ); - }); + //do k = 1,nzm + // for (int icrm=0; icrm(nzm-1,ny,ncrms) , YAKL_LAMBDA (int k, int j, int icrm) { + if ( k>0) { + shear(k,j,icrm) = ( scalar_wind_avg(k+1,j,icrm) - scalar_wind_avg(k-1,j,icrm) )/(z(k+1,icrm)-z(k-1,icrm)); + } + shear(0,j,icrm) = ( scalar_wind_avg(1,j,icrm) - scalar_wind_avg(0,j,icrm) ) / ( z(1,icrm) - z(0,icrm) ); + }); - /*!------------------------------------------------------------------------ - ! Use Poisson solver to calculate pressure gradient force (PGF) - ! pgf is diagnosed from w * du_si/dz using the poisson equation - ! (see Wu and Yanai 1994) - !------------------------------------------------------------------------*/ + /*!------------------------------------------------------------------------ + ! Use Poisson solver to calculate pressure gradient force (PGF) + ! pgf is diagnosed from w * d/dz(scalar_wind) using the poisson equation + ! (see Wu and Yanai 1994) + !------------------------------------------------------------------------*/ + + //----------------------------------------- + // compute forward fft of w + //----------------------------------------- + yakl::RealFFT1D fftx; + fftx.init(fftx.trig); + + // for (int k=0; k(nzm,ny,ncrms) , YAKL_LAMBDA (int k, int j, int icrm) { + SArray ftmp; + for (int i=0; i(nzm,nx) , YAKL_LAMBDA (int k, int i) { - pgf_hat(k,i) = 0.; - }); + if((nx%2) == 0) nh = nx/2 - 1; + if((nx%2) != 0) nh = (nx-1)/2; - // Loop through wavelengths - //do i = 2,nx - parallel_for( SimpleBounds<2>(nzm,nx), YAKL_LAMBDA (int k, int i) { - if (i>0) { - a(k,i) = dz(k+1) / ( dz(k+1) + dz(k) ); - // the factor of 1.25 crudely accounts for difference between 2D and 3D updraft geometry - b(k,i) = -0.5 * 1.25 * pow(k_arr(i), 2.0) * dz(k) * dz(k+1) - 1.0; - c(k,i) = dz(k) / ( dz(k+1) + dz(k) ); - rhs(k,i) = pow(k_arr(i), 2.) * w_hat(k,i) * shr(k) * dz(k) * dz(k+1); + parallel_for( nh, YAKL_LAMBDA (int j) { + k_arr(2*j+1) = 2.*pi*real(j+1)/(real(nx)*dx); //cos + k_arr(2*j+2) = 2.*pi*real(j+1)/(real(nx)*dx); //sin + if (j==0) { + k_arr(0) = 0.0; + } + }); - //lower boundary condition (symmetric) - if (k == 0) { - b(0,i) = b(0,i) + a(0,i); - a(0,i) = 0.0; - } + if ((nx%2) == 0) { k_arr(nx-1) = 2.*pi/(2.*dx); } //nyquist wavelength for even n + + //----------------------------------------- + // solve vertical structure equation + // for each zonal wavelength (k_arr) + // solution method involves constructing + // a tridiagonal matrix + //----------------------------------------- + //do k=1,nzm + // for (int j=0; j(nzm,ny,nx2,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + pgf_hat(k,j,i,icrm) = 0.; + }); - //upper boundary condition (symmetric) - if (k == nzm-1) { - b(nzm-1,i) = b(nzm-1,i) + a(nzm-1,i); - c(nzm-1,i) = 0.0; - } - } - }); + //----------------------------------------- + // Loop through wavelengths + //----------------------------------------- + //do k=1,nzm + // for (int j=0; j(nzm,ny,nx,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + if (i>0) { + a(k,j,i,icrm) = dz_loc(k+1,icrm) / ( dz_loc(k+1,icrm) + dz_loc(k,icrm) ); + // the factor of 1.25 crudely accounts for difference between 2D and 3D updraft geometry + b(k,j,i,icrm) = -0.5 * 1.25 * pow(k_arr(i), 2.0) * dz_loc(k,icrm) * dz_loc(k+1,icrm) - 1.0; + c(k,j,i,icrm) = dz_loc(k,icrm) / ( dz_loc(k+1,icrm) + dz_loc(k,icrm) ); + pgf_hat(k,j,i,icrm) = pow(k_arr(i), 2.) * w_hat(k,j,i,icrm) * shear(k,j,icrm) * dz_loc(k,icrm) * dz_loc(k+1,icrm); + + //lower boundary condition (symmetric) + if (k == 0) { + b(0,j,i,icrm) = b(0,j,i,icrm) + a(0,j,i,icrm); + a(0,j,i,icrm) = 0.0; + } - // gaussian elimination with no pivoting - //do k = 1,nzm-1 - parallel_for( nx, YAKL_LAMBDA (int i) { - if (i>0) { - for (auto k = 0; k < nzm-1; ++k) { - b(k+1,i) = b(k+1,i) - a(k+1,i) / b(k,i) * c(k,i); - rhs(k+1,i) = rhs(k+1,i) - a(k+1,i) / b(k,i) * rhs(k,i); + //upper boundary condition (symmetric) + if (k == nzm-1) { + b(nzm-1,j,i,icrm) = b(nzm-1,j,i,icrm) + a(nzm-1,j,i,icrm); + c(nzm-1,j,i,icrm) = 0.0; } - } - }); + } + }); + //----------------------------------------- + // gaussian elimination with no pivoting + //----------------------------------------- + //do k=1,nzm-1 + // for (int j=0; j(nzm-1,ny,nx,ncrms) , YAKL_DEVICE_LAMBDA (int k, int j, int i, int icrm) { + if (i>0) { + b(k+1,j,i,icrm) = b(k+1,j,i,icrm) - a(k+1,j,i,icrm) / b(k,j,i,icrm) * c(k,j,i,icrm); + pgf_hat(k+1,j,i,icrm) = pgf_hat(k+1,j,i,icrm) - a(k+1,j,i,icrm) / b(k,j,i,icrm) * pgf_hat(k,j,i,icrm); + } + }); - parallel_for( nx, YAKL_LAMBDA (int i) { - if (i>0) { - // backward substitution - rhs(nzm-1,i) = rhs(nzm-1,i) / b(nzm-1,i); - //do k=nzm-1,1,-1 - for (auto k = nzm-2; k > -1; --k) { - rhs(k,i) = ( rhs(k,i) - c(k,i) * rhs(k+1,i) ) / b(k,i); + //----------------------------------------- + // backward substitution + //----------------------------------------- + //do k=1,nzm-1 + // for (int j=0; j(nzm-1,ny,nx,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + if (i>0) { + // f90: do k=nzm-1,1,-1 + // cpp: for (auto k = nzm-2; k > -1; --k) { + int kk = nzm-1 - k; + if (kk == nzm-1) { + pgf_hat(kk,j,i,icrm) = pgf_hat(kk,j,i,icrm) / b(kk,j,i,icrm); + } else { + pgf_hat(kk,j,i,icrm) = ( pgf_hat(kk,j,i,icrm) - c(kk,j,i,icrm) * pgf_hat(kk+1,j,i,icrm) ) / b(kk,j,i,icrm); } - } - }); + } + }); - //do k = 1,nzm - parallel_for( SimpleBounds<2>(nzm,nx), YAKL_LAMBDA (int k, int i) { - pgf_hat(k,i) = rhs(k,i); - }); + // Note sure what this part does... + // something about the Nyquist freq and whether nx is odd or even + if (nx%2 == 0) { + //do k=1,nzm + // for (int j=0; j(nzm,ny,nx,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + pgf_hat(k,j,nx-1,icrm) = pgf_hat(k,j,nx-1,icrm) / 2.0; + }); + } + //----------------------------------------- + // invert fft of pgf_hat to get pgf + //----------------------------------------- + // for (int k=0; k(nzm,ny,ncrms) , YAKL_DEVICE_LAMBDA (int k, int j, int icrm) { + SArray ftmp; + for (int i=0; i(nzm,ny,nx,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + if (k == 0) { + tend(k,j,i,icrm) = 0.0; + } else { + tend(k,j,i,icrm) = -1.0 * pgf(k,j,i,icrm) * rho(k,icrm); } + }); - //----------------------------------------- - // invert fft of pgf_hat to get pgf - //----------------------------------------- - esmt_fft_backward(pgf_hat, pgf); - - //----------------------------------------- - // Compute final tendency - //---------------------------------------- - //do k = 1,nzm - // do i = 1,nx - parallel_for( SimpleBounds<2>(nzm,nx) , YAKL_LAMBDA (int k, int i) { - if (k == 0) { - tend(k,j,i) = 0.0; - } else { - tend(k,j,i) = -1.0 * pgf(k,i) * rho(k,icrm); - } - }); - } // j } void scalar_momentum_tend() { - /*!------------------------------------------------------------------ + /*!------------------------------------------------------------------ * Purpose: Calculate pressure gradient effects on scalar momentum * Author: Walter Hannah - Lawrence Livermore National Lab *------------------------------------------------------------------*/ @@ -323,30 +263,22 @@ void scalar_momentum_tend() { auto &u_esmt = :: u_esmt; auto &v_esmt = :: v_esmt; - real3d u_esmt_pgf_3D("u_esmt_pgf_3D",nzm,ny,nx); - real3d v_esmt_pgf_3D("v_esmt_pgf_3d",nzm,ny,nx); - - real3d u_esmt_tmp("u_esmt_tmp", nzm,ny,nx); - real3d v_esmt_tmp("v_esmt_tmp", nzm,ny,nx); - - //do icrm = 1 , ncrms - for (auto icrm = 0; icrm < ncrms; ++icrm) { - parallel_for( SimpleBounds<3>(nzm,ny,nx) , YAKL_LAMBDA (int k, int j, int i) { - u_esmt_tmp(k,j,i) = u_esmt(k,j+offy_s,i+offx_s,icrm); - v_esmt_tmp(k,j,i) = v_esmt(k,j+offy_s,i+offx_s,icrm); - }); + real4d u_esmt_pgf_3D("u_esmt_pgf_3D",nzm,ny,nx,ncrms); + real4d v_esmt_pgf_3D("v_esmt_pgf_3d",nzm,ny,nx,ncrms); + + // Calculate pressure gradient force tendency + scalar_momentum_pgf(u_esmt,u_esmt_pgf_3D); + scalar_momentum_pgf(v_esmt,v_esmt_pgf_3D); + + // Add PGF tendency + //do k=1,nzm + // for (int j=0; j(nzm,ny,nx,ncrms) , YAKL_LAMBDA (int k, int j, int i, int icrm) { + u_esmt(k,j+offy_s,i+offx_s,icrm) = u_esmt(k,j+offy_s,i+offx_s,icrm) + u_esmt_pgf_3D(k,j,i,icrm)*dtn; + v_esmt(k,j+offy_s,i+offx_s,icrm) = v_esmt(k,j+offy_s,i+offx_s,icrm) + v_esmt_pgf_3D(k,j,i,icrm)*dtn; + }); - scalar_momentum_pgf(icrm,u_esmt_tmp,u_esmt_pgf_3D); - scalar_momentum_pgf(icrm,v_esmt_tmp,v_esmt_pgf_3D); - - // Add PGF tendency - //do k=1,nzm - // do j=1,ny - // do i=1,nx - parallel_for( SimpleBounds<3>(nzm,ny,nx) , YAKL_LAMBDA (int k, int j, int i) { - u_esmt(k,j+offy_s,i+offx_s,icrm) = u_esmt(k,j+offy_s,i+offx_s,icrm) + u_esmt_pgf_3D(k,j,i)*dtn; - v_esmt(k,j+offy_s,i+offx_s,icrm) = v_esmt(k,j+offy_s,i+offx_s,icrm) + v_esmt_pgf_3D(k,j,i)*dtn; - }); - } } #endif diff --git a/components/eam/src/physics/crm/samxx/scalar_momentum.h b/components/eam/src/physics/crm/samxx/scalar_momentum.h index 5d01e9907b47..9be52623c5b8 100644 --- a/components/eam/src/physics/crm/samxx/scalar_momentum.h +++ b/components/eam/src/physics/crm/samxx/scalar_momentum.h @@ -8,10 +8,6 @@ extern "C" void rfft1i(int n, real* wsave, int lensav, int ier); extern "C" void rfft1f(int n, int inc, real* r, int lenr, real* wsave, int lensav, real* work, int lenwrk, int ier ); extern "C" void rfft1b(int n, int inc, real* r, int lenr, real* wsave, int lensav, real* work, int lenwrk, int ier ); -void esmt_fft_forward(real2d& arr_in, real1d& k_out, real2d& arr_out); - -void esmt_fft_backward(real2d& arr_in, real2d& arr_out); - void scalar_momentum_tend(); -void scalar_momentum_pgf( int icrm, real3d& u_s, real3d& tend ); +void scalar_momentum_pgf( real4d& scalar_wind, real4d& tend ); From 1d334cc16e5790999530d58e7707db97a4cb7827 Mon Sep 17 00:00:00 2001 From: dqwu Date: Tue, 16 Nov 2021 13:25:09 -0600 Subject: [PATCH 145/301] Fixing build errors with mpi-serial lib for anlgce Define NETCDF_PATH for E3SM to use a serial NetCDF lib installed on anlgce when mpilib is set to mpi-serial. This fix is required to run SMS_D_Ln9_Mmpi-serial.f19_g16_rx1.A, a cime_developer test that uses mpi-serial. --- cime_config/machines/config_machines.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5fa0fdbd860e..48987007254a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -994,6 +994,10 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld + + + /nfs/gce/projects/climate/software/netcdf/4.8.1c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0 + /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} From e9515af8e946c5be409f934e4c79d1d87dfbfc64 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 16 Nov 2021 15:32:13 -0500 Subject: [PATCH 146/301] GPU bug fix --- .../eam/src/physics/crm/samxx/scalar_momentum.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/components/eam/src/physics/crm/samxx/scalar_momentum.cpp b/components/eam/src/physics/crm/samxx/scalar_momentum.cpp index 94445824ce8a..398f4a4af7b4 100644 --- a/components/eam/src/physics/crm/samxx/scalar_momentum.cpp +++ b/components/eam/src/physics/crm/samxx/scalar_momentum.cpp @@ -29,7 +29,9 @@ void scalar_momentum_pgf( real4d& scalar_wind, real4d& tend ) { auto &zi = :: zi; auto &w = :: w; auto &rho = :: rho; - auto &dx = ::dx; + auto &dx = ::dx; + + real constexpr pi = 3.14159; real1d k_arr("k_arr",nx); real2d dz_loc("dz_loc",nzm+1,ncrms); @@ -128,12 +130,13 @@ void scalar_momentum_pgf( real4d& scalar_wind, real4d& tend ) { k_arr(2*j+1) = 2.*pi*real(j+1)/(real(nx)*dx); //cos k_arr(2*j+2) = 2.*pi*real(j+1)/(real(nx)*dx); //sin if (j==0) { - k_arr(0) = 0.0; + k_arr(j) = 0.0; } + if ( j==(nh-1) && (nx%2)==0) { + k_arr(nx-1) = 2.*pi/(2.*dx); //nyquist wavelength for even n + } }); - if ((nx%2) == 0) { k_arr(nx-1) = 2.*pi/(2.*dx); } //nyquist wavelength for even n - //----------------------------------------- // solve vertical structure equation // for each zonal wavelength (k_arr) From 2ddc280afc564beef05dcbf82c0848a557a9a22b Mon Sep 17 00:00:00 2001 From: Sha Feng Date: Tue, 16 Nov 2021 14:24:05 -0800 Subject: [PATCH 147/301] comment out lines that cause model to crash due to non-used variables --- components/eam/bld/build-namelist | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index cc37a1fcccc2..939aa61d102d 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -922,10 +922,12 @@ if ($co2_cycle) { # (user specification has higher precedence than the true value set above) if ($nl->get_value('co2_readflux_fuel') =~ /$TRUE/io) { add_default($nl, 'co2flux_fuel_file', 'sim_year'=>$sim_year); - add_default($nl, 'co2flux_fuel_vrt_emis_file', 'sim_year'=>$sim_year); + # remove the line below for testing + # add_default($nl, 'co2flux_fuel_vrt_emis_file', 'sim_year'=>$sim_year); } - - add_default($nl, 'co2_readflux_aircraft', 'val'=>'.true.'); + + #change the default value from '.true.' to '.false.' + add_default($nl, 'co2_readflux_aircraft', 'val'=>'.false.'); # Check whether user has explicitly turned off reading the aircraft CO2 dataset. # (user specification has higher precedence than the true value set above) @@ -940,7 +942,8 @@ if ($co2_cycle) { # This should be the same file as the one in the aircraft_specifier file. # This is a workaround to get this filepath into the cam.input_data_list file # to allow the CESM scripts to obtain all required data for a run. - add_default($nl, 'aircraft_co2_file'); + # remove the line below for the testing + # add_default($nl, 'aircraft_co2_file'); } } } From 66432b7db334187c8b49c07312e5d58682806e53 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Tue, 16 Nov 2021 14:53:26 -0800 Subject: [PATCH 148/301] Update path for Albany lib for cori-knl build. Remove path in file for unsupported cori-haswell build. --- cime_config/machines/cmake_macros/intel_cori-haswell.cmake | 2 -- cime_config/machines/cmake_macros/intel_cori-knl.cmake | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake index 89012b1f73f3..d526d628ca02 100644 --- a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake @@ -1,5 +1,3 @@ -set(ALBANY_PATH "/global/cfs/cdirs/e3sm/software/albany-trilinos/albany-install-2020-08-07") -string(APPEND FFLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model consistent -fimf-use-svml") if (NOT DEBUG) string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") endif() diff --git a/cime_config/machines/cmake_macros/intel_cori-knl.cmake b/cime_config/machines/cmake_macros/intel_cori-knl.cmake index a82be9c82bed..e3e5eca46561 100644 --- a/cime_config/machines/cmake_macros/intel_cori-knl.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-knl.cmake @@ -1,4 +1,4 @@ -set(ALBANY_PATH "/global/homes/m/mperego/e3sm-software/albany-trilinos/albany-install-2021-01-05") +set(ALBANY_PATH "/global/homes/m/mperego/e3sm-software/albany-trilinos/albany-install-2021-10-01") string(APPEND CONFIG_ARGS " --host=cray") if (MPILIB STREQUAL impi) string(APPEND CFLAGS " -axMIC-AVX512 -xCORE-AVX2") From 0f4a4ead8589e4dd80a11c3dd013b51b84c3452c Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Tue, 16 Nov 2021 14:56:10 -0800 Subject: [PATCH 149/301] Update Greenland 1-to-10km initial condition filename information. --- components/mpas-albany-landice/cime_config/buildnml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 2ce0027925ec..00fd2ed0a868 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -73,8 +73,8 @@ def buildnml(case, caseroot, compname): decomp_date += '150910' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis1to10km': - grid_date += '063020' - grid_prefix += 'gis_1to10km_betaOpt' + grid_date += '082421' + grid_prefix += 'gis_1to10km_r01' decomp_date += '051920' decomp_prefix += 'mpasli.graph.info.' else: From 1ae5d20735c9bdaf91847c91c1adc2cc9feb8faa Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Tue, 16 Nov 2021 14:58:21 -0800 Subject: [PATCH 150/301] Temporarily comment out warning lines associated with 1) glc dynamic runoff routing, 2) glc elev class 'too low' (below sea level). These warnings are so prolific that they very significantly impact coupled model performance. Note that this same warnings do NOT show up in IG cases. --- components/elm/src/biogeophys/BalanceCheckMod.F90 | 2 +- driver-mct/main/map_glc2lnd_mod.F90 | 4 ++-- share/nuopc/glc_elevclass_mod.F90 | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/elm/src/biogeophys/BalanceCheckMod.F90 b/components/elm/src/biogeophys/BalanceCheckMod.F90 index cadc774ea4dd..723e9a72ffe6 100644 --- a/components/elm/src/biogeophys/BalanceCheckMod.F90 +++ b/components/elm/src/biogeophys/BalanceCheckMod.F90 @@ -344,7 +344,7 @@ subroutine ColWaterBalanceCheck( bounds, num_do_smb_c, filter_do_smb_c, & do fc = 1,num_do_smb_c c = filter_do_smb_c(fc) g = col_pp%gridcell(c) - write(iulog,*)'WARNING: glc_dyn_runoff_routing = ', glc_dyn_runoff_routing(g) ! TKT +! write(iulog,*)'WARNING: glc_dyn_runoff_routing = ', glc_dyn_runoff_routing(g) ! TKT if (glc_dyn_runoff_routing(g)) then errh2o(c) = errh2o(c) + qflx_glcice_frz(c)*dtime errh2o(c) = errh2o(c) - qflx_glcice_melt(c)*dtime diff --git a/driver-mct/main/map_glc2lnd_mod.F90 b/driver-mct/main/map_glc2lnd_mod.F90 index 4ae638ed71cf..131a16c3dadc 100644 --- a/driver-mct/main/map_glc2lnd_mod.F90 +++ b/driver-mct/main/map_glc2lnd_mod.F90 @@ -255,8 +255,8 @@ subroutine get_glc_elevation_classes(glc_topo, glc_elevclass) case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) - write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) - write(logunit,*) glc_errcode_to_string(err_code) +! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) +! write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) diff --git a/share/nuopc/glc_elevclass_mod.F90 b/share/nuopc/glc_elevclass_mod.F90 index 3a984f642127..94aa514c54fe 100644 --- a/share/nuopc/glc_elevclass_mod.F90 +++ b/share/nuopc/glc_elevclass_mod.F90 @@ -211,8 +211,8 @@ subroutine glc_get_elevation_classes_without_bareland(glc_topo, glc_elevclass, l case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) - write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) - write(logunit,*) glc_errcode_to_string(err_code) +! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) +! write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) @@ -534,8 +534,8 @@ subroutine glc_get_fractional_icecov(nec, glc_topo, glc_icefrac, glc_icefrac_ec, case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) - write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) - write(logunit,*) glc_errcode_to_string(err_code) +! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) +! write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) From c25f9a1ce4adbb3cab8975ce9c0c6fccce9a632a Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Wed, 17 Nov 2021 07:49:46 -0800 Subject: [PATCH 151/301] Updates machine files to use PIO2 and other module changes --- cime_config/machines/config_machines.xml | 17 ++++++++++++----- cime_config/machines/config_pio.xml | 1 - 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5fa0fdbd860e..3fcee5c04ad5 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1965,8 +1965,12 @@ intel,pgi mvapich2 /lustre/$USER/cime_output_root + + /pic/projects/sooty2/$ENV{USER}/e3sm_inputdata + /pic/projects/sooty2/$ENV{USER}/e3sm_inputdata/atm/datm7 /lustre/$USER/archive/$CASE /lustre/climate/acme_baselines/$COMPILER /lustre/climate/acme_baselines/cprnc/cprnc @@ -2003,21 +2007,20 @@ perl/5.20.0 cmake/3.17.1 - python/2.7.8 svn/1.8.13 - intel/15.0.1 - mkl/15.0.1 + intel/19.0.5 + mkl/2019u5 pgi/14.10 - mvapich2/2.1 + mvapich2/2.3.1 - netcdf/4.3.2 + netcdf/4.6.3 /lustre/$USER/csmruns/$CASE/run @@ -2026,6 +2029,10 @@ $ENV{MKLROOT} $ENV{NETCDF_LIB}/../ 64M + + /share/apps/python/3.7.2/lib/:/share/apps/openssl/1.0.2r/lib:$ENV{LD_LIBRARY_PATH}:/share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64: + /share/apps/python/3.7.2/bin:$ENV{PATH} diff --git a/cime_config/machines/config_pio.xml b/cime_config/machines/config_pio.xml index 331a1db62886..cd22249221ee 100644 --- a/cime_config/machines/config_pio.xml +++ b/cime_config/machines/config_pio.xml @@ -5,7 +5,6 @@ 2 - 1 From 79d74c0c7df8cc04bb5178f18e8b8aa3e7440c93 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 17 Nov 2021 13:21:03 -0600 Subject: [PATCH 152/301] remove unused ttypes --- .../homme/src/share/cxx/HommexxEnums.hpp | 21 ++---- .../cxx/cxx_f90_interface_theta.cpp | 33 +++++---- .../theta-l_kokkos/cxx/prim_advance_exp.cpp | 67 ++++++------------- 3 files changed, 43 insertions(+), 78 deletions(-) diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 0c0f6aad9c67..7e0e11e921ef 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -106,25 +106,14 @@ enum class UpdateType { enum class TimeStepType { // Explicit - LF = 0, // LeapFrog - RK2 = 1, - IMEX_KG254_EX = 4, // Explicit table from IMEX_KG254 - ULLRICH_RK35 = 5, + ttype5 = 5, // Implicit-Explicit - IMEX_KG243 = 6, - IMEX_KG254 = 7, - IMEX_KG355 = 9, - IMEX_KG255 = 10, - - // Implicit - BE = 11, // Backward Euler - BDF2 = 12 -}; + ttype7_imex = 7, + ttype9_imex = 9, + ttype10_imex = 10 -inline bool is_implicit (const TimeStepType& ts) { - return ts==TimeStepType::BE || ts==TimeStepType::BDF2; -} +}; // ======= How to combine output/input during calculations ========== // diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 9a1f35d10b12..da42490cd426 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -105,22 +105,22 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.transport_alg = transport_alg; params.theta_hydrostatic_mode = theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; - if (time_step_type==0) { - params.time_step_type = TimeStepType::LF; - } else if (time_step_type==1) { - params.time_step_type = TimeStepType::RK2; - } else if (time_step_type==4) { - params.time_step_type = TimeStepType::IMEX_KG254_EX; - } else if (time_step_type==5) { - params.time_step_type = TimeStepType::ULLRICH_RK35; - } else if (time_step_type==6) { - params.time_step_type = TimeStepType::IMEX_KG243; + + if (time_step_type==5) { + //5 stage, 3rd order, explicit + params.time_step_type = TimeStepType::ttype5; } else if (time_step_type==7) { - params.time_step_type = TimeStepType::IMEX_KG254; + //5 stage, based on 2nd order explicit KGU table + //1st order (BE) implicit part + params.time_step_type = TimeStepType::ttype7_imex; } else if (time_step_type==9) { - params.time_step_type = TimeStepType::IMEX_KG355; + //5 stage, based on 3rd order explicit KGU table + //2nd order implicit table + params.time_step_type = TimeStepType::ttype9_imex; } else if (time_step_type==10) { - params.time_step_type = TimeStepType::IMEX_KG255; + //5 stage, based on the 2nd order explicit KGU table + //2nd order implicit table + params.time_step_type = TimeStepType::ttype10_imex; } //set nu_ratios values @@ -365,10 +365,9 @@ void init_functors_c (const bool& allocate_buffer) vrm.setup(); } - const bool need_dirk = (params.time_step_type==TimeStepType::IMEX_KG243 || - params.time_step_type==TimeStepType::IMEX_KG254 || - params.time_step_type==TimeStepType::IMEX_KG255 || - params.time_step_type==TimeStepType::IMEX_KG355); + const bool need_dirk = (params.time_step_type==TimeStepType::ttype7_imex || + params.time_step_type==TimeStepType::ttype9_imex || + params.time_step_type==TimeStepType::ttype10_imex ); if (need_dirk) { // Create dirk functor only if needed diff --git a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp index 45a349179917..7aab3b444620 100644 --- a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp @@ -20,12 +20,10 @@ namespace Homme { // Declare all the timestepping schemes routines -void RK2_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void imex_KG254_explicit_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void u3_5stage_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void imex_KG243_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void imex_KG254_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void imex_KG255_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype5_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype7_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype9_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype10_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); // -------------- IMPLEMENTATIONS -------------- // @@ -80,23 +78,17 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost #endif switch (params.time_step_type) { - case TimeStepType::RK2: - RK2_timestep (tl, dt, eta_ave_w); + case TimeStepType::ttype5: + ttype5_timestep (tl, dt, eta_ave_w); break; - case TimeStepType::IMEX_KG254_EX: - imex_KG254_explicit_timestep (tl, dt, eta_ave_w); + case TimeStepType::ttype7_imex: + ttype7_imex_timestep (tl, dt, eta_ave_w); break; - case TimeStepType::ULLRICH_RK35: - u3_5stage_timestep (tl, dt, eta_ave_w); + case TimeStepType::ttype9_imex: + ttype9_imex_timestep (tl, dt, eta_ave_w); break; - case TimeStepType::IMEX_KG243: - imex_KG243_timestep (tl, dt, eta_ave_w); - break; - case TimeStepType::IMEX_KG254: - imex_KG254_timestep (tl, dt, eta_ave_w); - break; - case TimeStepType::IMEX_KG255: - imex_KG255_timestep (tl, dt, eta_ave_w); + case TimeStepType::ttype10_imex: + ttype10_imex_timestep (tl, dt, eta_ave_w); break; default: { @@ -135,26 +127,9 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost } // Implementations of timestep schemes, in terms of CaarFunctor runs - -void RK2_timestep(const TimeLevel& /* tl */, - const Real /* dt */, - const Real /* eta_ave_w */) -{ - // TODO - assert(false); -} - -void imex_KG254_explicit_timestep(const TimeLevel& /* tl */, - const Real /* dt */, - const Real /* eta_ave_w */) +void ttype5_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w) { - // TODO - assert(false); -} - -void u3_5stage_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w) -{ - GPTLstart("tl-ae U3-5stage_timestep"); + GPTLstart("ttype5_timestep"); // Get elements structure Elements& elements = Context::singleton().get(); SimulationParams& params = Context::singleton().get(); @@ -223,10 +198,12 @@ void u3_5stage_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w // Stage 5: u5 = (5u1-u0)/4 + 3dt/4 RHS(u4), t_rhs = t + dt/5 + dt/5 + dt/3 + 2dt/3 functor.run(RKStageData(nm1, np1, np1, qn0, 3.0*dt/4.0, 3.0*eta_ave_w/4.0)); - GPTLstop("tl-ae U3-5stage_timestep"); + GPTLstop("ttype5_timestep"); } -void imex_KG243_timestep(const TimeLevel& /* tl */, + +//compare with ttype10_imex, should be almost identical +void ttype7_imex_timestep(const TimeLevel& /* tl */, const Real /* dt */, const Real /* eta_ave_w */) { @@ -234,7 +211,7 @@ void imex_KG243_timestep(const TimeLevel& /* tl */, assert(false); } -void imex_KG254_timestep(const TimeLevel& /* tl */, +void ttype9_imex_timestep(const TimeLevel& /* tl */, const Real /* dt */, const Real /* eta_ave_w */) { @@ -243,11 +220,11 @@ void imex_KG254_timestep(const TimeLevel& /* tl */, } -void imex_KG255_timestep(const TimeLevel& tl, +void ttype10_imex_timestep(const TimeLevel& tl, const Real dt_dyn, const Real eta_ave_w) { - GPTLstart("IMEX_KG255"); + GPTLstart("ttype10_imex_timestep"); // The context const auto& c = Context::singleton(); @@ -304,7 +281,7 @@ void imex_KG255_timestep(const TimeLevel& tl, caar.run(RKStageData(n0, np1, np1, qn0, dt, eta_ave_w, 1.0, 0.0, 1.0)); dirk.run(nm1, a2*dt, n0, a1*dt, np1, a3*dt, elements, hvcoord); - GPTLstop("IMEX_KG255"); + GPTLstop("ttype10_imex_timestep"); } } // namespace Homme From 40a614ef8e9dd51144f9f6455c86fc4003f41bb0 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Wed, 17 Nov 2021 14:54:57 -0800 Subject: [PATCH 153/301] minor fates sp mode comment cleanup --- components/elm/bld/namelist_files/namelist_definition.xml | 6 +++--- components/elm/src/biogeochem/SatellitePhenologyMod.F90 | 2 +- components/elm/src/data_types/VegetationType.F90 | 2 +- components/elm/src/main/elmfates_interfaceMod.F90 | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 2b56ac4d0487..acbb80a4a70f 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -309,12 +309,12 @@ Switch deciding which nutrient model to use in FATES. -Toggle to turn on FATES fixed biogeography mode (STUB ONLY, MUST BE FALSE). +Toggle to turn on FATES fixed biogeography mode (only relevant if FATES is being used). -Toggle to turn on no competition mode (only relevant if FATES is being used). +Toggle to turn on FATES no competition mode (only relevant if FATES is being used). -Toggle to turn on plant hydraulics(only relevant if FATES is on). +Toggle to turn on plant hydraulics (only relevant if FATES is on). null() ! This is an ACTIVE fates patch logical , pointer :: is_bareground (:) => null() ! ? real(r8), pointer :: wt_ed (:) => null() ! TODO mv ? can this be removed - real(r8), pointer :: sp_pftorder_index (:) => null() ! index to map 'p' onto the order of ED patches in SP mode. + real(r8), pointer :: sp_pftorder_index (:) => null() ! index to map 'p' onto the order of FATES patches in SP mode. logical , pointer :: is_fates (:) => null() ! true for patch vector space reserved ! for FATES. ! this is static and is true for all diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index cb4918b19aaa..20888ea72491 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -824,7 +824,7 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & this%fates(nc)%bc_in(s)%hlm_sp_tlai(ft) = canopystate_inst%tlai_patch(p) this%fates(nc)%bc_in(s)%hlm_sp_tsai(ft) = canopystate_inst%tsai_patch(p) this%fates(nc)%bc_in(s)%hlm_sp_htop(ft) = canopystate_inst%htop_patch(p) - if(canopystate_inst%htop_patch(p).lt.1.0e-20)then ! zero htop causes inifinite/nans. This is + if(canopystate_inst%htop_patch(p).lt.1.0e-20)then ! zero htop causes inifinite/nans. this%fates(nc)%bc_in(s)%hlm_sp_htop(ft) = 0.01_r8 endif end do ! p From 2fcc9ef5d77b6a9456e0ed7834693e03227ac116 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 18 Nov 2021 12:53:04 -0600 Subject: [PATCH 154/301] fixing nutop bug exposed by new tests --- .../src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 34a0ce9db907..ffec927d5887 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -322,7 +322,8 @@ void HyperviscosityFunctorImpl::run (const int np1, const Real dt, const Real et Kokkos::fence(); - //sponge layer + //sponge layer + if(m_data.nu_top > 0){ for (int icycle = 0; icycle < m_data.hypervis_subcycle_tom; ++icycle) { //m_policy_first_laplace has ref states, so cannot be reused now @@ -340,6 +341,7 @@ void HyperviscosityFunctorImpl::run (const int np1, const Real dt, const Real et Kokkos::parallel_for(m_policy_update_states2, *this); Kokkos::fence(); + } } //for for sponge layer } //run() From c892a13577f4020934e2fbe5d7b2a040bfeb3e9e Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 18 Nov 2021 12:54:19 -0600 Subject: [PATCH 155/301] formatting --- .../cxx/HyperviscosityFunctorImpl.cpp | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index ffec927d5887..e7573a750159 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -324,24 +324,24 @@ void HyperviscosityFunctorImpl::run (const int np1, const Real dt, const Real et //sponge layer if(m_data.nu_top > 0){ - for (int icycle = 0; icycle < m_data.hypervis_subcycle_tom; ++icycle) { - - //m_policy_first_laplace has ref states, so cannot be reused now - //laplace(fields) --> ttens, etc. - Kokkos::parallel_for(m_policy_nutop_laplace, *this); - Kokkos::fence(); - - //exchange is done on ttens, dptens, vtens, etc. - ///? do another timer or the same for all mpi in HV? - ///exchange on a subset of levels in future? - assert (m_be->is_registration_completed()); - GPTLstart("hvf-bexch"); - m_be->exchange(); - GPTLstop("hvf-bexch"); - - Kokkos::parallel_for(m_policy_update_states2, *this); - Kokkos::fence(); - } + for (int icycle = 0; icycle < m_data.hypervis_subcycle_tom; ++icycle) { + + //m_policy_first_laplace has ref states, so cannot be reused now + //laplace(fields) --> ttens, etc. + Kokkos::parallel_for(m_policy_nutop_laplace, *this); + Kokkos::fence(); + + //exchange is done on ttens, dptens, vtens, etc. + ///? do another timer or the same for all mpi in HV? + ///exchange on a subset of levels in future? + assert (m_be->is_registration_completed()); + GPTLstart("hvf-bexch"); + m_be->exchange(); + GPTLstop("hvf-bexch"); + + Kokkos::parallel_for(m_policy_update_states2, *this); + Kokkos::fence(); + } } //for for sponge layer } //run() From 6f0b1ef4ef7cb0343cfcf9f71884bf99053042bd Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 1 Jun 2021 17:07:45 -0400 Subject: [PATCH 156/301] add F20TR-MMFXX-CMIP6 compset new file: components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml modified: components/eam/cime_config/config_component.xml modified: components/eam/cime_config/config_compsets.xml --- .../use_cases/20TR_MMF-1mom_CMIP6.xml | 98 +++++++++++++++++++ .../eam/cime_config/config_component.xml | 1 + .../eam/cime_config/config_compsets.xml | 8 ++ 3 files changed, 107 insertions(+) create mode 100644 components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml diff --git a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml new file mode 100644 index 000000000000..681cb2d7d86f --- /dev/null +++ b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml @@ -0,0 +1,98 @@ + + + + +.true. + + + + + + + +.false. + + + + + +atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc +SERIAL + + +atm/cam/ggas/GHG_CMIP-1-2-0_Annual_Global_0000-2014_c20180105.nc +RAMPED + + +atm/cam/volc +CMIP_DOE-ACME_radiation_1850-2014_v3_c20171205.nc +VOLC_CMIP6 +SERIAL + + +.true. +.true. +.true. + + +atm/cam/chem/trop_mam/aero +mam4_0.9x1.2_L72_2000clim_c170323.nc +.false. +CYCLICAL +01 + + + + + + + + + --> + + + + + + + + + + + + --> + + + + + + + + + + + + + + + + + --> + + + + + +atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc +SERIAL + + + + + +'H2O2', 'H2SO4', 'SO2' + + +1850-2000 + + diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index bc7b7e097124..97ec8d5f4794 100644 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -146,6 +146,7 @@ aquaplanet_MMF-1mom scam_arm97_MMF-1mom scam_rico_MMF-1mom + 20TR_MMF-1mom_CMIP6 run_component_cam diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 74781017de34..25f55fc26b56 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -264,6 +264,14 @@ RICO_EAM%MMFXX-SCM_ELM%SP_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV + + + + F20TR-MMFXX-CMIP6 + + 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + From e5153c65710548ab08492f7d2673a2ddfdaec368 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Tue, 1 Jun 2021 18:53:24 -0400 Subject: [PATCH 157/301] Fix SSTICE file for MMF-AMIP compset --- components/data_comps/docn/cime_config/config_component.xml | 4 ++++ components/eam/cime_config/config_compsets.xml | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 8a5b87ed6dc6..d1be71ac31d8 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -187,6 +187,7 @@ $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_2010_clim_c20190821.nc $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc + $DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_ice_CMIP6_DECK_E3SM_1x1_c20180213.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc @@ -245,6 +246,7 @@ 1850 1869 1869 + 1869 run_component_cam_sstice env_run.xml @@ -270,6 +272,7 @@ 1850 1869 1869 + 1869 run_component_cam_sstice env_run.xml @@ -289,6 +292,7 @@ 2012 2016 2016 + 2016 run_component_cam_sstice env_run.xml diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 25f55fc26b56..acbcc57a6418 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -268,7 +268,6 @@ F20TR-MMFXX-CMIP6 - 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV From 4de7da32fb278f25173b331db4d9568106944f99 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 18 Nov 2021 16:04:02 -0600 Subject: [PATCH 158/301] Change failing ERP_Ld3.ne4_oQU240.F2010 test to ERP_Ln18.ne4_oQU240.F2010 --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 2424643a9260..62920f19003c 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -49,7 +49,7 @@ "e3sm_atm_developer" : { "inherit" : ("eam_theta_pg2"), "tests" : ( - "ERP_Ld3.ne4_oQU240.F2010", + "ERP_Ln18.ne4_oQU240.F2010", "SMS_Ln9.ne4_oQU240.F2010.eam-outfrq9s", "SMS.ne4_oQU240.F2010.eam-cosplite", "SMS_R_Ld5.ne4_ne4.FSCM5A97.eam-scm", From 7ea10bd1650bddd95a9345c4fe243bbf42c30b7b Mon Sep 17 00:00:00 2001 From: hydrotian Date: Thu, 18 Nov 2021 23:32:57 -0800 Subject: [PATCH 159/301] add test for V2 land river features --- cime_config/tests.py | 3 ++- .../elm/V2_ELM_MOSART_features/shell_commands | 2 ++ .../elm/V2_ELM_MOSART_features/user_nl_elm | 10 ++++++++++ .../elm/V2_ELM_MOSART_features/user_nl_mosart | 5 +++++ 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_elm create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart diff --git a/cime_config/tests.py b/cime_config/tests.py index 2424643a9260..04962f70599e 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -42,7 +42,8 @@ "ERS.MOS_USRDAT.RMOSGPCC.mosart-mos_usrdat", "SMS.MOS_USRDAT.RMOSGPCC.mosart-unstructure", "SMS.r05_r05.IELM.elm-topounit", - "ERS.ELM_USRDAT.I1850ELM.elm-usrdat" + "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", + "ERS.r05_05.IELM.elm-V2_ELM_MOSART_features" ) }, diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/shell_commands new file mode 100644 index 000000000000..66912745d973 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange --id ELM_BLDNML_OPTS --val "-bgc cn -irrig .true." diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_elm new file mode 100644 index 000000000000..42db7bc05082 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_elm @@ -0,0 +1,10 @@ + fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc' + flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/landuse.timeseries_0.5x0.5_HIST_simyr1850-2015_c211019.nc' + do_transient_pfts = .true + do_transient_crops = .true. + irrigate = .true. + create_crop_landunit = .true. + tw_irr = .true. + extra_gw_irr = .false. + firrig_data = .true. + use_lake_wat_storage=.true. diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart new file mode 100644 index 000000000000..52b5c8a41d72 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart @@ -0,0 +1,5 @@ +paraFile = '$DIN_LOC_ROOT/rof/mosart/global_reservoir_halfdegree_c20200917.nc' +frivinp_rtm = '$DIN_LOC_ROOT/rof/mosart/MOSART_Global_half_20210422.nc' +wrmflag = .true. +inundflag = .true. + From c8b5d6416b12054a8c2e3c680e295279b8f194a9 Mon Sep 17 00:00:00 2001 From: hydrotian Date: Fri, 19 Nov 2021 10:26:16 -0800 Subject: [PATCH 160/301] typo fix --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 04962f70599e..a7c75f6756bc 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -43,7 +43,7 @@ "SMS.MOS_USRDAT.RMOSGPCC.mosart-unstructure", "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", - "ERS.r05_05.IELM.elm-V2_ELM_MOSART_features" + "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features" ) }, From e38234b8d6d581fc15c254e28311f0c1d7c8346f Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Fri, 19 Nov 2021 16:17:44 -0600 Subject: [PATCH 161/301] tests --- .../test/reg_test/namelists/theta-fdc12.nl | 76 +++++++++++++++++++ .../run_tests/theta-fdc12-test21-kokkos.cmake | 23 ++++++ .../run_tests/theta-fdc12-test21.cmake | 23 ++++++ 3 files changed, 122 insertions(+) create mode 100644 components/homme/test/reg_test/namelists/theta-fdc12.nl create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake diff --git a/components/homme/test/reg_test/namelists/theta-fdc12.nl b/components/homme/test/reg_test/namelists/theta-fdc12.nl new file mode 100644 index 000000000000..61b85fdc4034 --- /dev/null +++ b/components/homme/test/reg_test/namelists/theta-fdc12.nl @@ -0,0 +1,76 @@ +&ctl_nl +vthreads = 1 +NThreads = 1 +partmethod = 4 +topology = "cube" +test_case = "dcmip2012_test2_1" +u_perturb = 1 +rotate_grid = 0 +ne = ${HOMME_TEST_NE} +qsize = ${HOMME_TEST_QSIZE} +ndays = ${HOMME_TEST_NDAYS} +statefreq = 9999 +restartfreq = 43200 +restartfile = "./R0001" +runtype = 0 +mesh_file = '/dev/null' +tstep = ${HOMME_TEST_TIME_STEP} +rsplit = ${HOMME_TEST_RSPLIT} +qsplit = 1 +integration = "explicit" +smooth = 0 +nu = ${HOMME_TEST_NU} +nu_div = ${HOMME_TEST_NU} +nu_p = ${HOMME_TEST_NU} +nu_q = ${HOMME_TEST_NU} +nu_s =-1 +nu_top = ${HOMME_TEST_NUTOP} +se_ftype = 0 +limiter_option = ${HOMME_TEST_LIM} +vert_remap_q_alg = 1 +hypervis_scaling = ${HOMME_TEST_HVSCALING} +hypervis_order = 2 +hypervis_subcycle = ${HOMME_TEST_HVS} +hypervis_subcycle_tom = ${HOMME_TEST_HVS_TOM} +theta_hydrostatic_mode = ${HOMME_THETA_HY_MODE} +theta_advect_form = ${HOMME_THETA_FORM} +tstep_type = ${HOMME_TTYPE} +/ +&solver_nl +precon_method = "identity" +maxits = 500 +tol = 1.e-9 +/ +&filter_nl +filter_type = "taylor" +transfer_type = "bv" +filter_freq = 0 +filter_mu = 0.04D0 +p_bv = 12.0D0 +s_bv = .666666666666666666D0 +wght_fm = 0.10D0 +kcut_fm = 2 +/ +&vert_nl +vform = "ccm" +vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' +vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' +/ + +&prof_inparm +profile_outpe_num = 100 +profile_single_file = .true. +/ + +! timunits: 0= steps, 1=days, 2=hours +&analysis_nl + interp_gridtype = 2 + output_timeunits = 1,1 + output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} + output_start_time = 0,0 + output_end_time = 30000,30000 + output_varnames1 = 'ps','zeta','u','v','T' + output_varnames2 = 'Q','Q2','Q3','Q4' + io_stride = 8 + output_type = 'netcdf' +/ diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake new file mode 100644 index 000000000000..0540fb8a55b6 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake @@ -0,0 +1,23 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test21-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc21.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dd1.nc + dd2.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake new file mode 100644 index 000000000000..7fbe0f9f7b12 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake @@ -0,0 +1,23 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test21) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc21.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dd1.nc + dd2.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) From c7f53c8fc6ad984a9cda6bb3c7a95fe5579738f6 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Fri, 19 Nov 2021 15:15:45 -0800 Subject: [PATCH 162/301] Point to updated 20 km GIS input file for use in testing. --- components/mpas-albany-landice/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 00fd2ed0a868..181a21c9510c 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -63,7 +63,7 @@ def buildnml(case, caseroot, compname): decomp_date += '20190326' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis20km': - grid_date += '150922' + grid_date += '111921' grid_prefix += 'gis20km' decomp_date += '150922' decomp_prefix += 'mpasli.graph.info.' From 25fc0aea8fb3e437c85c3c6cbc42787e35820123 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sat, 20 Nov 2021 18:43:23 -0700 Subject: [PATCH 163/301] shorten tests --- components/homme/test/reg_test/namelists/thetahs1.nl | 2 +- components/homme/test/reg_test/namelists/thetahs2.nl | 2 +- components/homme/test/reg_test/namelists/thetahs3.nl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/homme/test/reg_test/namelists/thetahs1.nl b/components/homme/test/reg_test/namelists/thetahs1.nl index 060e68860e05..f014f8a1dfad 100644 --- a/components/homme/test/reg_test/namelists/thetahs1.nl +++ b/components/homme/test/reg_test/namelists/thetahs1.nl @@ -14,7 +14,7 @@ restartfreq = 43200 restartfile = "./R0001" runtype = 0 mesh_file = '/dev/null' -tstep = 100 +tstep = 600 rsplit = 3 qsplit = 1 transport_alg = 0 diff --git a/components/homme/test/reg_test/namelists/thetahs2.nl b/components/homme/test/reg_test/namelists/thetahs2.nl index a696f7c6dd1d..6a74a2e8be41 100644 --- a/components/homme/test/reg_test/namelists/thetahs2.nl +++ b/components/homme/test/reg_test/namelists/thetahs2.nl @@ -14,7 +14,7 @@ restartfreq = 43200 restartfile = "./R0001" runtype = 0 mesh_file = '/dev/null' -tstep = 100 +tstep = 600 dt_remap_factor = 3 dt_tracer_factor = 6 transport_alg = 12 diff --git a/components/homme/test/reg_test/namelists/thetahs3.nl b/components/homme/test/reg_test/namelists/thetahs3.nl index 1233cd1ce1a0..958c110270bb 100644 --- a/components/homme/test/reg_test/namelists/thetahs3.nl +++ b/components/homme/test/reg_test/namelists/thetahs3.nl @@ -14,7 +14,7 @@ restartfreq = 43200 restartfile = "./R0001" runtype = 0 mesh_file = '/dev/null' -tstep = 100 +tstep = 600 dt_remap_factor = 3 dt_tracer_factor = 6 transport_alg = 12 From 859f29a7910ced90f77f3372d3fa18ae37d1f78b Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 21 Nov 2021 00:49:16 -0700 Subject: [PATCH 164/301] nonbfb --- .../test/reg_test/namelists/theta-fdc12.nl | 113 +++++++----------- .../test/reg_test/run_tests/test-list.cmake | 1 + .../run_tests/theta-fdc12-test21-kokkos.cmake | 6 +- .../run_tests/theta-fdc12-test21.cmake | 6 +- 4 files changed, 50 insertions(+), 76 deletions(-) diff --git a/components/homme/test/reg_test/namelists/theta-fdc12.nl b/components/homme/test/reg_test/namelists/theta-fdc12.nl index 61b85fdc4034..70519a497b51 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12.nl @@ -1,76 +1,49 @@ &ctl_nl -vthreads = 1 -NThreads = 1 -partmethod = 4 -topology = "cube" -test_case = "dcmip2012_test2_1" -u_perturb = 1 -rotate_grid = 0 -ne = ${HOMME_TEST_NE} -qsize = ${HOMME_TEST_QSIZE} -ndays = ${HOMME_TEST_NDAYS} -statefreq = 9999 -restartfreq = 43200 -restartfile = "./R0001" -runtype = 0 -mesh_file = '/dev/null' -tstep = ${HOMME_TEST_TIME_STEP} -rsplit = ${HOMME_TEST_RSPLIT} -qsplit = 1 -integration = "explicit" -smooth = 0 -nu = ${HOMME_TEST_NU} -nu_div = ${HOMME_TEST_NU} -nu_p = ${HOMME_TEST_NU} -nu_q = ${HOMME_TEST_NU} -nu_s =-1 -nu_top = ${HOMME_TEST_NUTOP} -se_ftype = 0 -limiter_option = ${HOMME_TEST_LIM} -vert_remap_q_alg = 1 -hypervis_scaling = ${HOMME_TEST_HVSCALING} -hypervis_order = 2 -hypervis_subcycle = ${HOMME_TEST_HVS} -hypervis_subcycle_tom = ${HOMME_TEST_HVS_TOM} -theta_hydrostatic_mode = ${HOMME_THETA_HY_MODE} -theta_advect_form = ${HOMME_THETA_FORM} -tstep_type = ${HOMME_TTYPE} -/ -&solver_nl -precon_method = "identity" -maxits = 500 -tol = 1.e-9 -/ -&filter_nl -filter_type = "taylor" -transfer_type = "bv" -filter_freq = 0 -filter_mu = 0.04D0 -p_bv = 12.0D0 -s_bv = .666666666666666666D0 -wght_fm = 0.10D0 -kcut_fm = 2 + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_1" ! test identifier + theta_hydrostatic_mode = .false. + rsplit = 1 + ne = ${HOMME_TEST_NE} + qsize = 0 ! num tracer fields + nmax = 1800 ! 7200s / 0.4s per step = 18000 steps + statefreq = 360 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.4 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 5 ! 1 => default method + nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 + nu_s = 3.2e7 + nu_p = 3.2e7 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 4000.0 ! mountain wavelength (m) + limiter_option = 9 + vert_remap_q_alg = 1 / &vert_nl -vform = "ccm" -vfile_mid = './vcoord/${HOMME_TEST_VCOORD_MID_FILE}' -vfile_int = './vcoord/${HOMME_TEST_VCOORD_INT_FILE}' -/ - -&prof_inparm -profile_outpe_num = 100 -profile_single_file = .true. + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) / - -! timunits: 0= steps, 1=days, 2=hours &analysis_nl - interp_gridtype = 2 - output_timeunits = 1,1 - output_frequency = ${HOMME_TEST_NDAYS},${HOMME_TEST_NDAYS} - output_start_time = 0,0 - output_end_time = 30000,30000 - output_varnames1 = 'ps','zeta','u','v','T' - output_varnames2 = 'Q','Q2','Q3','Q4' - io_stride = 8 - output_type = 'netcdf' + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 1800, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + !num_io_procs = 16 + io_stride = 8 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. / diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index e067cdf52240..b7ebb3335d32 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -97,6 +97,7 @@ IF (BUILD_HOMME_THETA_KOKKOS) theta-fhs1 theta-fhs2 theta-fhs3 + theta-fdc12-test21 ) #all tests that will be used for cxx-vs-F bfb testing diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake index 0540fb8a55b6..c4339e18f7e4 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake @@ -9,13 +9,13 @@ SET(EXEC_NAME theta-nlev128-kokkos) SET(NUM_CPUS 16) -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc21.nl) +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12.nl) SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dd1.nc - dd2.nc) + dcmip2012_test2_11.nc + dcmip2012_test2_12.nc) #DO NOT MOD diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake index 7fbe0f9f7b12..cefe6ae067dd 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake @@ -9,13 +9,13 @@ SET(EXEC_NAME theta-nlev128) SET(NUM_CPUS 16) -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc21.nl) +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12.nl) SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dd1.nc - dd2.nc) + dcmip2012_test2_11.nc + dcmip2012_test2_12.nc) #DO NOT MOD From 60f66096e6c4908608cb1e847c5e5f6cd3a724d7 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 21 Nov 2021 16:39:04 -0700 Subject: [PATCH 165/301] removing rearth 1 --- components/homme/src/share/cxx/ElementsGeometry.cpp | 4 ++++ components/homme/src/share/cxx/ElementsGeometry.hpp | 4 +++- components/homme/src/share/cxx/PhysicalConstants.hpp | 5 +++-- components/homme/src/share/cxx/SphereOperators.hpp | 2 ++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/components/homme/src/share/cxx/ElementsGeometry.cpp b/components/homme/src/share/cxx/ElementsGeometry.cpp index f11fd5ed5ab5..a0484c294ea7 100644 --- a/components/homme/src/share/cxx/ElementsGeometry.cpp +++ b/components/homme/src/share/cxx/ElementsGeometry.cpp @@ -17,6 +17,7 @@ namespace Homme { void ElementsGeometry::init(const int num_elems, const bool consthv, const bool alloc_gradphis, + const Real rearth, const bool alloc_sphere_coords) { // Sanity check assert (num_elems>0); @@ -24,6 +25,9 @@ void ElementsGeometry::init(const int num_elems, const bool consthv, const bool m_num_elems = num_elems; m_consthv = consthv; + assert( rearth > 0); + m_rearth = rearth; + // Coriolis force m_fcor = ExecViewManaged("FCOR", m_num_elems); diff --git a/components/homme/src/share/cxx/ElementsGeometry.hpp b/components/homme/src/share/cxx/ElementsGeometry.hpp index 4f19e6c1d377..22a3420459c1 100644 --- a/components/homme/src/share/cxx/ElementsGeometry.hpp +++ b/components/homme/src/share/cxx/ElementsGeometry.hpp @@ -49,6 +49,7 @@ class ElementsGeometry { ElementsGeometry() : m_num_elems(0) {} void init (const int num_elems, const bool consthv, const bool alloc_gradphis, + const Real rearth, const bool alloc_sphere_coords=false); void randomize (const int seed); @@ -69,7 +70,8 @@ class ElementsGeometry { private: bool m_consthv; - int m_num_elems; + int m_num_elems; + Real m_rearth; }; } // Homme diff --git a/components/homme/src/share/cxx/PhysicalConstants.hpp b/components/homme/src/share/cxx/PhysicalConstants.hpp index dfe58896778d..b7c4cc3feaa9 100644 --- a/components/homme/src/share/cxx/PhysicalConstants.hpp +++ b/components/homme/src/share/cxx/PhysicalConstants.hpp @@ -19,12 +19,13 @@ namespace PhysicalConstants constexpr Real Rgas = 287.04; constexpr Real cp = 1005.0; constexpr Real kappa = Rgas / cp; - constexpr Real rearth = 6.376e6; - constexpr Real rrearth = 1.0 / rearth; +// constexpr Real rearth = 6.376e6; +// constexpr Real rrearth = 1.0 / rearth; constexpr Real g = 9.80616; constexpr Real p0 = 100000; // [mbar] constexpr Real Tref = 288; + }; } // namespace Homme diff --git a/components/homme/src/share/cxx/SphereOperators.hpp b/components/homme/src/share/cxx/SphereOperators.hpp index 817860d8f24d..8888cf4ad798 100644 --- a/components/homme/src/share/cxx/SphereOperators.hpp +++ b/components/homme/src/share/cxx/SphereOperators.hpp @@ -75,6 +75,8 @@ class SphereOperators m_metdet = geometry.m_metdet; m_metinv = geometry.m_metinv; m_spheremp = geometry.m_spheremp; + m_rearth = geometry.m_rearth; + m_rrearth = 1./geometry.m_rearth; } template From 2330a08fe952ae2f0e1e3d6e14f6fa51884844ff Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 23 Nov 2021 11:20:06 -0600 Subject: [PATCH 166/301] Now that the LINUX macro is actually being loaded it appears not to work, so remove --- cime_config/machines/cmake_macros/intel_LINUX.cmake | 1 - 1 file changed, 1 deletion(-) delete mode 100644 cime_config/machines/cmake_macros/intel_LINUX.cmake diff --git a/cime_config/machines/cmake_macros/intel_LINUX.cmake b/cime_config/machines/cmake_macros/intel_LINUX.cmake deleted file mode 100644 index 6ab34e341885..000000000000 --- a/cime_config/machines/cmake_macros/intel_LINUX.cmake +++ /dev/null @@ -1 +0,0 @@ -string(APPEND FFLAGS " -mcmodel medium -shared-intel") From 34db72d32bf29b6bcf001e1e866c9e3d6d3090b1 Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Tue, 23 Nov 2021 12:20:06 -0600 Subject: [PATCH 167/301] changed variable avgOceanSurfaceDOCSum to avgOceanSurfaceDOCSemiLabile Verified with a short 10 day GCASE run. --- components/mpas-ocean/driver/ocn_comp_mct.F | 10 +++++----- .../src/shared/mpas_ocn_time_average_coupled.F | 12 ++++++------ .../mpas-ocean/src/tracer_groups/Registry_ecosys.xml | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/mpas-ocean/driver/ocn_comp_mct.F b/components/mpas-ocean/driver/ocn_comp_mct.F index c39f0b2897cf..7effa0a1a992 100644 --- a/components/mpas-ocean/driver/ocn_comp_mct.F +++ b/components/mpas-ocean/driver/ocn_comp_mct.F @@ -2497,7 +2497,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ avgOceanSurfaceDMS, & avgOceanSurfaceDMSP, & avgOceanSurfaceDOCr, & - avgOceanSurfaceDOCSum, & + avgOceanSurfaceDOCSemiLabile, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved @@ -2581,7 +2581,7 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) endif @@ -2668,9 +2668,9 @@ subroutine ocn_export_mct(o2x_o, errorCode) !{{{ o2x_o % rAttr(index_o2x_So_algae2, n) = max(0.0_RKIND,avgOceanSurfacePhytoC(2,i)) o2x_o % rAttr(index_o2x_So_algae3, n) = max(0.0_RKIND,avgOceanSurfacePhytoC(3,i)) o2x_o % rAttr(index_o2x_So_dic1, n) = max(0.0_RKIND,avgOceanSurfaceDIC(i)) - o2x_o % rAttr(index_o2x_So_doc1, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) - o2x_o % rAttr(index_o2x_So_doc2, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) - o2x_o % rAttr(index_o2x_So_doc3, n) = max(0.0_RKIND,avgOceanSurfaceDOCSum(i)) + o2x_o % rAttr(index_o2x_So_doc1, n) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_o % rAttr(index_o2x_So_doc2, n) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) + o2x_o % rAttr(index_o2x_So_doc3, n) = max(0.0_RKIND,avgOceanSurfaceDOCSemiLabile(i)) o2x_o % rAttr(index_o2x_So_don1, n) = 0.0_RKIND o2x_o % rAttr(index_o2x_So_no3, n) = max(0.0_RKIND,avgOceanSurfaceNO3(i)) o2x_o % rAttr(index_o2x_So_sio3, n) = max(0.0_RKIND,avgOceanSurfaceSiO3(i)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F index 211514efe173..4dc182e56219 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_time_average_coupled.F @@ -74,7 +74,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgOceanSurfaceDMS, & avgOceanSurfaceDMSP, & avgOceanSurfaceDOCr, & - avgOceanSurfaceDOCSum, & + avgOceanSurfaceDOCSemiLabile, & avgOceanSurfaceFeParticulate, & avgOceanSurfaceFeDissolved @@ -130,7 +130,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCr', avgOceanSurfaceDOCr) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeParticulate', avgOceanSurfaceFeParticulate) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceFeDissolved', avgOceanSurfaceFeDissolved) @@ -142,7 +142,7 @@ subroutine ocn_time_average_coupled_init(forcingPool)!{{{ avgOceanSurfacePhytoC(:,iCell) = 0.0_RKIND avgOceanSurfaceDIC(iCell) = 0.0_RKIND - avgOceanSurfaceDOCSum(iCell) = 0.0_RKIND + avgOceanSurfaceDOCSemiLabile(iCell) = 0.0_RKIND avgOceanSurfaceNO3(iCell) = 0.0_RKIND avgOceanSurfaceSiO3(iCell) = 0.0_RKIND avgOceanSurfaceNH4(iCell) = 0.0_RKIND @@ -231,7 +231,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel real (kind=RKIND), dimension(:), pointer :: avgOceanSurfaceDIC, & avgOceanSurfaceDON, & - avgOceanSurfaceDOCSum, & + avgOceanSurfaceDOCSemiLabile, & avgOceanSurfaceNO3, & avgOceanSurfaceSiO3, & avgOceanSurfaceNH4, & @@ -311,7 +311,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfacePhytoC', avgOceanSurfacePhytoC) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDIC', avgOceanSurfaceDIC) - call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSum', avgOceanSurfaceDOCSum) + call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceDOCSemiLabile', avgOceanSurfaceDOCSemiLabile) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNO3', avgOceanSurfaceNO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceSiO3', avgOceanSurfaceSiO3) call mpas_pool_get_array(ecosysSeaIceCoupling, 'avgOceanSurfaceNH4', avgOceanSurfaceNH4) @@ -340,7 +340,7 @@ subroutine ocn_time_average_coupled_accumulate(statePool, forcingPool, timeLevel avgOceanSurfaceDIC(iCell) = ( avgOceanSurfaceDIC(iCell) * nAccumulatedCoupled & + ecosysTracers(dic_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) - avgOceanSurfaceDOCSum(iCell) = ( avgOceanSurfaceDOCSum(iCell) * nAccumulatedCoupled & + avgOceanSurfaceDOCSemiLabile(iCell) = ( avgOceanSurfaceDOCSemiLabile(iCell) * nAccumulatedCoupled & + ecosysTracers(doc_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) avgOceanSurfaceSiO3(iCell) = ( avgOceanSurfaceSiO3(iCell) * nAccumulatedCoupled & + ecosysTracers(sio3_ind_MPAS,1,iCell) ) / ( nAccumulatedCoupled + 1) diff --git a/components/mpas-ocean/src/tracer_groups/Registry_ecosys.xml b/components/mpas-ocean/src/tracer_groups/Registry_ecosys.xml index 78f620036d44..3f7e6b94eaa5 100644 --- a/components/mpas-ocean/src/tracer_groups/Registry_ecosys.xml +++ b/components/mpas-ocean/src/tracer_groups/Registry_ecosys.xml @@ -687,8 +687,8 @@ - Date: Wed, 24 Nov 2021 15:23:58 -0800 Subject: [PATCH 168/301] Fixes a bug in O3 dry dep following Emmons et al. 2020 Currently, stomatal resistances for shaded and sunlit leaves are addd in series but they should be added in parallel Fixes #4677 [BFB] --- components/elm/src/biogeochem/DryDepVelocity.F90 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/elm/src/biogeochem/DryDepVelocity.F90 b/components/elm/src/biogeochem/DryDepVelocity.F90 index cbcbec3d70ab..f693f5816532 100644 --- a/components/elm/src/biogeochem/DryDepVelocity.F90 +++ b/components/elm/src/biogeochem/DryDepVelocity.F90 @@ -456,7 +456,9 @@ subroutine depvel_compute( bounds, & rlux(ispec)=1.e36_r8 else - rs=(fsun(pi)*rssun(pi))+(rssha(pi)*(1._r8-fsun(pi))) + !Following Emmons et al (2020, JAMES), the stomatal resistances for shaded and sunlit + !leaves should be added in parallel + rs = 1.0_r8 / ( fsun(pi)/rssun(pi) + (1.0_r8 - fsun(pi))/rssha(pi) ) if (rs==0._r8) then ! fvitt -- what to do when rs is zero ??? rsmx(ispec) = 1.e36_r8 else From d92d90db2df3b4c88ada62ddb31f8bcc702c39d6 Mon Sep 17 00:00:00 2001 From: hydrotian Date: Wed, 24 Nov 2021 19:51:50 -0800 Subject: [PATCH 169/301] daily MOSART output --- .../testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart index 52b5c8a41d72..93ddf2c4b8fb 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/V2_ELM_MOSART_features/user_nl_mosart @@ -2,4 +2,7 @@ paraFile = '$DIN_LOC_ROOT/rof/mosart/global_reservoir_halfdegree_c20200917.nc' frivinp_rtm = '$DIN_LOC_ROOT/rof/mosart/MOSART_Global_half_20210422.nc' wrmflag = .true. inundflag = .true. +rtmhist_mfilt=1 +rtmhist_nhtfrq=-24 + From f49c36a0b9fa72a6659b81bd6799495efaadd133 Mon Sep 17 00:00:00 2001 From: Brian O'Neill Date: Mon, 29 Nov 2021 16:26:28 -0500 Subject: [PATCH 170/301] GPU port of ocean vmix routines - replaced mesh pool with ocn_mesh module - minimized copies of tridiag coefs in solution loops - inlined tridiag and kpp_rhs routines - added OpenACC directives for loop kernals and data updates --- .../mpas-ocean/src/shared/mpas_ocn_vmix.F | 914 +++++++++--------- 1 file changed, 435 insertions(+), 479 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F index 51b09b173bb7..d3446dcc6196 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F @@ -25,6 +25,7 @@ module ocn_vmix use mpas_derived_types use mpas_pool_routines use mpas_timer + use ocn_mesh use mpas_constants use ocn_constants @@ -132,8 +133,7 @@ subroutine ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, ti integer :: err1, err2, err3 integer :: timeLevel - integer :: iEdge, iCell, nEdges, nCells - integer, dimension(:), pointer :: nEdgesArray, nCellsArray + integer :: iEdge, iCell, k, nEdges, nCells !----------------------------------------------------------------- ! @@ -149,28 +149,46 @@ subroutine ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, ti timeLevel = 1 end if - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - nEdges = nEdgesArray( 2 ) + nEdges = nEdgesHalo( 1 ) +#ifdef MPAS_OPENACC + !$acc parallel loop collapse(2) present(vertViscTopOfEdge) +#else !$omp parallel - !$omp do schedule(runtime) + !$omp do schedule(runtime) private(k) +#endif do iEdge = 1, nEdges - vertViscTopOfEdge(:, iEdge) = 0.0_RKIND + do k = 1, nVertLevels + vertViscTopOfEdge(k, iEdge) = 0.0_RKIND + end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif - nCells = nCellsArray( 2 ) + nCells = nCellsHalo( 1 ) +#ifdef MPAS_OPENACC + !$acc parallel loop collapse(2) present(vertDiffTopOfCell) +#else !$omp parallel - !$omp do schedule(runtime) + !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCells - vertDiffTopOfCell(:, iCell) = 0.0_RKIND + do k = 1, nVertLevels + vertDiffTopOfCell(k, iCell) = 0.0_RKIND end do + end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc update host(vertViscTopOfEdge, vertDiffTopOfCell) +#endif call ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err1, timeLevel) call ocn_vmix_coefs_redi_build(meshPool, statePool, err2, timeLevel) @@ -178,6 +196,9 @@ subroutine ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, ti err = ior(ior(err1, err2), err3) +#ifdef MPAS_OPENACC + !$acc update device(vertDiffTopOfCell, vertViscTopOfCell, vertNonLocalFlux) +#endif !-------------------------------------------------------------------- end subroutine ocn_vmix_coefs!}}} @@ -195,7 +216,7 @@ end subroutine ocn_vmix_coefs!}}} ! !----------------------------------------------------------------------- - subroutine ocn_vel_vmix_tend_implicit_rayleigh(meshPool, dt, kineticEnergyCell, vertViscTopOfEdge, layerThickness, & !{{{ + subroutine ocn_vel_vmix_tend_implicit_rayleigh(dt, kineticEnergyCell, vertViscTopOfEdge, layerThickness, & !{{{ layerThicknessEdge, normalVelocity, err) !----------------------------------------------------------------- @@ -204,9 +225,6 @@ subroutine ocn_vel_vmix_tend_implicit_rayleigh(meshPool, dt, kineticEnergyCell, ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - real (kind=RKIND), dimension(:,:), intent(in) :: & kineticEnergyCell !< Input: kinetic energy at cell @@ -245,41 +263,35 @@ subroutine ocn_vel_vmix_tend_implicit_rayleigh(meshPool, dt, kineticEnergyCell, ! !----------------------------------------------------------------- - integer :: iEdge, k, cell1, cell2, Nsurf, N, nEdges - integer, pointer :: nVertLevels - integer, dimension(:), pointer :: nEdgesArray - - integer, dimension(:), pointer :: minLevelEdgeBot, maxLevelEdgeTop - - integer, dimension(:,:), pointer :: cellsOnEdge + integer :: iEdge, k, cell1, cell2, Nsurf, N - real (kind=RKIND), dimension(:), allocatable :: A, B, C, velTemp + real (kind=RKIND), dimension(:), allocatable :: bTemp, C, rTemp + real (kind=RKIND) :: A, m real (kind=RKIND) :: edgeThicknessTotal err = 0 if(.not.velVmixOn) return - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) + allocate(bTemp(nVertLevels),C(nVertLevels),rTemp(nVertLevels)) - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - - nEdges = nEdgesArray( 1 ) - - allocate(A(nVertLevels),B(nVertLevels),C(nVertLevels),velTemp(nVertLevels)) +#ifdef MPAS_OPENACC + !$acc enter data create(bTemp, C, rTemp) + !$acc parallel loop present(maxLevelEdgeTop, minLevelEdgeBot, cellsOnEdge, & + !$acc layerThicknessEdge, layerThickness, vertViscTopOfEdge, normalVelocity, & + !$acc kineticEnergyCell) & + !$acc private(Nsurf, N, cell1, cell2, edgeThicknessTotal, A, bTemp, C, m, rTemp, k) +#else !$omp parallel !$omp do schedule(runtime) & - !$omp private(Nsurf, N, cell1, cell2, edgeThicknessTotal, k, A, B, C, velTemp) - do iEdge = 1, nEdges + !$omp private(Nsurf, N, cell1, cell2, edgeThicknessTotal, k, A, bTemp, C, m, rTemp) +#endif + do iEdge = 1, nEdgesOwned Nsurf = minLevelEdgeBot(iEdge) N = maxLevelEdgeTop(iEdge) if (N .gt. 0) then - ! Compute A(k), B(k), C(k) ! layerThicknessEdge is computed in compute_solve_diag, and is not available yet, ! so recompute layerThicknessEdge here. cell1 = cellsOnEdge(1,iEdge) @@ -288,54 +300,67 @@ subroutine ocn_vel_vmix_tend_implicit_rayleigh(meshPool, dt, kineticEnergyCell, do k = Nsurf, N layerThicknessEdge(k,iEdge) = 0.5_RKIND * (layerThickness(k,cell1) + layerThickness(k,cell2)) edgeThicknessTotal = edgeThicknessTotal + layerThicknessEdge(k,iEdge) - end do - - ! A is lower diagonal term - A(1:Nsurf)=0.0_RKIND - do k = Nsurf+1, N - A(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & - / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & - / layerThicknessEdge(k,iEdge) enddo - ! C is upper diagonal term - C(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N-1 - C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & - / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo + ! tridiagonal matrix algorithm + C(Nsurf) = -2.0_RKIND*dt*vertViscTopOfEdge(Nsurf+1,iEdge) & + / (layerThicknessEdge(Nsurf,iEdge) + layerThicknessEdge(Nsurf+1,iEdge)) & + / layerThicknessEdge(Nsurf,iEdge) + bTemp(Nsurf) = 1.0_RKIND - C(Nsurf) & + ! added Rayleigh terms + + dt*rayleighDampingCoef/((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal) + rTemp(Nsurf) = normalVelocity(Nsurf,iEdge) - ! B is diagonal term - B(1:Nsurf-1)=0.0_RKIND - B(Nsurf) = 1.0_RKIND - C(Nsurf) & - + dt*rayleighDampingCoef/((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal) + ! first pass: set the coefficients do k = Nsurf+1, N-1 - B(k) = 1.0_RKIND - A(k) - C(k) & - ! added Rayleigh terms - + dt*(rayleighDampingCoef/((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal)) + A = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & + / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & + / layerThicknessEdge(k,iEdge) + m = A/bTemp(k-1) + C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & + / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & + / layerThicknessEdge(k,iEdge) + bTemp(k) = 1.0_RKIND - A - C(k) & + ! added Rayleigh terms + + dt*(rayleighDampingCoef/((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal)) & + - m*C(k-1) + rTemp(k) = normalVelocity(k,iEdge) - m*rTemp(k-1) enddo - ! Apply bottom drag boundary condition on the viscous term - ! second line uses sqrt(2.0*kineticEnergyEdge(k,iEdge)) - B(N) = 1.0_RKIND - A(N) + dt*implicitBottomDragCoef & - * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) & - ! added Rayleigh terms - + dt*(rayleighBottomDampingCoef + & - rayleighDampingCoef /((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal)) - - call tridiagonal_solve(A(Nsurf+1:N),B(Nsurf:N),C(Nsurf:N-1),normalVelocity(Nsurf:N,iEdge),velTemp(Nsurf:N),N-Nsurf+1) + A = -2.0_RKIND*dt*vertViscTopOfEdge(N,iEdge) & + / (layerThicknessEdge(N-1,iEdge) + layerThicknessEdge(N,iEdge)) & + / layerThicknessEdge(N,iEdge) + m = A/bTemp(N-1) + ! x(N) = rTemp(N) / bTemp(N) + ! Apply bottom drag boundary condition on the viscous term + ! using sqrt(2.0*kineticEnergyEdge(k,iEdge)) + normalVelocity(N,iEdge) = (normalVelocity(N,iEdge) - m*rTemp(N-1)) & + / (1.0_RKIND - A + dt*implicitBottomDragCoef & + * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) & + ! added Rayleigh terms + + dt*(rayleighBottomDampingCoef + rayleighDampingCoef & + / ((1.0_RKIND - rayleighDepthVariable) + rayleighDepthVariable*edgeThicknessTotal)) & + - m*C(N-1)) + ! second pass: back substitution + do k = N-1, Nsurf, -1 + normalVelocity(k,iEdge) = (rTemp(k) - C(k)*normalVelocity(k+1,iEdge)) / bTemp(k) + enddo normalVelocity(1:Nsurf-1,iEdge) = 0.0_RKIND - normalVelocity(Nsurf:N,iEdge) = velTemp(Nsurf:N) normalVelocity(N+1:nVertLevels,iEdge) = 0.0_RKIND end if end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif - deallocate(A,B,C,velTemp) +#ifdef MPAS_OPENACC + !$acc exit data delete(bTemp, C, rTemp) +#endif + + deallocate(bTemp,C,rTemp) !-------------------------------------------------------------------- @@ -354,7 +379,7 @@ end subroutine ocn_vel_vmix_tend_implicit_rayleigh!}}} ! !----------------------------------------------------------------------- - subroutine ocn_vel_vmix_tend_implicit(meshPool, dt, kineticEnergyCell, vertViscTopOfEdge, layerThickness, & !{{{ + subroutine ocn_vel_vmix_tend_implicit(dt, kineticEnergyCell, vertViscTopOfEdge, layerThickness, & !{{{ layerThicknessEdge, normalVelocity, err) !----------------------------------------------------------------- @@ -363,9 +388,6 @@ subroutine ocn_vel_vmix_tend_implicit(meshPool, dt, kineticEnergyCell, vertViscT ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - real (kind=RKIND), dimension(:,:), intent(in) :: & kineticEnergyCell !< Input: kinetic energy at cell @@ -404,87 +426,92 @@ subroutine ocn_vel_vmix_tend_implicit(meshPool, dt, kineticEnergyCell, vertViscT ! !----------------------------------------------------------------- - integer :: iEdge, k, cell1, cell2, N, Nsurf, nEdges - integer, pointer :: nVertLevels - integer, dimension(:), pointer :: nEdgesArray - - integer, dimension(:), pointer :: maxLevelEdgeTop, minLevelEdgeBot + integer :: iEdge, k, cell1, cell2, N, Nsurf - integer, dimension(:,:), pointer :: cellsOnEdge - - real (kind=RKIND), dimension(:), allocatable :: A, B, C, velTemp + real (kind=RKIND), dimension(:), allocatable :: bTemp, C, rTemp + real (kind=RKIND) :: A, m err = 0 if(.not.velVmixOn) return - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) - - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - - nEdges = nEdgesArray( 1 ) + allocate(bTemp(nVertLevels),C(nVertLevels),rTemp(nVertLevels)) - allocate(A(nVertLevels),B(nVertLevels),C(nVertLevels),velTemp(nVertLevels)) +#ifdef MPAS_OPENACC + !$acc enter data create(bTemp, C, rTemp) + !$acc parallel loop present(maxLevelEdgeTop, minLevelEdgeBot, cellsOnEdge, & + !$acc layerThicknessEdge, layerThickness, vertViscTopOfEdge, normalVelocity, & + !$acc kineticEnergyCell) & + !$acc private(Nsurf, N, cell1, cell2, A, bTemp, C, m, rTemp, k) +#else !$omp parallel - !$omp do schedule(runtime) private(Nsurf, N, cell1, cell2, A, B, C, velTemp, k) - do iEdge = 1, nEdges + !$omp do schedule(runtime) private(Nsurf, N, cell1, cell2, A, bTemp, C, m, rTemp, k) +#endif + do iEdge = 1, nEdgesOwned N = maxLevelEdgeTop(iEdge) Nsurf = minLevelEdgeBot(iEdge) if (N .gt. 0) then - ! Compute A(k), B(k), C(k) ! layerThicknessEdge is computed in compute_solve_diag, and is not available yet, ! so recompute layerThicknessEdge here. cell1 = cellsOnEdge(1,iEdge) cell2 = cellsOnEdge(2,iEdge) do k = Nsurf, N layerThicknessEdge(k,iEdge) = 0.5_RKIND * (layerThickness(k,cell1) + layerThickness(k,cell2)) - end do - - ! A is lower diagonal term - A(1:Nsurf)=0.0_RKIND - do k = Nsurf+1, N - A(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & - / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & - / layerThicknessEdge(k,iEdge) enddo - ! C is upper diagonal term - C(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N-1 - C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & - / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo + ! tridiagonal matrix algorithm + C(Nsurf) = -2.0_RKIND*dt*vertViscTopOfEdge(Nsurf+1,iEdge) & + / (layerThicknessEdge(Nsurf,iEdge) + layerThicknessEdge(Nsurf+1,iEdge)) & + / layerThicknessEdge(Nsurf,iEdge) + bTemp(Nsurf) = 1.0_RKIND - C(Nsurf) + rTemp(Nsurf) = normalVelocity(Nsurf,iEdge) - ! B is diagonal term - B(1:Nsurf-1)=0.0_RKIND - B(Nsurf) = 1.0_RKIND - C(Nsurf) + ! first pass: set the coefficients do k = Nsurf+1, N-1 - B(k) = 1.0_RKIND - A(k) - C(k) + A = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & + / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & + / layerThicknessEdge(k,iEdge) + m = A/bTemp(k-1) + C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & + / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & + / layerThicknessEdge(k,iEdge) + bTemp(k) = 1.0_RKIND - A - C(k) - m*C(k-1) + rTemp(k) = normalVelocity(k,iEdge) - m*rTemp(k-1) enddo - ! Apply bottom drag boundary condition on the viscous term - ! second line uses sqrt(2.0*kineticEnergyEdge(k,iEdge)) - B(N) = 1.0_RKIND - A(N) + dt*implicitBottomDragCoef & - * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) - - call tridiagonal_solve(A(Nsurf+1:N),B(Nsurf:N),C(Nsurf:N-1),normalVelocity(Nsurf:N,iEdge),velTemp(Nsurf:N),N-Nsurf+1) + A = -2.0_RKIND*dt*vertViscTopOfEdge(N,iEdge) & + / (layerThicknessEdge(N-1,iEdge) + layerThicknessEdge(N,iEdge)) & + / layerThicknessEdge(N,iEdge) + m = A/bTemp(N-1) + ! x(N) = rTemp(N) / bTemp(N) + ! Apply bottom drag boundary condition on the viscous term + ! using sqrt(2.0*kineticEnergyEdge(k,iEdge)) + normalVelocity(N,iEdge) = (normalVelocity(N,iEdge) - m*rTemp(N-1)) & + / (1.0_RKIND - A + dt*implicitBottomDragCoef & + * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) & + - m*C(N-1)) + ! second pass: back substitution + do k = N-1, Nsurf, -1 + normalVelocity(k,iEdge) = (rTemp(k) - C(k)*normalVelocity(k+1,iEdge)) / bTemp(k) + enddo normalVelocity(1:Nsurf-1,iEdge) = 0.0_RKIND - normalVelocity(Nsurf:N,iEdge) = velTemp(Nsurf:N) normalVelocity(N+1:nVertLevels,iEdge) = 0.0_RKIND end if end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(bTemp, C, rTemp) +#endif - deallocate(A,B,C,velTemp) + deallocate(bTemp,C,rTemp) !-------------------------------------------------------------------- @@ -506,7 +533,7 @@ end subroutine ocn_vel_vmix_tend_implicit!}}} ! !----------------------------------------------------------------------- - subroutine ocn_vel_vmix_tend_implicit_spatially_variable(meshPool, bottomDrag, dt, kineticEnergyCell, & !{{{ + subroutine ocn_vel_vmix_tend_implicit_spatially_variable(bottomDrag, dt, kineticEnergyCell, & !{{{ vertViscTopOfEdge, layerThickness, & layerThicknessEdge, normalVelocity, err) @@ -516,9 +543,6 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable(meshPool, bottomDrag, d ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - real (kind=RKIND), dimension(:,:), intent(in) :: & kineticEnergyCell !< Input: kinetic energy at cell @@ -560,39 +584,34 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable(meshPool, bottomDrag, d ! !----------------------------------------------------------------- - integer :: iEdge, k, cell1, cell2, N, Nsurf, nEdges - integer, pointer :: nVertLevels + integer :: iEdge, k, cell1, cell2, N, Nsurf real (kind=RKIND) :: implicitCd - integer, dimension(:), pointer :: nEdgesArray - - integer, dimension(:), pointer :: minLevelEdgeBot, maxLevelEdgeTop - - integer, dimension(:,:), pointer :: cellsOnEdge - real (kind=RKIND), dimension(:), allocatable :: A, B, C, velTemp + real (kind=RKIND), dimension(:), allocatable :: bTemp, C, rTemp + real (kind=RKIND) :: A, m err = 0 if(.not.velVmixOn) return - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) - - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - - nEdges = nEdgesArray( 1 ) + allocate(bTemp(nVertLevels),C(nVertLevels),rTemp(nVertLevels)) - allocate(A(nVertLevels),B(nVertLevels),C(nVertLevels),velTemp(nVertLevels)) +#ifdef MPAS_OPENACC + !$acc enter data create(bTemp, C, rTemp) + !$acc enter data create(bottomDrag) + !$acc parallel loop present(maxLevelEdgeTop, minLevelEdgeBot, cellsOnEdge, & + !$acc layerThicknessEdge, layerThickness, vertViscTopOfEdge, normalVelocity, & + !$acc kineticEnergyCell, bottomDrag) & + !$acc private(Nsurf, N, cell1, cell2, implicitCd, A, bTemp, C, m, rTemp, k) +#else !$omp do schedule(runtime) - do iEdge = 1, nEdges +#endif + do iEdge = 1, nEdgesOwned Nsurf = minLevelEdgeBot(iEdge) N = maxLevelEdgeTop(iEdge) if (N .gt. 0) then - ! Compute A(k), B(k), C(k) ! layerThicknessEdge is computed in compute_solve_diag, and is not available yet, ! so recompute layerThicknessEdge here. cell1 = cellsOnEdge(1,iEdge) @@ -604,46 +623,56 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable(meshPool, bottomDrag, d ! average cell-based implicit bottom drag to edges implicitCd = 0.5_RKIND*(bottomDrag(cell1) + bottomDrag(cell2)) - ! A is lower diagonal term - A(1:Nsurf)=0.0_RKIND - do k = Nsurf+1, N - A(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & - / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo + ! tridiagonal matrix algorithm + C(Nsurf) = -2.0_RKIND*dt*vertViscTopOfEdge(Nsurf+1,iEdge) & + / (layerThicknessEdge(Nsurf,iEdge) + layerThicknessEdge(Nsurf+1,iEdge)) & + / layerThicknessEdge(Nsurf,iEdge) + bTemp(Nsurf) = 1.0_RKIND - C(Nsurf) + rTemp(Nsurf) = normalVelocity(Nsurf,iEdge) - ! C is upper diagonal term - C(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N-1 - C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & - / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo - - ! B is diagonal term - B(1:Nsurf-1)=0.0_RKIND - B(Nsurf) = 1.0_RKIND - C(Nsurf) + ! first pass: set the coefficients do k = Nsurf+1, N-1 - B(k) = 1.0_RKIND - A(k) - C(k) + A = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & + / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & + / layerThicknessEdge(k,iEdge) + m = A/bTemp(k-1) + C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & + / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & + / layerThicknessEdge(k,iEdge) + bTemp(k) = 1.0_RKIND - A - C(k) - m*C(k-1) + rTemp(k) = normalVelocity(k,iEdge) - m*rTemp(k-1) enddo - ! Apply bottom drag boundary condition on the viscous term - ! second line uses sqrt(2.0*kineticEnergyEdge(k,iEdge)) - ! use implicitCd from spatially variable bottom drag - B(N) = 1.0_RKIND - A(N) + dt*implicitCd & - * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) - - call tridiagonal_solve(A(Nsurf+1:N),B(Nsurf:N),C(Nsurf:N-1),normalVelocity(Nsurf:N,iEdge),velTemp(Nsurf:N),N-Nsurf+1) + A = -2.0_RKIND*dt*vertViscTopOfEdge(N,iEdge) & + / (layerThicknessEdge(N-1,iEdge) + layerThicknessEdge(N,iEdge)) & + / layerThicknessEdge(N,iEdge) + m = A/bTemp(N-1) + ! x(N) = rTemp(N) / bTemp(N) + ! Apply bottom drag boundary condition on the viscous term + ! using sqrt(2.0*kineticEnergyEdge(k,iEdge)) + normalVelocity(N,iEdge) = (normalVelocity(N,iEdge) - m*rTemp(N-1)) & + / (1.0_RKIND - A + dt*implicitCD & + * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) & + - m*C(N-1)) + ! second pass: back substitution + do k = N-1, Nsurf, -1 + normalVelocity(k,iEdge) = (rTemp(k) - C(k)*normalVelocity(k+1,iEdge)) / bTemp(k) + enddo normalVelocity(1:Nsurf-1,iEdge) = 0.0_RKIND - normalVelocity(Nsurf:N,iEdge) = velTemp(Nsurf:N) normalVelocity(N+1:nVertLevels,iEdge) = 0.0_RKIND end if end do +#ifndef MPAS_OPENACC !$omp end do +#endif - deallocate(A,B,C,velTemp) +#ifdef MPAS_OPENACC + !$acc exit data delete(bTemp, C, rTemp) + !$acc exit data delete(bottomDrag) +#endif + deallocate(bTemp,C,rTemp) !-------------------------------------------------------------------- @@ -667,9 +696,9 @@ end subroutine ocn_vel_vmix_tend_implicit_spatially_variable!}}} ! !----------------------------------------------------------------------- - subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forcingPool, bottomDrag, dt, & !{{{ + subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(forcingPool, bottomDrag, dt, & !{{{ kineticEnergyCell, vertViscTopOfEdge, layerThickness, & - layerThicknessEdge, normalVelocity, ssh, bottomDepth, err) + layerThicknessEdge, normalVelocity, ssh, err) !----------------------------------------------------------------- ! @@ -677,9 +706,6 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - type (mpas_pool_type), intent(inout) :: & forcingPool !< Input: forcing information @@ -700,8 +726,6 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc real (kind=RKIND), dimension(:), pointer :: ssh - real (kind=RKIND), dimension(:), pointer :: bottomDepth - !----------------------------------------------------------------- ! ! input/output variables @@ -728,16 +752,11 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc ! !----------------------------------------------------------------- - integer :: iEdge, k, cell1, cell2, N, Nsurf, nEdges - integer, pointer :: nVertLevels + integer :: iEdge, k, cell1, cell2, N, Nsurf real (kind=RKIND) :: implicitCd - integer, dimension(:), pointer :: nEdgesArray - - integer, dimension(:), pointer :: minLevelEdgeBot, maxLevelEdgeTop - integer, dimension(:,:), pointer :: cellsOnEdge - - real (kind=RKIND), dimension(:), allocatable :: A, B, C, velTemp + real (kind=RKIND), dimension(:), allocatable :: bTemp, C, rTemp + real (kind=RKIND) :: A, m ! vegetation_drag real (kind=RKIND), dimension(:), pointer :: vegetationHeight @@ -747,36 +766,29 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc integer, dimension(:), pointer ::vegetationMask real (kind=RKIND) :: old_bottom_Cd, lambda, beta, alpha, total_h real (kind=RKIND) :: inundation_depth, von_karman, cff1, cff2, cff3, cff4 - integer :: iCell, nCells - integer, pointer :: nCellsSolve + integer :: iCell err = 0 von_karman = 0.4_RKIND if(.not.velVmixOn) return - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) - call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) - - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - call mpas_pool_get_array(forcingPool, 'vegetationMask', vegetationMask) call mpas_pool_get_array(forcingPool, 'vegetationHeight', vegetationHeight) call mpas_pool_get_array(forcingPool, 'vegetationDensity', vegetationDensity) call mpas_pool_get_array(forcingPool, 'vegetationDiameter', vegetationDiameter) call mpas_pool_get_array(forcingPool, 'vegetationManning', vegetationManning) - nEdges = nEdgesArray( 1 ) - nCells = nCellsSolve + allocate(bTemp(nVertLevels),C(nVertLevels),rTemp(nVertLevels)) - allocate(A(nVertLevels),B(nVertLevels),C(nVertLevels),velTemp(nVertLevels)) +#ifdef MPAS_OPENACC + !$acc enter data create(bTemp, C, rTemp) + !$acc enter data create(ssh, bottomDrag) +#endif ! Compute bottomDrag (Manning roughness) induced by vegetation if (config_use_vegetation_drag .AND. config_use_vegetation_manning_equation) then - do iCell = 1, nCells + do iCell = 1, nCellsOwned if (vegetationDensity(iCell) * vegetationHeight(iCell) * vegetationDiameter(iCell) .eq. 0.0_RKIND) then vegetationMask(iCell) = 0 endif @@ -805,13 +817,21 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc enddo endif +#ifdef MPAS_OPENACC + !$acc enter data create(vegetationManning) + + !$acc parallel loop present(maxLevelEdgeTop, minLevelEdgeBot, cellsOnEdge, & + !$acc layerThicknessEdge, layerThickness, vertViscTopOfEdge, normalVelocity, & + !$acc kineticEnergyCell, bottomDrag, vegetationManning, ssh, bottomDepth) & + !$acc private(Nsurf, N, cell1, cell2, implicitCd, A, bTemp, C, m, rTemp, k) +#else !$omp do schedule(runtime) - do iEdge = 1, nEdges +#endif + do iEdge = 1, nEdgesOwned Nsurf = minLevelEdgeBot(iEdge) N = maxLevelEdgeTop(iEdge) if (N .gt. 0) then - ! Compute A(k), B(k), C(k) ! layerThicknessEdge is computed in compute_solve_diag, and is not available yet, ! so recompute layerThicknessEdge here. cell1 = cellsOnEdge(1,iEdge) @@ -838,46 +858,56 @@ subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forc (0.5_RKIND * (ssh(cell1) + ssh(cell2) + bottomDepth(cell1) + bottomDepth(cell2)))**(-1.0_RKIND/3.0_RKIND) endif - ! A is lower diagonal term - A(1:Nsurf)=0.0_RKIND - do k = Nsurf+1, N - A(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & - / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo - - ! C is upper diagonal term - C(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N-1 - C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & - / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & - / layerThicknessEdge(k,iEdge) - enddo + ! tridiagonal matrix algorithm + C(Nsurf) = -2.0_RKIND*dt*vertViscTopOfEdge(Nsurf+1,iEdge) & + / (layerThicknessEdge(Nsurf,iEdge) + layerThicknessEdge(Nsurf+1,iEdge)) & + / layerThicknessEdge(Nsurf,iEdge) + bTemp(Nsurf) = 1.0_RKIND - C(Nsurf) + rTemp(Nsurf) = normalVelocity(Nsurf,iEdge) - ! B is diagonal term - B(1:Nsurf-1)=0.0_RKIND - B(Nsurf) = 1.0_RKIND - C(Nsurf) + ! first pass: set the coefficients do k = Nsurf+1, N-1 - B(k) = 1.0_RKIND - A(k) - C(k) + A = -2.0_RKIND*dt*vertViscTopOfEdge(k,iEdge) & + / (layerThicknessEdge(k-1,iEdge) + layerThicknessEdge(k,iEdge)) & + / layerThicknessEdge(k,iEdge) + m = A/bTemp(k-1) + C(k) = -2.0_RKIND*dt*vertViscTopOfEdge(k+1,iEdge) & + / (layerThicknessEdge(k,iEdge) + layerThicknessEdge(k+1,iEdge)) & + / layerThicknessEdge(k,iEdge) + bTemp(k) = 1.0_RKIND - A - C(k) - m*C(k-1) + rTemp(k) = normalVelocity(k,iEdge) - m*rTemp(k-1) enddo - ! Apply bottom drag boundary condition on the viscous term - ! second line uses sqrt(2.0*kineticEnergyEdge(k,iEdge)) - ! use implicitCd from spatially variable bottom drag - B(N) = 1.0_RKIND - A(N) + dt*implicitCd & - * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) - - call tridiagonal_solve(A(Nsurf+1:N),B(Nsurf:N),C(Nsurf:N-1),normalVelocity(Nsurf:N,iEdge),velTemp(Nsurf:N),N-Nsurf+1) + A = -2.0_RKIND*dt*vertViscTopOfEdge(N,iEdge) & + / (layerThicknessEdge(N-1,iEdge) + layerThicknessEdge(N,iEdge)) & + / layerThicknessEdge(N,iEdge) + m = A/bTemp(N-1) + ! x(N) = rTemp(N) / bTemp(N) + ! Apply bottom drag boundary condition on the viscous term + ! using sqrt(2.0*kineticEnergyEdge(k,iEdge)) + normalVelocity(N,iEdge) = (normalVelocity(N,iEdge) - m*rTemp(N-1)) & + / (1.0_RKIND - A + dt*implicitCD & + * sqrt(kineticEnergyCell(N,cell1) + kineticEnergyCell(N,cell2)) / layerThicknessEdge(N,iEdge) & + - m*C(N-1)) + ! second pass: back substitution + do k = N-1, Nsurf, -1 + normalVelocity(k,iEdge) = (rTemp(k) - C(k)*normalVelocity(k+1,iEdge)) / bTemp(k) + enddo normalVelocity(1:Nsurf-1,iEdge) = 0.0_RKIND - normalVelocity(Nsurf:N,iEdge) = velTemp(Nsurf:N) normalVelocity(N+1:nVertLevels,iEdge) = 0.0_RKIND end if end do +#ifndef MPAS_OPENACC !$omp end do +#endif - deallocate(A,B,C,velTemp) +#ifdef MPAS_OPENACC + !$acc exit data delete(bTemp, C, rTemp) + !$acc exit data delete(ssh, bottomDrag, vegetationManning) +#endif + deallocate(bTemp,C,rTemp) !-------------------------------------------------------------------- @@ -897,7 +927,7 @@ end subroutine ocn_vel_vmix_tend_implicit_spatially_variable_mannings!}}} ! !----------------------------------------------------------------------- - subroutine ocn_tracer_vmix_tend_implicit(meshPool, dt, vertDiffTopOfCell, layerThickness, tracers, & + subroutine ocn_tracer_vmix_tend_implicit(dt, vertDiffTopOfCell, layerThickness, tracers, & vertNonLocalFlux, tracerGroupSurfaceFlux, config_cvmix_kpp_nonlocal_with_implicit_mix, & err)!{{{ @@ -907,9 +937,6 @@ subroutine ocn_tracer_vmix_tend_implicit(meshPool, dt, vertDiffTopOfCell, layerT ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - real (kind=RKIND), dimension(:,:), intent(in) :: & vertDiffTopOfCell !< Input: vertical mixing coefficients @@ -947,80 +974,112 @@ subroutine ocn_tracer_vmix_tend_implicit(meshPool, dt, vertDiffTopOfCell, layerT ! !----------------------------------------------------------------- - integer :: iCell, k, num_tracers, N, Nsurf, nCells - integer, pointer :: nVertLevels - integer, dimension(:), pointer :: nCellsArray + integer :: iCell, k, iTracer, num_tracers, N, Nsurf - integer, dimension(:), pointer :: maxLevelCell, minLevelCell - - real (kind=RKIND), dimension(:), allocatable :: A,B,C - real (kind=RKIND), dimension(:,:), allocatable :: tracersTemp, rhs + real (kind=RKIND), dimension(:), allocatable :: bTemp, C + real (kind=RKIND), dimension(:,:), allocatable :: rTemp + real (kind=RKIND) :: A, m err = 0 if(.not.tracerVmixOn) return - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) num_tracers = size(tracers, dim=1) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) + allocate(bTemp(nVertLevels),C(nVertLevels)) + allocate(rTemp(num_tracers,nVertLevels)) - allocate(A(nVertLevels),B(nVertLevels),C(nVertLevels),tracersTemp(num_tracers,nVertLevels)) - allocate(rhs(num_tracers,nVertLevels)) + call mpas_timer_start('vmix tracers tend imp loop', .false.) - nCells = nCellsArray( 1 ) +#ifdef MPAS_OPENACC + !$acc enter data create(bTemp, C, rTemp) - call mpas_timer_start('vmix tracers tend imp loop', .false.) + !$acc parallel loop present(maxLevelCell, minLevelCell, vertDiffTopOfCell, & + !$acc layerThickness, tracers, tracerGroupSurfaceFlux, vertNonLocalFlux) & + !$acc private(Nsurf, N, A, bTemp, C, m, rTemp, iTracer, k) +#else !$omp parallel - !$omp do schedule(runtime) private(Nsurf, N, A, B, C, rhs, tracersTemp, k) - do iCell = 1, nCells - ! Compute A(k), B(k), C(k) for tracers + !$omp do schedule(runtime) private(Nsurf, N, A, bTemp, C, m, rTemp, iTracer, k) +#endif + do iCell = 1, nCellsOwned Nsurf = minLevelCell(iCell) N = maxLevelCell(iCell) - ! A is lower diagonal term - A(1:Nsurf)=0.0_RKIND - do k = Nsurf+1, N - A(k) = -2.0_RKIND*dt*vertDiffTopOfCell(k,iCell) & - / (layerThickness(k-1,iCell) + layerThickness(k,iCell)) / layerThickness(k,iCell) - enddo + ! tridiagonal matrix algorithm + C(Nsurf) = -2.0_RKIND*dt*vertDiffTopOfCell(Nsurf+1,iCell) & + / (layerThickness(Nsurf,iCell) + layerThickness(Nsurf+1,iCell)) & + / layerThickness(Nsurf,iCell) + bTemp(Nsurf) = 1.0_RKIND - C(Nsurf) + if ( config_cvmix_kpp_nonlocal_with_implicit_mix ) then + do iTracer = 1, num_tracers + rTemp(iTracer,Nsurf) = tracers(iTracer,Nsurf,iCell) + dt*tracerGroupSurfaceFlux(iTracer,iCell) & + * (-vertNonLocalFlux(1, Nsurf+1,iCell) )/ layerThickness(Nsurf,iCell) + enddo + else + do iTracer = 1, num_tracers + rTemp(iTracer,Nsurf) = tracers(iTracer,Nsurf,iCell) + enddo + endif - ! C is upper diagonal term - C(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N-1 - C(k) = -2.0_RKIND*dt*vertDiffTopOfCell(k+1,iCell) & - / (layerThickness(k,iCell) + layerThickness(k+1,iCell)) / layerThickness(k,iCell) + ! first pass: set the coefficients + do k = Nsurf+1, N-1 + A = -2.0_RKIND*dt*vertDiffTopOfCell(k,iCell) & + / (layerThickness(k-1,iCell) + layerThickness(k,iCell)) / layerThickness(k,iCell) + m = A/bTemp(k-1) + C(k) = -2.0_RKIND*dt*vertDiffTopOfCell(k+1,iCell) & + / (layerThickness(k,iCell) + layerThickness(k+1,iCell)) / layerThickness(k,iCell) + bTemp(k) = 1.0_RKIND - A - C(k) - m*C(k-1) + if ( config_cvmix_kpp_nonlocal_with_implicit_mix ) then + do iTracer = 1, num_tracers + rTemp(iTracer,k) = tracers(iTracer,k,iCell) + dt*tracerGroupSurfaceFlux(iTracer,iCell) & + * (vertNonLocalFlux(1,k,iCell) - vertNonLocalFlux(1,k+1,iCell)) / layerThickness(k,iCell) & + - m*rTemp(iTracer,k-1) + enddo + else + do iTracer = 1, num_tracers + rTemp(iTracer,k) = tracers(iTracer,k,iCell) - m*rTemp(iTracer,k-1) + enddo + endif enddo - C(N) = 0.0_RKIND - ! B is diagonal term - B(1:Nsurf-1)=0.0_RKIND - do k = Nsurf, N - B(k) = 1.0_RKIND - A(k) - C(k) - enddo + A = -2.0_RKIND*dt*vertDiffTopOfCell(N,iCell) & + / (layerThickness(N-1,iCell) + layerThickness(N,iCell)) / layerThickness(N,iCell) + m = A/bTemp(N-1) + ! x(N) = rTemp(N) / bTemp(N) if ( config_cvmix_kpp_nonlocal_with_implicit_mix ) then - call ocn_compute_kpp_rhs(tracers(:,:,iCell), rhs(:,:), dt, Nsurf, N, num_tracers, & - layerThickness(:,iCell), vertNonLocalFlux(:,:,iCell), & - tracerGroupSurfaceFlux(:,iCell)) + do iTracer = 1, num_tracers + tracers(iTracer,N,iCell) = (tracers(iTracer,N,iCell) + dt*tracerGroupSurfaceFlux(iTracer,iCell) & + * vertNonLocalFlux(1,N,iCell) / layerThickness(N,iCell) & + - m*rTemp(iTracer,N-1)) / (1.0_RKIND - A - m*C(N-1)) + enddo else - rhs(:,:) = tracers(:,:,iCell) + do iTracer = 1, num_tracers + tracers(iTracer,N,iCell) = (tracers(iTracer,N,iCell) - m*rTemp(iTracer,N-1)) & + / (1.0_RKIND - A - m*C(N-1)) + enddo endif - - call tridiagonal_solve_mult(A(Nsurf+1:N), B(Nsurf:N), C(Nsurf:N-1), rhs(:,Nsurf:N), & - tracersTemp(:,Nsurf:N), N-Nsurf+1, num_tracers) - - tracers(:,Nsurf:N,iCell) = tracersTemp(:,Nsurf:N) - tracers(:,N+1:nVertLevels,iCell) = -1e34 + ! second pass: back subsititution + do k = N-1, Nsurf, -1 + do iTracer = 1, num_tracers + tracers(iTracer,k,iCell) = (rTemp(iTracer,k) - C(k)*tracers(iTracer,k+1,iCell)) / bTemp(k) + enddo + enddo tracers(:,1:Nsurf-1,iCell) = -1e34 + tracers(:,N+1:nVertLevels,iCell) = -1e34 + end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(bTemp, C, rTemp) +#endif call mpas_timer_stop('vmix tracers tend imp loop') - deallocate(A, B, C, tracersTemp, rhs) + deallocate(bTemp, C, rTemp) !-------------------------------------------------------------------- @@ -1051,15 +1110,12 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, type (mpas_pool_type), pointer :: tracersPool, tracersSurfaceFluxPool - integer :: iCell, timeLevel, k, cell1, cell2, iEdge, nCells, nEdges - integer, dimension(:), pointer :: nCellsArray, nEdgesArray - real (kind=RKIND), dimension(:), pointer :: bottomDrag, ssh, bottomDepth ! needed for depth-variable computation + integer :: iCell, timeLevel, k, cell1, cell2, iEdge, iTracer, num_tracers, nCells, nEdges, N, Nsurf + real (kind=RKIND), dimension(:), pointer :: bottomDrag, ssh real (kind=RKIND), dimension(:,:), pointer :: normalVelocity, layerThickness real (kind=RKIND), dimension(:,:), pointer :: tracerGroupSurfaceFlux real (kind=RKIND), dimension(:,:,:), pointer :: tracersGroup real (kind=RKIND), dimension(:,:,:), allocatable :: nonLocalFluxTend - integer, dimension(:), pointer :: maxLevelEdgeTop, minLevelEdgeTop, minLevelEdgeBot - integer, dimension(:,:), pointer :: cellsOnEdge type (mpas_pool_iterator_type) :: groupItr @@ -1081,83 +1137,108 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, call mpas_pool_get_array(statePool, 'layerThickness', layerThickness, timeLevel) call mpas_pool_get_array(statePool, 'ssh', ssh, timeLevel) - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'minLevelEdgeTop', minLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - call mpas_pool_get_array(meshPool, 'bottomDepth', bottomDepth) - - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_dimension(meshPool, 'nEdgesArray', nEdgesArray) - call mpas_pool_get_array(forcingPool, 'bottomDrag', bottomDrag) call mpas_timer_start('vmix coefs', .false.) call ocn_vmix_coefs(meshPool, statePool, forcingPool, scratchPool, err, timeLevel) call mpas_timer_stop('vmix coefs') - nCells = nCellsArray(1) ! if using CVMix, then viscosity has to be averaged from cell centers to cell edges if ( config_use_cvmix ) then - nEdges = nEdgesArray( 1 ) call mpas_timer_start('CVMix avg', .false.) +#ifdef MPAS_OPENACC + !$acc parallel loop present(cellsOnEdge, minLevelEdgeBot, maxLevelEdgeTop, & + !$acc vertViscTopOfCell, vertViscTopOfEdge) & + !$acc private(cell1, cell2, k, Nsurf, N) +#else !$omp parallel - !$omp do schedule(runtime) private(cell1, cell2, k) - do iEdge=1,nEdges - vertViscTopOfEdge(:, iEdge) = 0.0_RKIND + !$omp do schedule(runtime) private(cell1, cell2, k, Nsurf, N) +#endif + do iEdge=1,nEdgesOwned + Nsurf = minLevelEdgeBot(iEdge) + N = maxLevelEdgeTop(iEdge) + vertViscTopOfEdge(1:Nsurf-1, iEdge) = 0.0_RKIND + vertViscTopOfEdge(N+1:nVertLevels, iEdge) = 0.0_RKIND cell1 = cellsOnEdge(1,iEdge) cell2 = cellsOnEdge(2,iEdge) - do k=minLevelEdgeBot(iEdge),maxLevelEdgeTop(iEdge) +#ifdef MPAS_OPENACC + !$acc loop vector +#endif + do k=Nsurf, N vertViscTopOfEdge(k,iEdge) = 0.5_RKIND*(vertViscTopOfCell(k,cell2)+vertViscTopOfCell(k,cell1)) end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif call mpas_timer_stop('CVMix avg') endif ! if using GOTM, then viscosity has to be averaged from cell centers to cell edges if ( config_use_gotm ) then - nEdges = nEdgesArray( 1 ) call mpas_timer_start('GOTM avg', .false.) - !$omp do schedule(runtime) private(cell1, cell2, k) - do iEdge=1,nEdges - vertViscTopOfEdge(:, iEdge) = 0.0_RKIND +#ifdef MPAS_OPENACC + !$acc parallel loop present(cellsOnEdge, minLevelEdgeBot, maxLevelEdgeTop, & + !$acc vertViscTopOfCell, vertViscTopOfEdge) & + !$acc private(cell1, cell2, k, Nsurf, N) +#else + !$omp do schedule(runtime) private(cell1, cell2, k, Nsurf, N) +#endif + do iEdge=1,nEdgesOwned + Nsurf = minLevelEdgeBot(iEdge) + N = maxLevelEdgeTop(iEdge) + vertViscTopOfEdge(1:Nsurf-1, iEdge) = 0.0_RKIND + vertViscTopOfEdge(N+1:nVertLevels, iEdge) = 0.0_RKIND cell1 = cellsOnEdge(1,iEdge) cell2 = cellsOnEdge(2,iEdge) - do k=minLevelEdgeBot(iEdge),maxLevelEdgeTop(iEdge) +#ifdef MPAS_OPENACC + !$acc loop vector +#endif + do k=Nsurf, N vertViscTopOfEdge(k,iEdge) = 0.5_RKIND*(vertViscTopOfCell(k,cell2)+vertViscTopOfCell(k,cell1)) end do end do +#ifndef MPAS_OPENACC !$omp end do +#endif call mpas_timer_stop('GOTM avg') endif +#ifdef MPAS_OPENACC + !$acc update host(vertViscTopOfEdge) +#endif ! ! Implicit vertical solve for momentum ! call mpas_timer_start('vmix solve momentum', .false.) +#ifdef MPAS_OPENACC + !$acc enter data copyin(layerThickness, normalVelocity) +#endif if (config_use_implicit_bottom_drag_variable) then - call ocn_vel_vmix_tend_implicit_spatially_variable(meshPool, bottomDrag, dt, kineticEnergyCell, & + call ocn_vel_vmix_tend_implicit_spatially_variable(bottomDrag, dt, kineticEnergyCell, & vertViscTopOfEdge, layerThickness, layerThickEdge, normalVelocity, err) else if (config_use_implicit_bottom_drag_variable_mannings.or. & config_use_implicit_bottom_roughness) then ! update bottomDrag via Cd=g*n^2*h^(-1/3) - call ocn_vel_vmix_tend_implicit_spatially_variable_mannings(meshPool, forcingPool, bottomDrag, & + call ocn_vel_vmix_tend_implicit_spatially_variable_mannings(forcingPool, bottomDrag, & dt, kineticEnergyCell, & vertViscTopOfEdge, layerThickness, layerThickEdge, normalVelocity, & - ssh, bottomDepth, err) + ssh, err) else if (config_Rayleigh_friction.or. & config_Rayleigh_bottom_friction.or. & config_Rayleigh_damping_depth_variable) then - call ocn_vel_vmix_tend_implicit_rayleigh(meshPool, dt, kineticEnergyCell, & + call ocn_vel_vmix_tend_implicit_rayleigh(dt, kineticEnergyCell, & vertViscTopOfEdge, layerThickness, layerThickEdge, normalVelocity, err) else - call ocn_vel_vmix_tend_implicit(meshPool, dt, kineticEnergyCell, & + call ocn_vel_vmix_tend_implicit(dt, kineticEnergyCell, & vertViscTopOfEdge, layerThickness, layerThickEdge, normalVelocity, err) end if +#ifdef MPAS_OPENACC + !$acc exit data copyout(normalVelocity) +#endif call mpas_timer_stop('vmix solve momentum') ! @@ -1170,16 +1251,30 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, if ( groupItr % memberType == MPAS_POOL_FIELD ) then call mpas_pool_get_array(tracersPool, groupItr % memberName, tracersGroup, timeLevel) +#ifdef MPAS_OPENACC + !$acc enter data copyin(tracersGroup) +#endif ! store tracers if (trim(groupItr % memberName) == 'activeTracers') then if (config_compute_active_tracer_budgets) then + num_tracers = size(activeTracerVertMixTendency, dim=1) +#ifdef MPAS_OPENACC + !$acc parallel loop gang vector collapse(3) present(activeTracerVertMixTendency, tracersGroup) +#else !$omp parallel - !$omp do schedule(runtime) - do iCell = 1, nCells - activeTracerVertMixTendency(:,:,iCell)=tracersGroup(:,:,iCell) + !$omp do schedule(runtime) private(k, iTracer) +#endif + do iCell = 1, nCellsOwned + do k = 1, nVertLevels + do iTracer = 1, num_tracers + activeTracerVertMixTendency(iTracer,k,iCell)=tracersGroup(iTracer,k,iCell) end do + end do + end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif endif endif @@ -1192,8 +1287,11 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, call mpas_pool_get_array(tracersSurfaceFluxPool, trim(modifiedGroupName), & tracerGroupSurfaceFlux) endif +#ifdef MPAS_OPENACC + !$acc enter data copyin(tracerGroupSurfaceFlux) +#endif - call ocn_tracer_vmix_tend_implicit(meshPool, dt, vertDiffTopOfCell, layerThickness, tracersGroup, & + call ocn_tracer_vmix_tend_implicit(dt, vertDiffTopOfCell, layerThickness, tracersGroup, & vertNonLocalFlux, tracerGroupSurfaceFlux, & config_cvmix_kpp_nonlocal_with_implicit_mix, err) end if @@ -1201,21 +1299,42 @@ subroutine ocn_vmix_implicit(dt, meshPool, statePool, forcingPool, scratchPool, ! difference tracers to compute influence of vertical mixing and divide by dt if (trim(groupItr % memberName) == 'activeTracers') then if (config_compute_active_tracer_budgets) then +#ifdef MPAS_OPENACC + !$acc parallel loop gang vector collapse(3) present(activeTracerVertMixTendency, tracersGroup) +#else !$omp parallel - !$omp do schedule(runtime) - do iCell = 1, nCells - activeTracerVertMixTendency(:,:,iCell) = & - (tracersGroup(:,:,iCell) - activeTracerVertMixTendency(:,:,iCell)) / dt + !$omp do schedule(runtime) private(k, iTracer) +#endif + do iCell = 1, nCellsOwned + do k = 1, nVertLevels + do iTracer = 1, num_tracers + activeTracerVertMixTendency(iTracer,k,iCell) = & + (tracersGroup(iTracer,k,iCell) - activeTracerVertMixTendency(iTracer,k,iCell)) / dt + end do end do + end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc update host(activeTracerVertMixTendency) +#endif endif endif +#ifdef MPAS_OPENACC + !$acc exit data copyout(tracersGroup) delete(tracerGroupSurfaceFlux) +#endif end if end do call mpas_timer_stop('vmix solve tracers') +#ifdef MPAS_OPENACC + !$acc exit data delete(layerThickness) +#endif + call mpas_timer_stop('vmix imp') end subroutine ocn_vmix_implicit!}}} @@ -1290,169 +1409,6 @@ subroutine ocn_vmix_init(domain, err)!{{{ !-------------------------------------------------------------------- end subroutine ocn_vmix_init!}}} - -!*********************************************************************** -! -! routine tridiagonal_solve -! -!> \brief Solve the matrix equation Ax=r for x, where A is tridiagonal. -!> \author Mark Petersen -!> \date September 2011 -!> \details -!> Solve the matrix equation Ax=r for x, where A is tridiagonal. -!> A is an nxn matrix, with: -!> a sub-diagonal, filled from 1:n-1 (a(1) appears on row 2) -!> b diagonal, filled from 1:n -!> c sup-diagonal, filled from 1:n-1 (c(1) apears on row 1) -! -!----------------------------------------------------------------------- - subroutine tridiagonal_solve(a,b,c,r,x,n) !{{{ - - !----------------------------------------------------------------- - ! - ! input variables - ! - !----------------------------------------------------------------- - - integer,intent(in) :: n - real (KIND=RKIND), dimension(n), intent(in) :: a,b,c,r - - !----------------------------------------------------------------- - ! - ! output variables - ! - !----------------------------------------------------------------- - - real (KIND=RKIND), dimension(n), intent(out) :: x - - !----------------------------------------------------------------- - ! - ! local variables - ! - !----------------------------------------------------------------- - - real (KIND=RKIND), dimension(n) :: bTemp,rTemp - real (KIND=RKIND) :: m - integer i - - ! Use work variables for b and r - bTemp(1) = b(1) - rTemp(1) = r(1) - - ! First pass: set the coefficients - do i = 2,n - m = a(i-1)/bTemp(i-1) - bTemp(i) = b(i) - m*c(i-1) - rTemp(i) = r(i) - m*rTemp(i-1) - end do - - x(n) = rTemp(n)/bTemp(n) - ! Second pass: back-substition - do i = n-1, 1, -1 - x(i) = (rTemp(i) - c(i)*x(i+1))/bTemp(i) - end do - - end subroutine tridiagonal_solve !}}} - -!*********************************************************************** -! -! routine tridiagonal_solve_mult -! -!> \brief Solve multiple matrix equations Ax=r for x, where A is tridiagonal. -!> \author Mark Petersen -!> \date September 2011 -!> \details -!> Solve the matrix equation Ax=r for x, where A is tridiagonal. -!> A is an nxn matrix, with: -!> a sub-diagonal, filled from 1:n-1 (a(1) appears on row 2) -!> b diagonal, filled from 1:n -!> c sup-diagonal, filled from 1:n-1 (c(1) apears on row 1) -! -!----------------------------------------------------------------------- -subroutine tridiagonal_solve_mult(a,b,c,r,x,n,nSystems)!{{{ - - integer,intent(in) :: n, nSystems - real (KIND=RKIND), dimension(n), intent(in) :: a,b,c - real (KIND=RKIND), dimension(nSystems,n), intent(in) :: r - real (KIND=RKIND), dimension(nSystems,n), intent(out) :: x - real (KIND=RKIND), dimension(n) :: bTemp - real (KIND=RKIND), dimension(nSystems,n) :: rTemp - real (KIND=RKIND) :: m - integer i,j - - ! Use work variables for b and r - bTemp(1) = b(1) - do j = 1,nSystems - rTemp(j,1) = r(j,1) - end do - - ! First pass: set the coefficients - do i = 2,n - m = a(i-1)/bTemp(i-1) - bTemp(i) = b(i) - m*c(i-1) - do j = 1,nSystems - rTemp(j,i) = r(j,i) - m*rTemp(j,i-1) - end do - end do - - do j = 1,nSystems - x(j,n) = rTemp(j,n)/bTemp(n) - end do - ! Second pass: back-substition - do i = n-1, 1, -1 - do j = 1,nSystems - x(j,i) = (rTemp(j,i) - c(i)*x(j,i+1))/bTemp(i) - end do - end do - -end subroutine tridiagonal_solve_mult!}}} - -!*********************************************************************** -! -! subroutine ocn_compute_kpp_rhs -! -!> \brief Computes the non local flux tendency for KPP -!> \author Luke Van Roekel -!> \date October 2017 -!> \details -!> Computes non local flux tendency from KPP when -!> config_cvmix_kpp_nonlocal_with_implicit_mix = .true. -!> otherwise this term is computed in ocn_tend_tracer -! -!----------------------------------------------------------------------- - -subroutine ocn_compute_kpp_rhs(tracers, rhs, dt, minLevelCell, maxLevelCell, nTracers, & - layerThickness, vertNonLocalFlux, tracerGroupSurfaceFlux)!{{{ - - real (kind=RKIND), intent(in) :: dt - real (kind=RKIND), dimension(:,:), intent(in) :: vertNonLocalFlux, tracers - real (kind=RKIND), dimension(:), intent(in) :: layerThickness, tracerGroupSurfaceFlux - real (kind=RKIND), dimension(:,:), intent(out) :: rhs - integer, intent(in) :: minLevelCell, maxLevelCell, nTracers - integer :: iTracer, k - - rhs(:,1:minLevelCell-1) = 0.0_RKIND - - do k=minLevelCell+1,maxLevelCell-1 - do iTracer=1,nTracers - rhs(iTracer, k) = tracers(iTracer,k) + dt * tracerGroupSurfaceFlux(iTracer) * & - (vertNonLocalFlux(1,k) - vertNonLocalFlux(1,k+1)) / layerThickness(k) - enddo - enddo - - k=minLevelCell - do iTracer=1,nTracers - rhs(iTracer, k) = tracers(iTracer,k) + dt * tracerGroupSurfaceFlux(iTracer) * & - (-vertNonLocalFlux(1,k+1) )/ layerThickness(k) - enddo - - k=maxLevelCell - do iTracer=1,nTracers - rhs(iTracer,k) = tracers(iTracer,k) + dt * tracerGroupSurfaceFlux(iTracer) * & - vertNonLocalFlux(1,k) / layerThickness(k) - enddo - -end subroutine ocn_compute_kpp_rhs!}}} !*********************************************************************** end module ocn_vmix From 8a6a0e9c428ca233945d82aaf2acd6685c09e2f1 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 29 Nov 2021 14:55:38 -0800 Subject: [PATCH 171/301] Undo commenting out of glc elev class warnings in nuopc driver (since not regularly use by e3sm). --- share/nuopc/glc_elevclass_mod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/nuopc/glc_elevclass_mod.F90 b/share/nuopc/glc_elevclass_mod.F90 index 94aa514c54fe..114b6438cce9 100644 --- a/share/nuopc/glc_elevclass_mod.F90 +++ b/share/nuopc/glc_elevclass_mod.F90 @@ -211,8 +211,8 @@ subroutine glc_get_elevation_classes_without_bareland(glc_topo, glc_elevclass, l case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) -! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) -! write(logunit,*) glc_errcode_to_string(err_code) + write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) + write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) From 5c771a510937aa1238f73b707333e925569c9b04 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 29 Nov 2021 15:05:04 -0800 Subject: [PATCH 172/301] Undo additional commenting out of glc elev class warnings in nuopc driver (because e3sm does not regularly use this driver). --- share/nuopc/glc_elevclass_mod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/nuopc/glc_elevclass_mod.F90 b/share/nuopc/glc_elevclass_mod.F90 index 114b6438cce9..3a984f642127 100644 --- a/share/nuopc/glc_elevclass_mod.F90 +++ b/share/nuopc/glc_elevclass_mod.F90 @@ -534,8 +534,8 @@ subroutine glc_get_fractional_icecov(nec, glc_topo, glc_icefrac, glc_icefrac_ec, case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) -! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) -! write(logunit,*) glc_errcode_to_string(err_code) + write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) + write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) From a5cb66a2c5a40d30c2498697bc97c6b386e40bac Mon Sep 17 00:00:00 2001 From: Brian O'Neill Date: Mon, 29 Nov 2021 18:40:27 -0500 Subject: [PATCH 173/301] remove deleted subroutines from module member function declarations declarations for nonexistant member functions appears to have caused ci build error --- components/mpas-ocean/src/shared/mpas_ocn_vmix.F | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F index d3446dcc6196..02047b5752a2 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix.F @@ -51,15 +51,11 @@ module ocn_vmix ! !-------------------------------------------------------------------- - private :: tridiagonal_solve, & - tridiagonal_solve_mult - public :: ocn_vmix_coefs, & ocn_vel_vmix_tend_implicit, & ocn_tracer_vmix_tend_implicit, & ocn_vmix_init, & - ocn_vmix_implicit, & - ocn_compute_kpp_rhs + ocn_vmix_implicit !-------------------------------------------------------------------- ! From 42b5e7bf4b14117b952c324d7df6e1182aed9fe7 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 29 Nov 2021 16:04:33 -0800 Subject: [PATCH 174/301] Update code so that ice sheet upper surface value passed to coupler is the max of either the ice sheet surface or 0 (sea level). Corrects excessive warnings regarding elevation classes below sea level. --- components/mpas-albany-landice/driver/glc_comp_mct.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/driver/glc_comp_mct.F b/components/mpas-albany-landice/driver/glc_comp_mct.F index 9418e6126710..c5f2f4b0604b 100644 --- a/components/mpas-albany-landice/driver/glc_comp_mct.F +++ b/components/mpas-albany-landice/driver/glc_comp_mct.F @@ -1475,7 +1475,7 @@ subroutine glc_export_mct(g2x_g, errorCode) g2x_g % rAttr(index_g2x_Figg_rofi,n)=0.0 !...and remove these placeholders g2x_g % rAttr(index_g2x_Fogg_rofl,n) = 0.0 !Attach to subglacial liquid flux once present - g2x_g % rAttr(index_g2x_Sg_topo, n) = upperSurface(i) + g2x_g % rAttr(index_g2x_Sg_topo, n) = max(0.0, upperSurface(i)) !updated to avoid warning for values below sea level g2x_g % rAttr(index_g2x_Sg_tbot, n) = temperature(nVertlevels,i) - SHR_CONST_TKTRIP ! layerThickness is not populated on init, so calculating like this instead: g2x_g % rAttr(index_g2x_Sg_dztbot, n) = layerThicknessFractions(nVertLevels) * thickness(i) / 2.0 From c463b6e9b483a1e1db6b7ec9fba918f86723057d Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 29 Nov 2021 16:11:01 -0800 Subject: [PATCH 175/301] Make glc elev class warnings active again (commit number 42b5e7bf4b14117b952c324d7df6e1182aed9fe7 fixes the problem of excessive warnings for case where ice sheet upper surface and bed topography elevations are equal to one another). --- driver-mct/main/map_glc2lnd_mod.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/driver-mct/main/map_glc2lnd_mod.F90 b/driver-mct/main/map_glc2lnd_mod.F90 index 131a16c3dadc..4ae638ed71cf 100644 --- a/driver-mct/main/map_glc2lnd_mod.F90 +++ b/driver-mct/main/map_glc2lnd_mod.F90 @@ -255,8 +255,8 @@ subroutine get_glc_elevation_classes(glc_topo, glc_elevclass) case (GLC_ELEVCLASS_ERR_NONE) ! Do nothing case (GLC_ELEVCLASS_ERR_TOO_LOW, GLC_ELEVCLASS_ERR_TOO_HIGH) -! write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) -! write(logunit,*) glc_errcode_to_string(err_code) + write(logunit,*) subname, ': WARNING, for glc_pt, topo = ', glc_pt, glc_topo(glc_pt) + write(logunit,*) glc_errcode_to_string(err_code) case default write(logunit,*) subname, ': ERROR getting elevation class for glc_pt = ', glc_pt write(logunit,*) glc_errcode_to_string(err_code) From cb988336e7002df5b717c837a6816a6524b3269d Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 29 Nov 2021 18:37:19 -0600 Subject: [PATCH 176/301] remove phys const rearth from sp ops --- .../homme/src/share/cxx/SphereOperators.hpp | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/components/homme/src/share/cxx/SphereOperators.hpp b/components/homme/src/share/cxx/SphereOperators.hpp index 8888cf4ad798..a5c9dd91bfa4 100644 --- a/components/homme/src/share/cxx/SphereOperators.hpp +++ b/components/homme/src/share/cxx/SphereOperators.hpp @@ -13,7 +13,6 @@ #include "ReferenceElement.hpp" #include "Dimensions.hpp" #include "KernelVariables.hpp" -#include "PhysicalConstants.hpp" #include "utilities/SubviewUtils.hpp" #include "utilities/ViewUtils.hpp" @@ -145,8 +144,8 @@ class SphereOperators dsdx += dvv(l, i) * scalar(j, i); dsdy += dvv(l, i) * scalar(i, j); } - temp_v_buf(0, j, l) = dsdx * PhysicalConstants::rrearth; - temp_v_buf(1, l, j) = dsdy * PhysicalConstants::rrearth; + temp_v_buf(0, j, l) = dsdx * m_rrearth; + temp_v_buf(1, l, j) = dsdy * m_rrearth; }); kv.team_barrier(); @@ -182,8 +181,8 @@ class SphereOperators dsdx += dvv(l, i) * scalar(j, i); dsdy += dvv(l, i) * scalar(i, j); } - temp_v_buf(0, j, l) = dsdx * PhysicalConstants::rrearth; - temp_v_buf(1, l, j) = dsdy * PhysicalConstants::rrearth; + temp_v_buf(0, j, l) = dsdx * m_rrearth; + temp_v_buf(1, l, j) = dsdy * m_rrearth; }); kv.team_barrier(); @@ -234,7 +233,7 @@ class SphereOperators dvdy += dvv(igp, kgp) * gv_buf(1, kgp, jgp); } div_v(igp,jgp) = (dudx + dvdy) * ((1.0 / metdet(igp,jgp)) * - PhysicalConstants::rrearth); + m_rrearth); }); kv.team_barrier(); } @@ -283,7 +282,7 @@ class SphereOperators for (int jgp = 0; jgp < NP; ++jgp) { dd -= (spheremp(ngp, jgp) * gv_buf(0, ngp, jgp) * dvv(jgp, mgp) + spheremp(jgp, mgp) * gv_buf(1, jgp, mgp) * dvv(jgp, ngp)) * - PhysicalConstants::rrearth; + m_rrearth; } div_v(ngp, mgp) = dd; }); @@ -331,7 +330,7 @@ class SphereOperators } vort(igp, jgp) = (dvdx - dudy) * ((1.0 / metdet(igp, jgp)) * - PhysicalConstants::rrearth); + m_rrearth); }); kv.team_barrier(); } @@ -396,8 +395,8 @@ class SphereOperators v0 += dvv(jgp, kgp) * scalar(igp, kgp, ilev); v1 += dvv(igp, kgp) * scalar(kgp, jgp, ilev); } - v0 *= PhysicalConstants::rrearth; - v1 *= PhysicalConstants::rrearth; + v0 *= m_rrearth; + v1 *= m_rrearth; grad_s(0,igp,jgp,ilev) = D_inv(0,0,igp,jgp) * v0 + D_inv(0,1,igp,jgp) * v1; grad_s(1,igp,jgp,ilev) = D_inv(1,0,igp,jgp) * v0 + D_inv(1,1,igp,jgp) * v1; }); @@ -430,8 +429,8 @@ class SphereOperators dsdx += dvv(jgp, kgp) * scalar(igp, kgp, ilev); dsdy += dvv(igp, kgp) * scalar(kgp, jgp, ilev); } - dsdx *= PhysicalConstants::rrearth; - dsdy *= PhysicalConstants::rrearth; + dsdx *= m_rrearth; + dsdy *= m_rrearth; grad_s(0,igp,jgp,ilev) += D_inv(0,0,igp,jgp) * dsdx + D_inv(0,1,igp,jgp) * dsdy; grad_s(1,igp,jgp,ilev) += D_inv(1,0,igp,jgp) * dsdx + D_inv(1,1,igp,jgp) * dsdy; }); @@ -496,7 +495,7 @@ class SphereOperators dudx += dvv(jgp, kgp) * gv_buf(0, igp, kgp, ilev); dvdy += dvv(igp, kgp) * gv_buf(1, kgp, jgp, ilev); } - combine((dudx + dvdy) * (1.0 / metdet(igp, jgp) * PhysicalConstants::rrearth), + combine((dudx + dvdy) * (1.0 / metdet(igp, jgp) * m_rrearth), div_v(igp, jgp, ilev), alpha, beta); }); }); @@ -551,7 +550,7 @@ class SphereOperators } const Scalar qtensijk0 = add_hyperviscosity ? qtens(igp,jgp,ilev) : 0; qtens(igp,jgp,ilev) = (qdp(igp,jgp,ilev) + - alpha*((dudx + dvdy) * (1.0 / metdet(igp,jgp) * PhysicalConstants::rrearth)) + + alpha*((dudx + dvdy) * (1.0 / metdet(igp,jgp) * m_rrearth)) + qtensijk0); }); }); @@ -601,7 +600,7 @@ class SphereOperators dudy += dvv(igp, kgp) * vcov_buf(0, kgp, jgp, ilev); } vort(igp, jgp, ilev) = (dvdx - dudy) * (1.0 / metdet(igp, jgp) * - PhysicalConstants::rrearth); + m_rrearth); }); }); kv.team_barrier(); @@ -651,7 +650,7 @@ class SphereOperators dudy += dvv(igp, kgp) * sphere_buf(0, kgp, jgp, ilev); } vort(igp, jgp, ilev) = (dvdx - dudy) * (1.0 / metdet(igp, jgp) * - PhysicalConstants::rrearth); + m_rrearth); }); }); kv.team_barrier(); @@ -702,7 +701,7 @@ class SphereOperators // Here, v is the temporary buffer, aliased on the input v. dd -= (spheremp(ngp, jgp) * v(0, ngp, jgp, ilev) * dvv(jgp, mgp) + spheremp(jgp, mgp) * v(1, jgp, mgp, ilev) * dvv(jgp, ngp)) * - PhysicalConstants::rrearth; + m_rrearth; } div_v(ngp, mgp, ilev) = dd; }); @@ -806,8 +805,8 @@ class SphereOperators Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team, NUM_LEV_REQUEST), [&] (const int& ilev) { const auto& sb0 = sphere_buf(0, igp, jgp, ilev); const auto& sb1 = sphere_buf(1, igp, jgp, ilev); - curls(0,igp,jgp,ilev) = (D(0,0,igp,jgp) * sb0 + D(1,0,igp,jgp) * sb1) * PhysicalConstants::rrearth; - curls(1,igp,jgp,ilev) = (D(0,1,igp,jgp) * sb0 + D(1,1,igp,jgp) * sb1) * PhysicalConstants::rrearth; + curls(0,igp,jgp,ilev) = (D(0,0,igp,jgp) * sb0 + D(1,0,igp,jgp) * sb1) * m_rrearth; + curls(1,igp,jgp,ilev) = (D(0,1,igp,jgp) * sb0 + D(1,1,igp,jgp) * sb1) * m_rrearth; }); }); kv.team_barrier(); @@ -839,10 +838,10 @@ class SphereOperators } curls(0,ngp,mgp,ilev) = beta*curls(0,ngp,mgp,ilev) + alpha * ( D(0,0,ngp,mgp)*sb0 + D(1,0,ngp,mgp)*sb1 ) - * PhysicalConstants::rrearth; + * m_rrearth; curls(1,ngp,mgp,ilev) = beta*curls(1,ngp,mgp,ilev) + alpha * ( D(0,1,ngp,mgp)*sb0 + D(1,1,ngp,mgp)*sb1 ) - * PhysicalConstants::rrearth; + * m_rrearth; }); }); kv.team_barrier(); @@ -883,8 +882,8 @@ class SphereOperators b1 -= (mpnj * metinv(1,0,ngp,mgp) * md * snj * djm + mpjm * metinv(1,1,ngp,mgp) * md * sjm * djn); } - grads(0,ngp,mgp,ilev) = (D(0,0,ngp,mgp) * b0 + D(1,0,ngp,mgp) * b1) * PhysicalConstants::rrearth; - grads(1,ngp,mgp,ilev) = (D(0,1,ngp,mgp) * b0 + D(1,1,ngp,mgp) * b1) * PhysicalConstants::rrearth; + grads(0,ngp,mgp,ilev) = (D(0,0,ngp,mgp) * b0 + D(1,0,ngp,mgp) * b1) * m_rrearth; + grads(1,ngp,mgp,ilev) = (D(0,1,ngp,mgp) * b0 + D(1,1,ngp,mgp) * b1) * m_rrearth; }); }); kv.team_barrier(); @@ -940,13 +939,13 @@ class SphereOperators + vec_sph2cart(0,1,igp,jgp)*laplace1(igp,jgp,ilev) + vec_sph2cart(0,2,igp,jgp)*laplace2(igp,jgp,ilev) + 2.0*spheremp(igp,jgp)*vector(0,igp,jgp,ilev) - *(PhysicalConstants::rrearth)*(PhysicalConstants::rrearth); + *(m_rrearth)*(m_rrearth); laplace(1,igp,jgp,ilev) = vec_sph2cart(1,0,igp,jgp)*laplace0(igp,jgp,ilev) + vec_sph2cart(1,1,igp,jgp)*laplace1(igp,jgp,ilev) + vec_sph2cart(1,2,igp,jgp)*laplace2(igp,jgp,ilev) + 2.0*spheremp(igp,jgp)*vector(1,igp,jgp,ilev) - *(PhysicalConstants::rrearth)*(PhysicalConstants::rrearth); + *(m_rrearth)*(m_rrearth); #else laplace(0,igp,jgp,ilev) = vec_sph2cart(0,0,igp,jgp)*laplace0(igp,jgp,ilev) + vec_sph2cart(0,1,igp,jgp)*laplace1(igp,jgp,ilev) @@ -998,7 +997,7 @@ class SphereOperators vorticity_sphere(kv,vector,vort); curl_sphere_wk_testcov_update(kv,-1.0,1.0,vort,grad_curl_cov); - const auto re2 = PhysicalConstants::rrearth*PhysicalConstants::rrearth; + const auto re2 = m_rrearth*m_rrearth; Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, np_squared), [&](const int loop_idx) { const int igp = loop_idx / NP; //slow From 1c76f58a322b5d281976868d365e7b3c5ba3f4de Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 18 Nov 2021 17:37:33 -0700 Subject: [PATCH 177/301] Homme: improved Kokkos.cmake script The improvement allows to use this script, avoid building kokkos, and avoid assuming location of the include/lib paths. Before this mods, the script automatically addes some include paths that were not necessarily the correct ones. --- components/homme/cmake/Kokkos.cmake | 57 +++++++++++++++++------------ 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/components/homme/cmake/Kokkos.cmake b/components/homme/cmake/Kokkos.cmake index 0a0ebb9da7cc..b0a32cf1cf4b 100644 --- a/components/homme/cmake/Kokkos.cmake +++ b/components/homme/cmake/Kokkos.cmake @@ -1,38 +1,45 @@ +set (KOKKOS_LIBRARIES "kokkoscore;kokkoscontainers") +set (KOKKOS_TPL_LIBRARIES "dl") + set (KOKKOS_INSTALLATION_NEEDED FALSE CACHE BOOL "") -if (NOT DEFINED E3SM_KOKKOS_PATH) - # Build kokkos submodule if user did not specify KOKKOS_PATH. +if (DEFINED E3SM_KOKKOS_PATH) + message (STATUS "The E3SM installation of kokkos in ${E3SM_KOKKOS_PATH} will be used. Here are the details:") + set (KOKKOS_INCLUDE_DIR ${E3SM_KOKKOS_PATH}/include) + set (KOKKOS_LIBRARY_DIR ${E3SM_KOKKOS_PATH}/lib64) + + message (" KOKKOS_INCLUDE_DIR: ${KOKKOS_INCLUDE_DIR}") + message (" KOKKOS_LIBRARY_DIR: ${KOKKOS_LIBRARY_DIR}") + message (" KOKKOS_LIBRARIES: ${KOKKOS_LIBRARIES}") + set (KOKKOS_LIBS_ARE_TARGETS FALSE) +elseif (E3SM_INTERNAL_KOKKOS_ALREADY_BUILT) + message (STATUS "All kokkos libraries needed by Homme are already built by this cmake project.\n") + set (KOKKOS_LIBS_ARE_TARGETS TRUE) +else() + # As last resort, build kokkos submodule set (KOKKOS_SRC ${CMAKE_SOURCE_DIR}/../../externals/kokkos) set (KOKKOS_BUILD_DIR ${CMAKE_BINARY_DIR}/kokkos/build) # kokkos-containers is likely not needed, but it's built anyways, # so we may as well add its include path. kokkos-algorithms may # be used for the Kokkos_Random.hpp header. - SET (KOKKOS_INCLUDE_DIR + set (KOKKOS_INCLUDE_DIR ${KOKKOS_SRC}/core/src ${KOKKOS_SRC}/containers/src ${KOKKOS_SRC}/algorithms/src ${KOKKOS_BUILD_DIR}) - SET (KOKKOS_LIBRARY_DIR ${KOKKOS_BUILD_DIR}) - message ("Using Kokkos built internally to E3SM, in ${KOKKOS_BUILD_DIR}") - if (NOT E3SM_INTERNAL_KOKKOS_ALREADY_BUILT) - # Nobody else already added kokkos subdirectory, so we will do it - set (KOKKOS_INSTALLATION_NEEDED TRUE) - set (E3SM_INTERNAL_KOKKOS_ALREADY_BUILT TRUE) - endif() + set (KOKKOS_LIBRARY_DIR ${KOKKOS_BUILD_DIR}) + message ("Building Kokkos (from E3SM internal submodule) in ${KOKKOS_BUILD_DIR}") -else () - message ("The installation of kokkos in ${E3SM_KOKKOS_PATH} will be used.") - SET (KOKKOS_INCLUDE_DIR ${E3SM_KOKKOS_PATH}/include) - SET (KOKKOS_LIBRARY_DIR ${E3SM_KOKKOS_PATH}/lib64) -endif () + # Make a note that we need to add the kokkos subdir (in the cmake sense) + set (KOKKOS_INSTALLATION_NEEDED TRUE) -SET (KOKKOS_LIBRARIES "kokkoscore;kokkoscontainers") -SET (KOKKOS_TPL_LIBRARIES "dl") + # Notify that E3SM's internal kokkos is built + set (E3SM_INTERNAL_KOKKOS_ALREADY_BUILT TRUE) -MESSAGE (STATUS "Kokkos installation found! Here are the details:") -MESSAGE (" KOKKOS_INCLUDE_DIR: ${KOKKOS_INCLUDE_DIR}") -MESSAGE (" KOKKOS_LIBRARY_DIR: ${KOKKOS_LIBRARY_DIR}") -MESSAGE (" KOKKOS_LIBRARIES: ${KOKKOS_LIBRARIES}") + # Make a note that, since kokkos is built internally, it's a valid cmake target + # (in particular, we don't need to set compile/include/link flags) + set (KOKKOS_LIBS_ARE_TARGETS TRUE) +endif () macro(install_kokkos_if_needed) if (KOKKOS_INSTALLATION_NEEDED) @@ -44,6 +51,10 @@ macro(install_kokkos_if_needed) endmacro() macro(link_to_kokkos targetName) - TARGET_INCLUDE_DIRECTORIES(${targetName} SYSTEM PUBLIC ${KOKKOS_INCLUDE_DIR}) - TARGET_LINK_LIBRARIES(${targetName} ${KOKKOS_TPL_LIBRARIES} ${KOKKOS_LIBRARIES} -L${KOKKOS_LIBRARY_DIR}) + if (KOKKOS_LIBS_ARE_TARGETS) + target_link_libraries (${targetName} ${KOKKOS_LIBRARIES}) + else() + target_include_directories(${targetName} SYSTEM PUBLIC ${KOKKOS_INCLUDE_DIR}) + target_link_libraries(${targetName} ${KOKKOS_TPL_LIBRARIES} ${KOKKOS_LIBRARIES} -L${KOKKOS_LIBRARY_DIR}) + endif() endmacro(link_to_kokkos) From d87b55e19a35dd612d2874a3c0df9f4dbc3dcaa9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 30 Nov 2021 10:51:03 -0700 Subject: [PATCH 178/301] Homme: in theta-l_kokkos, compile forcing stuff only in standalone mode --- components/homme/src/theta-l_kokkos/prim_driver_mod.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 0e1584f1ab4f..ce6bc90dba9e 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -422,14 +422,16 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) +#if !defined(CAM) && !defined(SCREAM) + ! Test forcing is only for standalone Homme (and only for some tests/configurations) if (compute_forcing_and_push_to_c) then -! if ( .true. ) then call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) call t_startf('push_to_cxx') call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & elem_derived_FPHI, elem_derived_FQ) call t_stopf('push_to_cxx') endif +#endif call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep) @@ -442,8 +444,6 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv call init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics) if (push_to_f) then -! if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then -! if (.true.) then ! Set pointers to states elem_state_v_ptr = c_loc(elem_state_v) elem_state_w_i_ptr = c_loc(elem_state_w_i) From f88fa0ab23916292533af30b563589b5294bc282 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 30 Nov 2021 17:23:25 -0600 Subject: [PATCH 179/301] changes to more rearth out of physicsconstants: move rearth to simulationParameters, which will add it to geometry, and then to sphereOperators. had to touch both preqxx and thetaxx. change for Compose is temporary, not sure about proper change. preqxx and theta tests (incl. dcmip2012 test 2.1) are bfb. unit tests (builds) are broken. --- .../preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp | 1 + .../preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp | 8 +++++--- .../src/preqx_kokkos/cxx/prim_advance_exp.cpp | 14 ++++++-------- .../homme/src/preqx_kokkos/preqx_f2c_mod.F90 | 4 ++-- .../homme/src/preqx_kokkos/prim_driver_mod.F90 | 3 ++- .../share/cxx/ComposeTransportImplTrajectory.cpp | 6 +++++- components/homme/src/share/cxx/Elements.cpp | 3 ++- components/homme/src/share/cxx/Elements.hpp | 1 + .../homme/src/share/cxx/ElementsGeometry.hpp | 3 ++- .../homme/src/share/cxx/PhysicalConstants.hpp | 5 +++-- .../homme/src/share/cxx/SimulationParams.hpp | 2 ++ components/homme/src/share/cxx/SphereOperators.hpp | 3 +++ .../theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 5 ++++- .../homme/src/theta-l_kokkos/prim_driver_mod.F90 | 3 ++- .../homme/src/theta-l_kokkos/theta_f2c_mod.F90 | 5 +++-- .../run_tests/theta-fdc12-test21-kokkos.cmake | 3 +-- .../reg_test/run_tests/theta-fdc12-test21.cmake | 3 +-- 17 files changed, 45 insertions(+), 27 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp b/components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp index 75c6dfd6b7d3..688007e27f24 100644 --- a/components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp +++ b/components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp @@ -11,6 +11,7 @@ #include "SimulationParams.hpp" #include "KernelVariables.hpp" #include "SphereOperators.hpp" +#include "PhysicalConstants.hpp" #include diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index 518631569006..f300cf33c5ca 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -39,7 +39,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling, const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, - const int& dt_remap_factor, const int& dt_tracer_factor) + const int& dt_remap_factor, const int& dt_tracer_factor, + const double& rearth) { // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more @@ -68,7 +69,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.remap_alg = RemapAlg::PPM_FIXED_MEANS; } if (time_step_type==5) { - params.time_step_type = TimeStepType::ULLRICH_RK35;; + params.time_step_type = TimeStepType::ttype5; } params.limiter_option = limiter_option; @@ -92,6 +93,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; + params.rearth = rearth; //set nu_ratios values if (params.nu != params.nu_div) { @@ -236,7 +238,7 @@ void init_elements_c (const int& num_elems) const SimulationParams& params = c.get(); const bool consthv = (params.hypervis_scaling==0.0); - e.init (num_elems, consthv, /* alloc_gradphis = */ false); + e.init (num_elems, consthv, /* alloc_gradphis = */ false, params.rearth); // Init also the tracers structure Tracers& t = c.create (); diff --git a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp index 158d87f749c6..3e221fc049a0 100644 --- a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp @@ -51,7 +51,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost // Perform time-advance switch (params.time_step_type) { - case TimeStepType::ULLRICH_RK35: + case TimeStepType::ttype5: // Perform RK stages u3_5stage_timestep(tl, dt, eta_ave_w); break; @@ -71,13 +71,11 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost } #endif - if (!is_implicit(params.time_step_type)) { - // Get and run the HVF - HyperviscosityFunctor& functor = Context::singleton().get(); - GPTLstart("tl-ae advance_hypervis_dp"); - functor.run(tl.np1,dt,eta_ave_w); - GPTLstop("tl-ae advance_hypervis_dp"); - } + // Get and run the HVF + HyperviscosityFunctor& functor = Context::singleton().get(); + GPTLstart("tl-ae advance_hypervis_dp"); + functor.run(tl.np1,dt,eta_ave_w); + GPTLstop("tl-ae advance_hypervis_dp"); #ifdef ENERGY_DIAGNOSTICS if (compute_diagnostics) { diff --git a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 index d40e37c0921a..0b8ad51bd21c 100644 --- a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 +++ b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 @@ -12,14 +12,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, hypervis_order, hypervis_subcycle, hypervis_scaling, & ftype, prescribed_wind, moisture, & disable_diagnostics, use_cpstar, transport_alg, & - dt_remap_factor, dt_tracer_factor) bind(c) + dt_remap_factor, dt_tracer_factor, rearth) bind(c) use iso_c_binding, only: c_int, c_bool, c_double, c_ptr ! ! Inputs ! integer(kind=c_int), intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type integer(kind=c_int), intent(in) :: state_frequency, qsize, dt_remap_factor, dt_tracer_factor - real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling + real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, rearth integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, ftype, transport_alg logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar end subroutine init_simulation_params_c diff --git a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 index b7d33e707b63..ad31e0ec1523 100644 --- a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 @@ -8,6 +8,7 @@ module prim_driver_mod use element_mod, only : element_t use prim_driver_base, only : deriv1, smooth_topo_datasets use prim_cxx_driver_base, only : prim_init1, prim_finalize + use physical_constants, only : rearth implicit none @@ -88,7 +89,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) LOGICAL(disable_diagnostics,c_bool), & LOGICAL(use_cpstar==1,c_bool), & transport_alg, & - dt_remap_factor, dt_tracer_factor) + dt_remap_factor, dt_tracer_factor, rearth) ! Initialize time level structure in C++ call init_time_level_c(tl%nm1, tl%n0, tl%np1, tl%nstep, tl%nstep0) diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index 2822d3a6ef43..ff020252a7e6 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -435,7 +435,11 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const int num_phys_lev = this->num_phys_lev; const auto m_sphere_cart = geo.m_sphere_cart; //todo get scale_factor into PhysicalConstants - const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth; + +//???? + //const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth; + const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth0; + const auto m_dep_pts = m_data.dep_pts; const auto calc_departure_point = KOKKOS_LAMBDA (const MT& team) { KernelVariables kv(team, tu_ne); diff --git a/components/homme/src/share/cxx/Elements.cpp b/components/homme/src/share/cxx/Elements.cpp index 1a96cdc32469..904671bbaee3 100644 --- a/components/homme/src/share/cxx/Elements.cpp +++ b/components/homme/src/share/cxx/Elements.cpp @@ -13,13 +13,14 @@ namespace Homme { void Elements::init(const int num_elems, const bool consthv, const bool alloc_gradphis, + const Real rearth, const bool alloc_sphere_coords) { // Sanity check assert (num_elems>0); m_num_elems = num_elems; - m_geometry.init(num_elems,consthv,alloc_gradphis,alloc_sphere_coords); + m_geometry.init(num_elems,consthv,alloc_gradphis,rearth,alloc_sphere_coords); m_state.init(num_elems); m_derived.init(num_elems); m_forcing.init(num_elems); diff --git a/components/homme/src/share/cxx/Elements.hpp b/components/homme/src/share/cxx/Elements.hpp index b66395898032..b5520c0fe38d 100644 --- a/components/homme/src/share/cxx/Elements.hpp +++ b/components/homme/src/share/cxx/Elements.hpp @@ -42,6 +42,7 @@ class Elements { int num_elems () const { return m_num_elems; } void init (const int num_elems, const bool consthv, const bool alloc_gradphis, + const Real rearth, const bool alloc_sphere_coords=false); void randomize (const int seed, const Real max_pressure = 1.0); void randomize (const int seed, const Real max_pressure, const Real ps0, const Real hyai0); diff --git a/components/homme/src/share/cxx/ElementsGeometry.hpp b/components/homme/src/share/cxx/ElementsGeometry.hpp index 22a3420459c1..4832b2ec2b7b 100644 --- a/components/homme/src/share/cxx/ElementsGeometry.hpp +++ b/components/homme/src/share/cxx/ElementsGeometry.hpp @@ -48,6 +48,8 @@ class ElementsGeometry { ElementsGeometry() : m_num_elems(0) {} + Real m_rearth; + void init (const int num_elems, const bool consthv, const bool alloc_gradphis, const Real rearth, const bool alloc_sphere_coords=false); @@ -71,7 +73,6 @@ class ElementsGeometry { private: bool m_consthv; int m_num_elems; - Real m_rearth; }; } // Homme diff --git a/components/homme/src/share/cxx/PhysicalConstants.hpp b/components/homme/src/share/cxx/PhysicalConstants.hpp index b7c4cc3feaa9..9965190cc32a 100644 --- a/components/homme/src/share/cxx/PhysicalConstants.hpp +++ b/components/homme/src/share/cxx/PhysicalConstants.hpp @@ -19,8 +19,9 @@ namespace PhysicalConstants constexpr Real Rgas = 287.04; constexpr Real cp = 1005.0; constexpr Real kappa = Rgas / cp; -// constexpr Real rearth = 6.376e6; -// constexpr Real rrearth = 1.0 / rearth; +// real Earth + constexpr Real rearth0 = 6.376e6; + constexpr Real rrearth0 = 1.0 / rearth0; constexpr Real g = 9.80616; constexpr Real p0 = 100000; // [mbar] diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index 6c0d781682a1..141b7a8c78e0 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -56,6 +56,7 @@ struct SimulationParams int hypervis_subcycle_tom; double hypervis_scaling; double nu_ratio1, nu_ratio2; //control balance between div and vort components in vector laplace + double rearth; //propagated then to Geometry and SphereOps // Use this member to check whether the struct has been initialized bool params_set; @@ -93,6 +94,7 @@ inline void SimulationParams::print () { printf (" disable_diagnostics: %s\n", (disable_diagnostics ? "yes" : "no")); printf (" theta_hydrostatic_mode: %s\n", (theta_hydrostatic_mode ? "yes" : "no")); printf (" prescribed_wind: %s\n", (prescribed_wind ? "yes" : "no")); + printf (" rearth: %f\n", rearth); printf ("\n**********************************************************\n"); } diff --git a/components/homme/src/share/cxx/SphereOperators.hpp b/components/homme/src/share/cxx/SphereOperators.hpp index a5c9dd91bfa4..30101e2ae961 100644 --- a/components/homme/src/share/cxx/SphereOperators.hpp +++ b/components/homme/src/share/cxx/SphereOperators.hpp @@ -1032,6 +1032,9 @@ class SphereOperators ExecViewManaged m_metdet; ExecViewManaged m_d; ExecViewManaged m_dinv; + + Real m_rearth; + Real m_rrearth; }; } // namespace Homme diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 545f81d20095..90c8aad73d12 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -43,7 +43,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const double& hypervis_scaling, const double& dcmip16_mu, const int& ftype, const int& theta_adv_form, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const bool& theta_hydrostatic_mode, const char** test_case, - const int& dt_remap_factor, const int& dt_tracer_factor) + const int& dt_remap_factor, const int& dt_tracer_factor, + const double& rearth) { // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more @@ -105,6 +106,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.transport_alg = transport_alg; params.theta_hydrostatic_mode = theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; + params.rearth = rearth; if (time_step_type==5) { //5 stage, 3rd order, explicit @@ -261,6 +263,7 @@ void init_elements_c (const int& num_elems) const bool consthv = (params.hypervis_scaling==0.0); e.init (num_elems, consthv, /* alloc_gradphis = */ true, + params.rearth, /* alloc_sphere_coords = */ params.transport_alg > 0); // Init also the tracers structure diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 0e1584f1ab4f..949647ba4976 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -10,6 +10,7 @@ module prim_driver_mod use element_mod, only : element_t use prim_driver_base, only : deriv1, smooth_topo_datasets use prim_cxx_driver_base, only : prim_init1, prim_finalize + use physical_constants, only : rearth implicit none @@ -107,7 +108,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) transport_alg, & LOGICAL(theta_hydrostatic_mode,c_bool), & c_loc(test_name), & - dt_remap_factor, dt_tracer_factor) + dt_remap_factor, dt_tracer_factor, rearth) ! Initialize time level structure in C++ call init_time_level_c(tl%nm1, tl%n0, tl%np1, tl%nstep, tl%nstep0) diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 85b61befab38..daa81c166486 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -14,7 +14,7 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & - dt_tracer_factor) bind(c) + dt_tracer_factor, rearth) bind(c) use iso_c_binding, only: c_int, c_bool, c_double, c_ptr ! ! Inputs @@ -22,7 +22,8 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, integer(kind=c_int), intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type integer(kind=c_int), intent(in) :: dt_remap_factor, dt_tracer_factor, transport_alg integer(kind=c_int), intent(in) :: state_frequency, qsize - real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, dcmip16_mu + real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, dcmip16_mu, & + rearth integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake index c4339e18f7e4..d2323e7b1ad4 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake @@ -14,8 +14,7 @@ SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test2_11.nc - dcmip2012_test2_12.nc) + dcmip2012_test2_11.nc) #DO NOT MOD diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake index cefe6ae067dd..5ea1163d83fb 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake @@ -14,8 +14,7 @@ SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: SET(NC_OUTPUT_FILES - dcmip2012_test2_11.nc - dcmip2012_test2_12.nc) + dcmip2012_test2_11.nc) #DO NOT MOD From eef566404c2ddcc842c9d70f7f17cc2fe85b91cb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 30 Nov 2021 20:07:58 -0600 Subject: [PATCH 180/301] fix ut tests, compose. all is bfb, ut tests pass --- .../homme/src/share/cxx/ComposeTransportImplTrajectory.cpp | 4 +--- components/homme/src/share/cxx/SphereOperators.hpp | 6 ++++++ .../homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp | 4 +++- .../homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp | 4 +++- .../homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp | 3 ++- .../homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp | 3 ++- components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp | 3 ++- components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp | 2 ++ components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp | 5 +++-- components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp | 3 ++- components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp | 3 ++- .../homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp | 3 ++- 12 files changed, 30 insertions(+), 13 deletions(-) diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index ff020252a7e6..d463d3bf4313 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -436,9 +436,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const auto m_sphere_cart = geo.m_sphere_cart; //todo get scale_factor into PhysicalConstants -//???? - //const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth; - const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth0; + const auto scale_factor = m_data.geometry_type == 1 ? 1 : geo.m_rearth; const auto m_dep_pts = m_data.dep_pts; const auto calc_departure_point = KOKKOS_LAMBDA (const MT& team) { diff --git a/components/homme/src/share/cxx/SphereOperators.hpp b/components/homme/src/share/cxx/SphereOperators.hpp index 30101e2ae961..05d0516636f2 100644 --- a/components/homme/src/share/cxx/SphereOperators.hpp +++ b/components/homme/src/share/cxx/SphereOperators.hpp @@ -61,6 +61,12 @@ class SphereOperators setup (geometry, ref_FE); } + //only for unit tests + SphereOperators (const Real rearth) + { + m_rearth = rearth; + m_rrearth = 1./rearth; + } void setup (const ElementsGeometry& geometry, const ReferenceElement& ref_FE) { diff --git a/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp index abfc45abcb01..cd54b00b0159 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/random_init_ut.cpp @@ -2,7 +2,9 @@ #include "ElementsState.hpp" #include "ElementsGeometry.hpp" +#include "PhysicalConstants.hpp" #include "Types.hpp" + #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" @@ -60,7 +62,7 @@ TEST_CASE("d_dinv_check", "Testing Elements::random_init") { std::cout << "seed: " << seed << (catchRngSeed==0 ? " (catch rng seed was 0)\n" : "\n"); ElementsGeometry geometry; - geometry.init(num_elems, false, /*alloc_gradphis = */ false); + geometry.init(num_elems, false, /*alloc_gradphis = */ false, PhysicalConstants::rearth0); geometry.randomize(seed); HostViewManaged d("host d", num_elems); diff --git a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp index 622b55f78577..142231fbabb4 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp @@ -4,6 +4,8 @@ #include "RemapFunctor.hpp" #include "PpmRemap.hpp" +#include "PhysicalConstants.hpp" + #include "utilities/SubviewUtils.hpp" #include "utilities/TestUtils.hpp" @@ -23,7 +25,7 @@ TEST_CASE("remap_interface", "vertical remap") { std::cout << "seed: " << seed << (catchRngSeed==0 ? " (catch rng seed was 0)\n" : "\n"); Elements elements; - elements.init(num_elems,seed, /*alloc_gradphis = */ false); + elements.init(num_elems,seed, /*alloc_gradphis = */ false, PhysicalConstants::rearth0); elements.randomize(seed); Tracers tracers; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp index 99165ef37aba..e5b4b2039cac 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp @@ -6,6 +6,7 @@ #include "KernelVariables.hpp" #include "SphereOperators.hpp" #include "Types.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" @@ -305,7 +306,7 @@ class compute_sphere_operator_test_ml { ExecViewManaged::HostMirror vector_output_host; - SphereOperators sphere_ops; + SphereOperators sphere_ops{PhysicalConstants::rearth0}; Real nu_ratio; Real alpha; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp index 4ab6755d4687..4860d7a985a4 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp @@ -6,6 +6,7 @@ #include "KernelVariables.hpp" #include "SphereOperators.hpp" #include "Types.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" @@ -162,7 +163,7 @@ class compute_sphere_operator_test { ExecViewManaged::HostMirror vector_output_host; - SphereOperators sphere_ops; + SphereOperators sphere_ops{PhysicalConstants::rearth0}; // tag for laplace_simple() struct TagSimpleLaplace {}; diff --git a/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp index 944c0feb4297..2611e5d7e3a9 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/caar_ut.cpp @@ -7,6 +7,7 @@ #include "CaarFunctorImpl.hpp" #include "SimulationParams.hpp" #include "Tracers.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SyncUtils.hpp" @@ -110,7 +111,7 @@ TEST_CASE("caar", "caar_testing") { const int num_elems = c.get().get_num_local_elements(); auto& elems = c.create(); - elems.init(num_elems,false,true); + elems.init(num_elems,false,true,PhysicalConstants::rearth0); const auto max_pressure = 1000.0 + hvcoord.ps0; // This ensures max_p > ps0 auto& geo = elems.m_geometry; elems.m_geometry.randomize(seed); // Only needed for phis and gradphis diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 2a003187cae0..1094ba4ed28e 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -20,6 +20,7 @@ #include "profiling.hpp" #include "ErrorDefs.hpp" #include "VerticalRemapManager.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SyncUtils.hpp" @@ -154,6 +155,7 @@ struct Session { init_geometry_f90(); auto& geo = c.get(); + geo.m_rearth = PhysicalConstants::rearth0; auto& sphop = c.create(); sphop.setup(geo, ref_FE); diff --git a/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp index 95c499e831de..5209eda0898e 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/dirk_ut.cpp @@ -10,6 +10,7 @@ #include "mpi/Connectivity.hpp" #include "SimulationParams.hpp" #include "Elements.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SyncUtils.hpp" @@ -628,7 +629,7 @@ static void init_elems (int, int nelemd, Random& r, const HybridVCoord& hvcoord, const int nlev = NUM_PHYSICAL_LEV, np = NP; const auto all = Kokkos::ALL(); - e.init(nelemd, false, true); + e.init(nelemd, false, true, PhysicalConstants::rearth0); const auto max_pressure = 1000 + hvcoord.ps0; auto& geo = e.m_geometry; e.m_geometry.randomize(r.gen_seed()); @@ -849,7 +850,7 @@ TEST_CASE ("dirk_toplevel_testing") { c2f(e); compute_stage_value_dirk_f90(nm1+1, alphadtwt_nm1*dt2, n0+1, alphadtwt_n0*dt2, np1+1, dt2); Elements ef90; - ef90.init(nelemd, false, true); + ef90.init(nelemd, false, true, PhysicalConstants::rearth0); f2c(ef90); const auto phif = cmvdc(ef90.m_state.m_phinh_i); diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index d57f61dd9482..0df38355edce 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -9,6 +9,7 @@ #include "ElementsForcing.hpp" #include "HybridVCoord.hpp" #include "ForcingFunctor.hpp" +#include "PhysicalConstants.hpp" #include "SimulationParams.hpp" #include "Tracers.hpp" #include "Types.hpp" @@ -64,7 +65,7 @@ TEST_CASE("forcing", "forcing") { hv.random_init(seed); auto& geo = c.create(); - geo.init(num_elems,true, /* alloc_gradphis = */ true); + geo.init(num_elems,true, /* alloc_gradphis = */ true, PhysicalConstants::rearth0); geo.randomize(seed); auto& state = c.create(); diff --git a/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp index 95fc68f33f37..25a43fb7c47d 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/hv_ut.cpp @@ -14,6 +14,7 @@ #include "SphereOperators.hpp" #include "mpi/MpiBuffersManager.hpp" #include "mpi/Connectivity.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SyncUtils.hpp" @@ -195,7 +196,7 @@ TEST_CASE("hvf", "biharmonic") { const int num_elems = c.get().get_num_local_elements(); auto& geo = c.create(); - geo.init(num_elems,false,true); + geo.init(num_elems,false,true,PhysicalConstants::rearth0); geo.randomize(seed); auto& state = c.create(); diff --git a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp index 3e2e3ea4dfcc..e78cfb0e0fae 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp @@ -11,6 +11,7 @@ #include "HybridVCoord.hpp" #include "Tracers.hpp" #include "mpi/Connectivity.hpp" +#include "PhysicalConstants.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SyncUtils.hpp" @@ -105,7 +106,7 @@ TEST_CASE("remap", "remap_testing") { // Create and init elements/tracers auto& elems = c.create(); - elems.init(num_elems,false,true); + elems.init(num_elems,false,true,PhysicalConstants::rearth0); const auto max_pressure = 1000.0 + hvcoord.ps0; // This ensures max_p > ps0 auto& geo = elems.m_geometry; elems.m_geometry.randomize(seed); // Only needed for phis and gradphis From b0c33b01f0db90b4c25b461e3a79e8fc955d9fde Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 30 Nov 2021 20:16:41 -0600 Subject: [PATCH 181/301] formatting --- .../homme/src/share/cxx/ComposeTransportImplTrajectory.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp index 30a0ca16e868..47a33ad8776c 100644 --- a/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp +++ b/components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp @@ -435,9 +435,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) { const int num_phys_lev = this->num_phys_lev; const auto m_sphere_cart = geo.m_sphere_cart; //todo get scale_factor into PhysicalConstants - const auto scale_factor = m_data.geometry_type == 1 ? 1 : geo.m_rearth; - const auto m_dep_pts = m_data.dep_pts; const auto calc_departure_point = KOKKOS_LAMBDA (const MT& team) { KernelVariables kv(team, tu_ne); From 69cd15aa8ecf83019da5f21989046484af7ceff3 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 1 Dec 2021 14:58:20 -0600 Subject: [PATCH 182/301] more tests, bfb --- .../reg_test/namelists/theta-fdc12-test21.nl | 49 +++++++++++++++++++ .../reg_test/namelists/theta-fdc12-test22.nl | 49 +++++++++++++++++++ .../reg_test/namelists/theta-fdc12-test3.nl | 44 +++++++++++++++++ .../test/reg_test/run_tests/test-list.cmake | 2 + .../run_tests/theta-fdc12-test21-kokkos.cmake | 2 +- .../run_tests/theta-fdc12-test21.cmake | 2 +- .../run_tests/theta-fdc12-test22-kokkos.cmake | 22 +++++++++ .../run_tests/theta-fdc12-test22.cmake | 22 +++++++++ .../run_tests/theta-fdc12-test3-kokkos.cmake | 22 +++++++++ .../run_tests/theta-fdc12-test3.cmake | 22 +++++++++ 10 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 components/homme/test/reg_test/namelists/theta-fdc12-test21.nl create mode 100644 components/homme/test/reg_test/namelists/theta-fdc12-test22.nl create mode 100644 components/homme/test/reg_test/namelists/theta-fdc12-test3.nl create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test22-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test22.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test3-kokkos.cmake create mode 100644 components/homme/test/reg_test/run_tests/theta-fdc12-test3.cmake diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl new file mode 100644 index 000000000000..78719a20c038 --- /dev/null +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl @@ -0,0 +1,49 @@ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_1" ! test identifier + theta_hydrostatic_mode = .false. + rsplit = 1 + ne = ${HOMME_TEST_NE} + qsize = 0 ! num tracer fields + nmax = 1800 ! 7200s / 0.4s per step = 18000 steps + statefreq = 360 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.4 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 10 ! 1 => default method + nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 + nu_s = 3.2e7 + nu_p = 3.2e7 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 4000.0 ! mountain wavelength (m) + limiter_option = 9 + vert_remap_q_alg = 1 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 1800, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + !num_io_procs = 16 + io_stride = 8 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl new file mode 100644 index 000000000000..b2707a43cd75 --- /dev/null +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl @@ -0,0 +1,49 @@ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_2" ! test identifier + theta_hydrostatic_mode = .false. + rsplit = 1 + ne = ${HOMME_TEST_NE} + qsize = 0 ! num tracer fields + nmax = 1800 ! 7200s / 0.4s per step = 18000 steps + statefreq = 360 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 0.4 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 10 ! 1 => default method + nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 + nu_s = 3.2e7 + nu_p = 3.2e7 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 12752.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) + dcmip2_x_h0 = 250.0 ! mountain height (m) + dcmip2_x_d = 5000.0 ! mountain half-width (m) + dcmip2_x_xi = 4000.0 ! mountain wavelength (m) + limiter_option = 9 + vert_remap_q_alg = 1 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 1800, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + !num_io_procs = 16 + io_stride = 8 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl new file mode 100644 index 000000000000..3f4d4540d9fe --- /dev/null +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl @@ -0,0 +1,44 @@ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test3" ! test identifier + theta_hydrostatic_mode = .false. + rsplit = 1 + ne = ${HOMME_TEST_NE} + qsize = 0 ! num tracer fields + nmax = 1800 ! 7200s / 0.4s per step = 18000 steps + statefreq = 360 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 1.0 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 10 ! 1 => default method + nu = 5.0e8 ! reduced planet hyperviscosity hv/500^3 + nu_p = 5.0e8 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 50969.0 ! reduced planet radius rearth = a/500.0 + omega = 0.0 ! earth angular speed = 0.0 + limiter_option = 9 + vert_remap_q_alg = 1 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 1800, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + !num_io_procs = 16 + io_stride = 8 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index b7ebb3335d32..8dcde6790a1e 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -98,6 +98,8 @@ IF (BUILD_HOMME_THETA_KOKKOS) theta-fhs2 theta-fhs3 theta-fdc12-test21 + theta-fdc12-test22 + theta-fdc12-test3 ) #all tests that will be used for cxx-vs-F bfb testing diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake index d2323e7b1ad4..3b1ddabaa71b 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21-kokkos.cmake @@ -9,7 +9,7 @@ SET(EXEC_NAME theta-nlev128-kokkos) SET(NUM_CPUS 16) -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12.nl) +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test21.nl) SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake index 5ea1163d83fb..6bfc00ada13a 100644 --- a/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test21.cmake @@ -9,7 +9,7 @@ SET(EXEC_NAME theta-nlev128) SET(NUM_CPUS 16) -SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12.nl) +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test21.nl) SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) # compare all of these files against baselines: diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test22-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test22-kokkos.cmake new file mode 100644 index 000000000000..e82d8154e5a2 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test22-kokkos.cmake @@ -0,0 +1,22 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test22-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test22.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dcmip2012_test2_21.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test22.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test22.cmake new file mode 100644 index 000000000000..945083dad418 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test22.cmake @@ -0,0 +1,22 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test22) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test22.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dcmip2012_test2_21.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test3-kokkos.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test3-kokkos.cmake new file mode 100644 index 000000000000..a144ada14d76 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test3-kokkos.cmake @@ -0,0 +1,22 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test3-kokkos) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128-kokkos) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test3.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dcmip2012_test31.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) diff --git a/components/homme/test/reg_test/run_tests/theta-fdc12-test3.cmake b/components/homme/test/reg_test/run_tests/theta-fdc12-test3.cmake new file mode 100644 index 000000000000..c01dc22b6788 --- /dev/null +++ b/components/homme/test/reg_test/run_tests/theta-fdc12-test3.cmake @@ -0,0 +1,22 @@ +# The name of this test (should be the basename of this file) +#example of name theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu +#or theta-form0-ttype5-hvs1-hvst0-r3-q1-nutop0-samenu-kokkos +#adding BB to each var to avoid unwanted substitutions + +SET(TEST_NAME theta-fdc12-test3) +# The specifically compiled executable that this test uses +SET(EXEC_NAME theta-nlev128) + +SET(NUM_CPUS 16) + +SET(NAMELIST_FILES ${HOMME_ROOT}/test/reg_test/namelists/theta-fdc12-test3.nl) +SET(VCOORD_FILES ${HOMME_ROOT}/test/vcoord/sab*-128.ascii) + +# compare all of these files against baselines: +SET(NC_OUTPUT_FILES + dcmip2012_test31.nc) + + +#DO NOT MOD +SET (HOMME_TEST_VCOORD_INT_FILE sabi-128.ascii) +SET (HOMME_TEST_VCOORD_MID_FILE sabm-128.ascii) From 307b0dd2f5b5b72acd0621a2163aa6679cf7b061 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 2 Dec 2021 19:17:01 -0600 Subject: [PATCH 183/301] Update BGC compsets to turn ocn bgc back on, add new ocn bgc init file --- cime_config/allactive/config_compsets.xml | 4 ++-- components/mpas-ocean/cime_config/buildnml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index fece8e1e010a..da99c4ba68c6 100644 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -91,7 +91,7 @@ BGCEXP_CNTL_CNPRDCTC_1850 - 1850_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO_MOSART_SGLC_SWAV_BGC%BCRC + 1850_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC @@ -181,7 +181,7 @@ BGCEXP_CNTL_CNPECACNT_1850 - 1850_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO_MOSART_SGLC_SWAV_BGC%BCRC + 1850_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index ce2e6a66f080..573bda96c61e 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -222,6 +222,11 @@ def buildnml(case, caseroot, compname): if ocn_ic_mode == 'spunup': ic_date = '201001' ic_prefix = 'mpaso.EC30to60E2r2.rstFromG-anvil' + # modify initial condition file if ocean BGC is turned on + if ocn_bgc in ['eco_only', 'eco_and_dms', 'eco_and_macromolecules', 'eco_and_dms_and_macromolecules']: + ic_date = '211030' + ic_prefix = 'ocean.EC30to60E2r2.210210+MARBL_ICfromOMIP+ecoAux+5xFESED' + eco_forcing_file = 'ecoForcingSurfaceMonthly.EC30to60E2r2.20201221.nc' elif ocn_grid == 'WC14to60E2r3': decomp_date = '200714' From 89b5192fc2ca07f0664da0dd0fd99b8467ece109 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 3 Dec 2021 09:10:35 +0100 Subject: [PATCH 184/301] Fix filename interval for 2 analysis member restarts The Eliassen-Palm and time-filters analysis members both had hard-coded restart file intervals of 1 year. This causes restarts with these analysis members to fail in situations where the restart interval does not match the file interval. This merge changes the filename interval to be the same as the output interval to match restart streams in all other analysis members. --- .../mpas-ocean/src/analysis_members/Registry_eliassen_palm.xml | 2 +- .../mpas-ocean/src/analysis_members/Registry_time_filters.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/Registry_eliassen_palm.xml b/components/mpas-ocean/src/analysis_members/Registry_eliassen_palm.xml index da06802173df..ce431778d53d 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_eliassen_palm.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_eliassen_palm.xml @@ -643,7 +643,7 @@ type="input;output" mode="forward;analysis" filename_template="restarts/eliassenPalm_restart.$Y-$M-$D.nc" - filename_interval="01-00-00_00:00:00" + filename_interval="output_interval" input_interval="initial_only" output_interval="stream:restart:output_interval" packages="eliassenPalmAMPKG" diff --git a/components/mpas-ocean/src/analysis_members/Registry_time_filters.xml b/components/mpas-ocean/src/analysis_members/Registry_time_filters.xml index afdab8acee8e..5877f8b56c25 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_time_filters.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_time_filters.xml @@ -103,7 +103,7 @@ Date: Fri, 3 Dec 2021 14:41:05 -0600 Subject: [PATCH 185/301] ttype9 in xx, bfb for dcmip12-test3 --- .../theta-l_kokkos/cxx/prim_advance_exp.cpp | 115 ++++++++++++++++-- 1 file changed, 104 insertions(+), 11 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp index 7aab3b444620..45d0e3537097 100644 --- a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp @@ -20,9 +20,9 @@ namespace Homme { // Declare all the timestepping schemes routines -void ttype5_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void ttype7_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); -void ttype9_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype5_timestep (const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype7_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_ave_w); +void ttype9_imex_timestep (const TimeLevel& tl, const Real dt, const Real eta_ave_w); void ttype10_imex_timestep(const TimeLevel& tl, const Real dt, const Real eta_ave_w); // -------------- IMPLEMENTATIONS -------------- // @@ -79,13 +79,13 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost switch (params.time_step_type) { case TimeStepType::ttype5: - ttype5_timestep (tl, dt, eta_ave_w); + ttype5_timestep (tl, dt, eta_ave_w); break; case TimeStepType::ttype7_imex: - ttype7_imex_timestep (tl, dt, eta_ave_w); + ttype7_imex_timestep (tl, dt, eta_ave_w); break; case TimeStepType::ttype9_imex: - ttype9_imex_timestep (tl, dt, eta_ave_w); + ttype9_imex_timestep (tl, dt, eta_ave_w); break; case TimeStepType::ttype10_imex: ttype10_imex_timestep (tl, dt, eta_ave_w); @@ -211,12 +211,105 @@ void ttype7_imex_timestep(const TimeLevel& /* tl */, assert(false); } -void ttype9_imex_timestep(const TimeLevel& /* tl */, - const Real /* dt */, - const Real /* eta_ave_w */) +//note that ttype9 and ttype10 caqnnot be generalized easily into +//one routine because of the +//summation in expl part in stage 5 in ttype9 +void ttype9_imex_timestep(const TimeLevel& tl, + const Real dt_dyn, + const Real eta_ave_w) { - // TODO - assert(false); + + GPTLstart("ttype9_imex_timestep"); + + // The context + const auto& c = Context::singleton(); + SimulationParams& params = Context::singleton().get(); + + // Get elements, hvcoord, and functors + auto& elements = c.get(); + auto& hvcoord = c.get(); + auto& dirk = c.get(); + auto& caar = c.get(); + + const int nm1 = tl.nm1; + const int n0 = tl.n0; + const int np1 = tl.np1; + const int qn0 = tl.n0_qdp; + + // Stage 1 + Real dt = dt_dyn/5.0; +// if one wants to map F call compute_andor_apply_rhs(n1,n2,n3,qn0,... +// to caar call below, then they need to use timelevels this way +// caar(n2,n3,n1,qn0,... + caar.run(RKStageData(n0, n0, nm1, qn0, dt, eta_ave_w/4.0, 1.0, 0.0, 1.0)); + dirk.run(nm1, 0.0, n0, 0.0, nm1, dt, elements, hvcoord); + + // Stage 2 + dt = dt_dyn/5.0; + caar.run(RKStageData(n0, nm1, np1, qn0, dt, 0.0, 1.0, 0.0, 1.0)); + dirk.run(nm1, 0.0, n0, 0.0, np1, dt, elements, hvcoord); + + // Stage 3 + dt = dt_dyn/3.0; + caar.run(RKStageData(n0, np1, np1, qn0, dt, 0.0, 1.0, 0.0, 1.0)); + dirk.run(nm1, 0.0, n0, 0.0, np1, dt, elements, hvcoord); + + // Stage 4 + dt = 2.0*dt_dyn/3.0; + caar.run(RKStageData(n0, np1, np1, qn0, dt, 0.0, 1.0, 0.0, 1.0)); + dirk.run(nm1, 0.0, n0, 0.0, np1, dt, elements, hvcoord); + + // Stage 5 + dt = 3.0*dt_dyn/4.0; + caar.run(RKStageData(nm1, np1, np1, qn0, dt, 3.0*eta_ave_w/4.0, 1.0, 0.0, 1.0)); + // u(np1) = [u1 + 3dt/4 RHS(u4)] + 1/4 (u1 - u0) + { + const auto v = elements.m_state.m_v; + const auto w = elements.m_state.m_w_i; + const auto vtheta_dp = elements.m_state.m_vtheta_dp; + const auto phinh = elements.m_state.m_phinh_i; + const auto dp3d = elements.m_state.m_dp3d; + const auto hydrostatic_mode = params.theta_hydrostatic_mode; + + Kokkos::parallel_for( + Kokkos::RangePolicy(0, elements.num_elems()*NP*NP*NUM_LEV), + KOKKOS_LAMBDA(const int it) { + const int ie = it / (NP*NP*NUM_LEV); + const int igp = (it / (NP*NUM_LEV)) % NP; + const int jgp = (it / NUM_LEV) % NP; + const int ilev = it % NUM_LEV; + v(ie,np1,0,igp,jgp,ilev) += (v(ie,nm1,0,igp,jgp,ilev)-v(ie,n0,0,igp,jgp,ilev))/4.0; + v(ie,np1,1,igp,jgp,ilev) += (v(ie,nm1,1,igp,jgp,ilev)-v(ie,n0,1,igp,jgp,ilev))/4.0; + vtheta_dp(ie,np1,igp,jgp,ilev) += (vtheta_dp(ie,nm1,igp,jgp,ilev)-vtheta_dp(ie,n0,igp,jgp,ilev))/4.0; + dp3d(ie,np1,igp,jgp,ilev) += (dp3d(ie,nm1,igp,jgp,ilev)-dp3d(ie,n0,igp,jgp,ilev))/4.0; + if (!hydrostatic_mode) { + w(ie,np1,igp,jgp,ilev) += (w(ie,nm1,igp,jgp,ilev)-w(ie,n0,igp,jgp,ilev))/4.0; + phinh(ie,np1,igp,jgp,ilev) += (phinh(ie,nm1,igp,jgp,ilev)-phinh(ie,n0,igp,jgp,ilev))/4.0; + } + }); + if (NUM_LEV_P>NUM_LEV && !hydrostatic_mode) { + const int LAST_INT = NUM_LEV_P-1; + Kokkos::parallel_for( + Kokkos::RangePolicy(0, elements.num_elems()*NP*NP), + KOKKOS_LAMBDA(const int it) { + const int ie = it / (NP*NP); + const int igp = (it / NP) % NP; + const int jgp = it % NP; + w(ie,np1,igp,jgp,LAST_INT) += (w(ie,nm1,igp,jgp,LAST_INT)-w(ie,n0,igp,jgp,LAST_INT))/4.0; + }); + } + //LIMITER TO ADD! + } + ExecSpace::impl_static_fence(); + + Real a1 = 5.0/18.0; + Real a2 = 1.0/36.0; + Real a3 = 8.0/18.0; + dt = dt_dyn; + dirk.run(nm1, a2*dt, n0, a1*dt, np1, a3*dt, elements, hvcoord); + + GPTLstop("ttype9_imex_timestep"); + } From 524443bd43427e4a44b78ac49e706d4ac8ebb909 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Fri, 3 Dec 2021 15:37:30 -0600 Subject: [PATCH 186/301] make bfb for test22 --- .../homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp index 45d0e3537097..7aad30b021d9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/prim_advance_exp.cpp @@ -302,11 +302,10 @@ void ttype9_imex_timestep(const TimeLevel& tl, } ExecSpace::impl_static_fence(); - Real a1 = 5.0/18.0; - Real a2 = 1.0/36.0; - Real a3 = 8.0/18.0; - dt = dt_dyn; - dirk.run(nm1, a2*dt, n0, a1*dt, np1, a3*dt, elements, hvcoord); + Real a1 = 5.0*dt_dyn/18.0; + Real a2 = dt_dyn/36.0; + Real a3 = 8.0*dt_dyn/18.0; + dirk.run(nm1, a2, n0, a1, np1, a3, elements, hvcoord); GPTLstop("ttype9_imex_timestep"); From 38243bf9533e824008671e367c0412babccf645f Mon Sep 17 00:00:00 2001 From: Yilin Fang Date: Fri, 3 Dec 2021 20:36:08 -0800 Subject: [PATCH 187/301] Correction to resolve restart bfb error. --- .../elm/src/biogeophys/PhotosynthesisMod.F90 | 141 ++++++------------ 1 file changed, 46 insertions(+), 95 deletions(-) diff --git a/components/elm/src/biogeophys/PhotosynthesisMod.F90 b/components/elm/src/biogeophys/PhotosynthesisMod.F90 index 8eefad80c135..c2825484e975 100644 --- a/components/elm/src/biogeophys/PhotosynthesisMod.F90 +++ b/components/elm/src/biogeophys/PhotosynthesisMod.F90 @@ -3365,22 +3365,9 @@ subroutine calcstress(p,c,t,x,bsun,bsha,gb_mol,gs_mol_sun,gs_mol_sha,qsatl,qaf, ! cannot invert the matrix, solve for x algebraically assuming no flux exit end if - if (laisun(p)>tol_lai.and.laisha(p)>tol_lai)then - !dx = matmul(A,f) - !cu_error = cublasdgemv(h, 0,nvegwcs , nvegwcs, 1d0, A, nvegwcs, f, 1, 0d0, dx, 1) - call matvec_acc(1,nvegwcs,dx,A,f) - else - !reduces to 3x3 system - !in this case, dx is not always [sun,sha,xyl,root] - !sun and sha flip depending on which is lai==0 - dx(sun)=0._r8 - !dx(sha:root)=matmul(A(sha:root,sha:root),f(sha:root)) - call matvec_acc(sha,root,dx,A,f ) - !NOTE: root-sha +1 = 3 is hardcoded - !!cu_error = cublasdgemv(h, 0,root-sha +1 ,root-sha +1, 1d0, & - !! A(sha:root,sha:root), root-sha +1, f(sha:root), 1, 0d0, dx(sha:root), 1) - - endif + !dx = matmul(A,f) + !cu_error = cublasdgemv(h, 0,nvegwcs , nvegwcs, 1d0, A, nvegwcs, f, 1, 0d0, dx, 1) + call matvec_acc(1,nvegwcs,dx,A,f) if ( maxval(abs(dx)) > 50000._r8) then @@ -3388,28 +3375,13 @@ subroutine calcstress(p,c,t,x,bsun,bsha,gb_mol,gs_mol_sun,gs_mol_sha,qsatl,qaf, end if - if (laisun(p)>tol_lai.and.laisha(p)>tol_lai)then - x=x+dx - elseif (laisha(p)>tol_lai) then - x=x+dx - x(sun)=x(xyl) ! psi_sun = psi_xyl because laisun==0 - else - x(xyl:root)=x(xyl:root)+dx(xyl:root) - x(sun)=x(sun)+dx(sha) ! implementation ugly bit, chose to flip dx(sun) and dx(sha) for laisha==0 case - x(sha)=x(xyl) ! psi_sha = psi_xyl because laisha==0 - - endif - + x=x+dx if ( sqrt(sum(dx*dx)) < toldx) then !step in vegwp small -> exit exit end if - ! this is a catch to force spac gradient to atmosphere - if ( x(xyl) > x(root) ) x(xyl) = x(root) - if ( x(sun) > x(xyl) ) x(sun) = x(xyl) - if ( x(sha) > x(xyl) ) x(sha) = x(xyl) end do @@ -3577,72 +3549,51 @@ subroutine spacA(p,c,x,invA,qflx_sun,qflx_sha,flag, & - tsai(p) * params_inst%kmax(veg_pp%itype(p),xyl) / htop(p) * dfr * (x(root)-x(xyl)-grav1)& - sum(k_soil_root(p,1:nlevsoi)) + if (laisha(p)<=tol_lai) then + A(2,2) = -1._r8 + A(2,3) = 1._r8 + end if + + if (laisun(p)<=tol_lai) then + A(1,1) = -1._r8 + A(1,3) = 1._r8 + end if + + invfactor=1._r8 A=invfactor*A !matrix inversion - if (laisun(p)>tol_lai .and. laisha(p)>tol_lai) then - ! general case - determ=A(4,4)*A(2,2)*A(3,3)*A(1,1) - A(4,4)*A(2,2)*A(3,1)*A(1,3)& + determ=A(4,4)*A(2,2)*A(3,3)*A(1,1) - A(4,4)*A(2,2)*A(3,1)*A(1,3)& - A(4,4)*A(3,2)*A(2,3)*A(1,1) - A(4,3)*A(1,1)*A(2,2)*A(3,4) - if ( abs(determ) <= 1.e-50_r8 ) then - flag = .true. !tells calling function that the matrix is not invertible - return - else - flag = .false. - end if - - leading = 1._r8/determ - - !algebraic inversion of the matrix - invA(1,1)=leading*A(4,4)*A(2,2)*A(3,3) - leading*A(4,4)*A(3,2)*A(2,3) - leading*A(4,3)*A(2,2)*A(3,4) - invA(2,1)=leading*A(2,3)*A(4,4)*A(3,1) - invA(3,1)=-leading*A(4,4)*A(2,2)*A(3,1) - invA(4,1)=leading*A(4,3)*A(2,2)*A(3,1) - invA(1,2)=leading*A(1,3)*A(4,4)*A(3,2) - invA(2,2)=leading*A(4,4)*A(3,3)*A(1,1)-leading*A(4,4)*A(3,1)*A(1,3)-leading*A(4,3)*A(1,1)*A(3,4) - invA(3,2)=-leading*A(1,1)*A(4,4)*A(3,2) - invA(4,2)=leading*A(4,3)*A(1,1)*A(3,2) - invA(1,3)=-leading*A(1,3)*A(2,2)*A(4,4) - invA(2,3)=-leading*A(2,3)*A(1,1)*A(4,4) - invA(3,3)=leading*A(2,2)*A(1,1)*A(4,4) - invA(4,3)=-leading*A(4,3)*A(1,1)*A(2,2) - invA(1,4)=leading*A(1,3)*A(3,4)*A(2,2) - invA(2,4)=leading*A(2,3)*A(3,4)*A(1,1) - invA(3,4)=-leading*A(3,4)*A(1,1)*A(2,2) - invA(4,4)=leading*A(2,2)*A(3,3)*A(1,1)-leading*A(2,2)*A(3,1)*A(1,3)-leading*A(3,2)*A(2,3)*A(1,1) - invA=invfactor*invA !undo inversion scaling + if ( abs(determ) <= 1.e-50_r8 ) then + flag = .true. !tells calling function that the matrix is not invertible + return else - ! if laisun or laisha ==0 invert the corresponding 3x3 matrix - ! if both are zero, this routine is not called - if (laisha(p)<=tol_lai) then - ! shift nonzero matrix values so that both 3x3 cases can be inverted with the same code - A(2,2)=A(1,1) - A(3,2)=A(3,1) - A(2,3)=A(1,3) - endif - determ=A(2,2)*A(3,3)*A(4,4)-A(3,4)*A(2,2)*A(4,3)-A(2,3)*A(3,2)*A(4,4) - if ( abs(determ) <= 1.e-50_r8 ) then - flag = .true. !tells calling function that the matrix is not invertible - return - else - flag = .false. - end if + flag = .false. + end if - !algebraic inversion of the 3x3 matrix stored in A(2:4,2:4) - invA(2,2)=A(3,3)*A(4,4)-A(3,4)*A(4,3) - invA(2,3)=-A(2,3)*A(4,4) - invA(2,4)=A(3,4)*A(2,3) - invA(3,2)=-A(3,2)*A(4,4) - invA(3,3)=A(2,2)*A(4,4) - invA(3,4)=-A(3,4)*A(2,2) - invA(4,2)=A(3,2)*A(4,3) - invA(4,3)=-A(2,2)*A(4,3) - invA(4,4)=A(2,2)*A(3,3)-A(2,3)*A(3,2) - invA=1._r8/determ*invA + leading = 1._r8/determ - endif + !algebraic inversion of the matrix + invA(1,1)=leading*A(4,4)*A(2,2)*A(3,3) - leading*A(4,4)*A(3,2)*A(2,3) - leading*A(4,3)*A(2,2)*A(3,4) + invA(2,1)=leading*A(2,3)*A(4,4)*A(3,1) + invA(3,1)=-leading*A(4,4)*A(2,2)*A(3,1) + invA(4,1)=leading*A(4,3)*A(2,2)*A(3,1) + invA(1,2)=leading*A(1,3)*A(4,4)*A(3,2) + invA(2,2)=leading*A(4,4)*A(3,3)*A(1,1)-leading*A(4,4)*A(3,1)*A(1,3)-leading*A(4,3)*A(1,1)*A(3,4) + invA(3,2)=-leading*A(1,1)*A(4,4)*A(3,2) + invA(4,2)=leading*A(4,3)*A(1,1)*A(3,2) + invA(1,3)=-leading*A(1,3)*A(2,2)*A(4,4) + invA(2,3)=-leading*A(2,3)*A(1,1)*A(4,4) + invA(3,3)=leading*A(2,2)*A(1,1)*A(4,4) + invA(4,3)=-leading*A(4,3)*A(1,1)*A(2,2) + invA(1,4)=leading*A(1,3)*A(3,4)*A(2,2) + invA(2,4)=leading*A(2,3)*A(3,4)*A(1,1) + invA(3,4)=-leading*A(3,4)*A(1,1)*A(2,2) + invA(4,4)=leading*A(2,2)*A(3,3)*A(1,1)-leading*A(2,2)*A(3,1)*A(1,3)-leading*A(3,2)*A(2,3)*A(1,1) + invA=invfactor*invA !undo inversion scaling end associate @@ -3721,12 +3672,12 @@ subroutine spacF(p,c,x,f,qflx_sun,qflx_sha, & !!TODO tsai(p)*params_inst%kmax(veg_pp%itype(p),xyl) / htop(p) * fr * (x(root)-x(xyl)-grav1) if (laisha(p) Date: Mon, 6 Dec 2021 11:59:58 -0800 Subject: [PATCH 188/301] Update date format for new gis20km input file to YYYYMMDD format. --- components/mpas-albany-landice/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 181a21c9510c..612e3ceda113 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -63,7 +63,7 @@ def buildnml(case, caseroot, compname): decomp_date += '20190326' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis20km': - grid_date += '111921' + grid_date += '20211119' grid_prefix += 'gis20km' decomp_date += '150922' decomp_prefix += 'mpasli.graph.info.' From 6719607a7544fdea2eefa20c5a0e807d49a71083 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 6 Dec 2021 18:02:22 -0600 Subject: [PATCH 189/301] delete old nl --- .../test/reg_test/namelists/theta-fdc12.nl | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 components/homme/test/reg_test/namelists/theta-fdc12.nl diff --git a/components/homme/test/reg_test/namelists/theta-fdc12.nl b/components/homme/test/reg_test/namelists/theta-fdc12.nl deleted file mode 100644 index 70519a497b51..000000000000 --- a/components/homme/test/reg_test/namelists/theta-fdc12.nl +++ /dev/null @@ -1,49 +0,0 @@ -&ctl_nl - nthreads = 1 - partmethod = 4 ! mesh parition method: 4 = space filling curve - topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2012_test2_1" ! test identifier - theta_hydrostatic_mode = .false. - rsplit = 1 - ne = ${HOMME_TEST_NE} - qsize = 0 ! num tracer fields - nmax = 1800 ! 7200s / 0.4s per step = 18000 steps - statefreq = 360 ! number of steps between screen dumps - restartfreq = -1 ! don't write restart files if < 0 - runtype = 0 ! 0 => new run - tstep = 0.4 ! largest timestep in seconds - integration = 'explicit' ! explicit time integration - tstep_type = 5 ! 1 => default method - nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 - nu_s = 3.2e7 - nu_p = 3.2e7 - hypervis_order = 2 ! 2 = hyperviscosity - hypervis_subcycle = 1 ! 1 = no hyperviz subcycling - rearth = 12752.0 ! reduced planet radius rearth = a/500.0 - omega = 0.0 ! earth angular speed = 0.0 - dcmip2_x_ueq = 20.0 ! windspeed at equator (m/s) - dcmip2_x_h0 = 250.0 ! mountain height (m) - dcmip2_x_d = 5000.0 ! mountain half-width (m) - dcmip2_x_xi = 4000.0 ! mountain wavelength (m) - limiter_option = 9 - vert_remap_q_alg = 1 -/ -&vert_nl - vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain - vanalytic = 1 ! set vcoords in initialization routine - vtop = 3.2818e-2 ! vertical coordinate at top of atm (z=30km) -/ -&analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file - output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 1800, ! 720 seconds (10+1 outputs) - output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file - interp_type = 0 ! 0=native grid, 1=bilinear - output_type ='netcdf' ! netcdf or pnetcdf - !num_io_procs = 16 - io_stride = 8 -/ -&prof_inparm - profile_outpe_num = 100 - profile_single_file = .true. -/ From def827f02b4355418bc56233a7093f01ea69ade3 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 6 Dec 2021 20:29:09 -0600 Subject: [PATCH 190/301] redo namelists --- .../homme/test/reg_test/namelists/theta-fdc12-test22.nl | 2 +- .../homme/test/reg_test/namelists/theta-fdc12-test3.nl | 2 +- components/homme/test/reg_test/namelists/thetahs1.nl | 2 +- components/homme/test/reg_test/namelists/thetahs3.nl | 2 +- components/homme/test/reg_test/run_tests/test-list.cmake | 8 ++++++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl index b2707a43cd75..2f681b1c7ff1 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl @@ -13,7 +13,7 @@ runtype = 0 ! 0 => new run tstep = 0.4 ! largest timestep in seconds integration = 'explicit' ! explicit time integration - tstep_type = 10 ! 1 => default method + tstep_type = 9 ! 1 => default method nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 nu_s = 3.2e7 nu_p = 3.2e7 diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl index 3f4d4540d9fe..ae6f75441ceb 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl @@ -13,7 +13,7 @@ runtype = 0 ! 0 => new run tstep = 1.0 ! largest timestep in seconds integration = 'explicit' ! explicit time integration - tstep_type = 10 ! 1 => default method + tstep_type = 9 ! 1 => default method nu = 5.0e8 ! reduced planet hyperviscosity hv/500^3 nu_p = 5.0e8 hypervis_order = 2 ! 2 = hyperviscosity diff --git a/components/homme/test/reg_test/namelists/thetahs1.nl b/components/homme/test/reg_test/namelists/thetahs1.nl index f014f8a1dfad..554c503c8006 100644 --- a/components/homme/test/reg_test/namelists/thetahs1.nl +++ b/components/homme/test/reg_test/namelists/thetahs1.nl @@ -32,7 +32,7 @@ vert_remap_q_alg = 1 hypervis_scaling = 3.0 hypervis_order = 2 hypervis_subcycle = 5 -hypervis_subcycle_tom = 6 +hypervis_subcycle_tom = 4 theta_hydrostatic_mode = false theta_advect_form = 1 tstep_type = 10 diff --git a/components/homme/test/reg_test/namelists/thetahs3.nl b/components/homme/test/reg_test/namelists/thetahs3.nl index 958c110270bb..efb4bcf98e1f 100644 --- a/components/homme/test/reg_test/namelists/thetahs3.nl +++ b/components/homme/test/reg_test/namelists/thetahs3.nl @@ -36,7 +36,7 @@ hypervis_subcycle_q = 6 hypervis_subcycle_tom = 6 theta_hydrostatic_mode = false theta_advect_form = 1 -tstep_type = 10 +tstep_type = 9 moisture = 'notdry' / &solver_nl diff --git a/components/homme/test/reg_test/run_tests/test-list.cmake b/components/homme/test/reg_test/run_tests/test-list.cmake index 8dcde6790a1e..6ceb4f18504f 100644 --- a/components/homme/test/reg_test/run_tests/test-list.cmake +++ b/components/homme/test/reg_test/run_tests/test-list.cmake @@ -92,6 +92,14 @@ IF (BUILD_HOMME_THETA_KOKKOS) #manually added, create namelists and cmake files for F and cxx by hand #all have to be named theta-f* to make setup work (this could be fixed) +# theta-fhs1 Held-Suarez, no topo, moist, ftype0, EUL, tensor HV, NH, ttype10, v_alg1 +# theta-fhs2 no topo, moist, ftype0, SL, tensor HV, NH, ttype10, v_agl10 +# theta-fhs3 no topo, moist, ftype2, SL, tensor HV, NH, ttype9 +# theta-fdc12-test21 ttype10 +# theta-fdc12-test22 ttype9 +# theta-fdc12-test3 ttype9 + + SET(TESTS_WITHOUT_AUTO theta-f1-tt10-hvs1-hvst0-r2-qz10-nutopoff-GB-sl theta-fhs1 From a28c3203c2c330457d568e0ca70b69cc29c16a80 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 6 Dec 2021 18:35:27 -0800 Subject: [PATCH 191/301] Update date format for new gis1to10km input file to YYYYMMDD format. --- components/mpas-albany-landice/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 612e3ceda113..58c2e2554e03 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -73,7 +73,7 @@ def buildnml(case, caseroot, compname): decomp_date += '150910' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis1to10km': - grid_date += '082421' + grid_date += '20210824' grid_prefix += 'gis_1to10km_r01' decomp_date += '051920' decomp_prefix += 'mpasli.graph.info.' From 99b39fc702882ee5768ba8cfc8708f2f6a500e30 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 6 Dec 2021 21:37:25 -0600 Subject: [PATCH 192/301] implement AMB suggestions --- .../src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp | 3 +++ .../homme/test/reg_test/namelists/theta-fdc12-test21.nl | 8 ++++---- .../homme/test/reg_test/namelists/theta-fdc12-test22.nl | 8 ++++---- .../homme/test/reg_test/namelists/theta-fdc12-test3.nl | 8 ++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 90c8aad73d12..9535e4545ea5 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -123,6 +123,9 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, //5 stage, based on the 2nd order explicit KGU table //2nd order implicit table params.time_step_type = TimeStepType::ttype10_imex; + } else { + Errors::runtime_abort("Invalid time_step_time" + + std::to_string(time_step_type), Errors::err_not_implemented); } //set nu_ratios values diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl index 78719a20c038..87effe0cb1d1 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl @@ -7,8 +7,8 @@ rsplit = 1 ne = ${HOMME_TEST_NE} qsize = 0 ! num tracer fields - nmax = 1800 ! 7200s / 0.4s per step = 18000 steps - statefreq = 360 ! number of steps between screen dumps + nmax = 180 ! 7200s / 0.4s per step = 18000 steps + statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run tstep = 0.4 ! largest timestep in seconds @@ -36,8 +36,8 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 1800, ! 720 seconds (10+1 outputs) - output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + output_frequency = 180, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo_i' ! variables to write to file interp_type = 0 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf !num_io_procs = 16 diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl index 2f681b1c7ff1..08b3a8013f63 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl @@ -7,8 +7,8 @@ rsplit = 1 ne = ${HOMME_TEST_NE} qsize = 0 ! num tracer fields - nmax = 1800 ! 7200s / 0.4s per step = 18000 steps - statefreq = 360 ! number of steps between screen dumps + nmax = 180 ! 7200s / 0.4s per step = 18000 steps + statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run tstep = 0.4 ! largest timestep in seconds @@ -36,8 +36,8 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 1800, ! 720 seconds (10+1 outputs) - output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + output_frequency = 180, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo_i' ! variables to write to file interp_type = 0 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf !num_io_procs = 16 diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl index ae6f75441ceb..4605e2d4e841 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl @@ -7,8 +7,8 @@ rsplit = 1 ne = ${HOMME_TEST_NE} qsize = 0 ! num tracer fields - nmax = 1800 ! 7200s / 0.4s per step = 18000 steps - statefreq = 360 ! number of steps between screen dumps + nmax = 180 ! 7200s / 0.4s per step = 18000 steps + statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run tstep = 1.0 ! largest timestep in seconds @@ -31,8 +31,8 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 0, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 1800, ! 720 seconds (10+1 outputs) - output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + output_frequency = 180, ! 720 seconds (10+1 outputs) + output_varnames1 ='T','ps','u','v','omega','geo_i' ! variables to write to file interp_type = 0 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf !num_io_procs = 16 From 0442712f784adf1af039bdcaa8cce73e2c8c8e62 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 6 Dec 2021 22:53:15 -0600 Subject: [PATCH 193/301] start --- .../eam/bld/config_files/definition.xml | 2 +- components/eam/bld/configure | 15 +++++++++++- .../eam/cime_config/config_component.xml | 2 +- .../testmods_dirs/eam/hommexx/shell_commands | 2 +- components/eam/src/dynamics/se/dyn_comp.F90 | 2 ++ components/homme/src/share/control_mod.F90 | 2 ++ .../homme/src/share/cxx/prim_driver.cpp | 24 ++++++++++--------- components/homme/src/share/cxx/prim_step.cpp | 8 ++++--- 8 files changed, 39 insertions(+), 18 deletions(-) diff --git a/components/eam/bld/config_files/definition.xml b/components/eam/bld/config_files/definition.xml index fa1cbff53b43..eab43cf8a16c 100644 --- a/components/eam/bld/config_files/definition.xml +++ b/components/eam/bld/config_files/definition.xml @@ -29,7 +29,7 @@ Component interfaces: mct or esmf. Default: mct. Dynamics package: eul, sld, fv, or se. - + SE dynamical core option. diff --git a/components/eam/bld/configure b/components/eam/bld/configure index 16c6e0a58b9d..4dd551982928 100755 --- a/components/eam/bld/configure +++ b/components/eam/bld/configure @@ -785,7 +785,8 @@ if (defined $opts{'dyn'}) { $cfg_ref->set('dyn_target', 'preqx'); } if (substr($opts{'dyn_target'},0,5) eq 'theta') { - # theta_* uses 'theta-l' namelist defaults + # theta-l (f code) that thetaxx (cxx code) use 'theta-l' namelist defaults + # ??????????????? or theta ? $cfg_ref->set('dyn_target', 'theta-l'); } print "SE dycore target options:\n"; @@ -2652,6 +2653,14 @@ sub write_filepath print $fh "$camsrcdir/homme/src/preqx/share\n"; print $fh "$camsrcdir/homme/src/share/compose\n"; } + elsif ($opts{'dyn_target'} eq 'thetaxx') { + print $fh "$camsrcdir/homme/src/share/cxx\n"; + print $fh "$camsrcdir/homme/src/share/cxx/mpi\n"; + print $fh "$camsrcdir/homme/src/theta-l_kokkos\n"; + print $fh "$camsrcdir/homme/src/theta-l_kokkos/cxx\n"; + print $fh "$camsrcdir/homme/src/theta-l/share\n"; + print $fh "$camsrcdir/homme/src/share/compose\n"; + } } # Parallelization utilies @@ -2764,6 +2773,10 @@ sub write_cppdefs print $fh " -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE " } + if ($dyn eq "se" and $opts{"dyn_target"} eq "thetaxx") { + print $fh " -DKOKKOS_TARGET -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE " + } + if ($dyn eq "se" and $opts{"dyn_target"} eq "preqx_kokkos") { print $fh " -DKOKKOS_TARGET -DHOMME_ENABLE_COMPOSE " } diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index bc7b7e097124..8bfbc82205ba 100644 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -28,7 +28,7 @@ char - preqx,preqx_kokkos,preqx_acc,theta-l + preqx,preqx_kokkos,preqx_acc,theta-l,thetaxx theta-l build_component_cam env_build.xml diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands index 7994d4cdeba2..41f894c0b474 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands @@ -1 +1 @@ -./xmlchange CAM_TARGET=preqx_kokkos +./xmlchange CAM_TARGET=thetaxx diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index b34342d45dc3..947fac55b15b 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -12,6 +12,7 @@ Module dyn_comp use time_manager, only: is_first_step use spmd_utils, only : iam, npes_cam => npes use pio, only: file_desc_t + use control_mod, only: nsplit_iteration implicit none private @@ -366,6 +367,7 @@ subroutine dyn_run( dyn_state, rc ) if (.not. use_3dfrc) then do n=1,se_nsplit + nsplit_iteration = n ! forward-in-time RK, with subcycling call t_startf('prim_run_subcycle') call prim_run_subcycle(dyn_state%elem,hybrid,nets,nete,& diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 03bbd31f1fd3..6385716585c2 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -68,6 +68,8 @@ module control_mod ! Default values make qsplit and rsplit control the time steps. integer, public :: dt_remap_factor = -1, dt_tracer_factor = -1 + integer, public :: nsplit_iteration = 0 + integer, public :: LFTfreq=0 ! leapfrog-trapazoidal frequency (shallow water only) ! interspace a lf-trapazoidal step every LFTfreq leapfrogs ! 0 = disabled diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index ead9c0355aec..87589aa0cc56 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -104,21 +104,23 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np if ( ! independent_time_steps) { tl.update_tracers_levels(params.dt_tracer_factor); -/// remove -//#ifndef CAM -// apply_test_forcing (); -//#endif - // Apply forcing. - // In standalone mode, params.ftype == ForcingAlg::FORCING_DEBUG - // Corresponds to ftype == 0 in Fortran - if(params.ftype == ForcingAlg::FORCING_DEBUG) { + // to make it readable, duplicate some code +#if defined(CAM) + if(params.ftype == ForcingAlg::FORCING_2) { + apply_cam_forcing_dynamics(dt_remap); + } +#elif defined(SCREAM) + if(params.ftype == ForcingAlg::FORCING_2) { apply_cam_forcing(dt_remap); } - // Corresponds to ftype == 2 in Fortran - else if(params.ftype == ForcingAlg::FORCING_2) { - apply_cam_forcing_dynamics(dt_remap); +#else + // Corresponds to ftype == 0 or ftype == 2 in Fortran + if(params.ftype == ForcingAlg::FORCING_0 || + params.ftype == ForcingAlg::FORCING_2 ) { + apply_cam_forcing(dt_remap); } +#endif if (compute_diagnostics) { Diagnostics& diags = context.get(); diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index 8cdf8c294713..5e31083b7876 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -128,11 +128,13 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { tl.update_tracers_levels(params.dt_tracer_factor); - const bool forcing_0or2 = (params.ftype == ForcingAlg::FORCING_DEBUG || + const bool forcing_0or2 = (params.ftype == ForcingAlg::FORCING_0 || params.ftype == ForcingAlg::FORCING_2); bool apply_forcing; -#ifdef CAM - apply_forcing = params.ftype == ForcingAlg::FORCING_DEBUG; + + //decide on tracer forcing +#if defined(CAM) || defined(SCREAM) + apply_forcing = params.ftype == ForcingAlg::FORCING_2; #else apply_forcing = forcing_0or2; #endif From 1d0191e9767cad7c4523ef054ffacc0f72707e53 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 6 Dec 2021 22:59:54 -0600 Subject: [PATCH 194/301] add nstep logic --- components/homme/src/share/cxx/prim_driver.cpp | 3 ++- components/homme/src/share/cxx/prim_step.cpp | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index 87589aa0cc56..22db5e690481 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -112,7 +112,8 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np } #elif defined(SCREAM) if(params.ftype == ForcingAlg::FORCING_2) { - apply_cam_forcing(dt_remap); + if(nstep_iteration == 1) apply_cam_forcing_tracers(dt_remap); + apply_cam_forcing_dynamics(dt_remap); } #else // Corresponds to ftype == 0 or ftype == 2 in Fortran diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index 5e31083b7876..5d946cdc3f69 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -133,8 +133,11 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { bool apply_forcing; //decide on tracer forcing -#if defined(CAM) || defined(SCREAM) +#if defined(CAM) apply_forcing = params.ftype == ForcingAlg::FORCING_2; +#elif defined(SCREAM) + apply_forcing = (params.ftype == ForcingAlg::FORCING_2 && + nstep_iteration == 1 ); #else apply_forcing = forcing_0or2; #endif From 588dcea85267870eaec437950878b4780758be27 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 7 Dec 2021 09:20:01 -0700 Subject: [PATCH 195/301] fixing random fails --- components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index d3284a3ef902..709493a673d1 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -142,6 +142,7 @@ struct Session { p.transport_alg = 0; p.moisture = MoistDry::MOIST; p.theta_hydrostatic_mode = false; + p.rearth = PhysicalConstants::rearth0; p.params_set = true; const auto hyai = cmvdc(h.hybrid_ai); From 1b1f762d7f715661d0835e971ebb14204e1c8b19 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 7 Dec 2021 09:39:34 -0700 Subject: [PATCH 196/301] avoid crash when on gpu --- components/homme/test/reg_test/namelists/theta-fdc12-test21.nl | 2 +- components/homme/test/reg_test/namelists/theta-fdc12-test22.nl | 2 +- components/homme/test/reg_test/namelists/theta-fdc12-test3.nl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl index 87effe0cb1d1..dd3055652d28 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test21.nl @@ -6,7 +6,7 @@ theta_hydrostatic_mode = .false. rsplit = 1 ne = ${HOMME_TEST_NE} - qsize = 0 ! num tracer fields + qsize = 1 ! num tracer fields nmax = 180 ! 7200s / 0.4s per step = 18000 steps statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl index 08b3a8013f63..f215d9396c83 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test22.nl @@ -6,7 +6,7 @@ theta_hydrostatic_mode = .false. rsplit = 1 ne = ${HOMME_TEST_NE} - qsize = 0 ! num tracer fields + qsize = 1 ! num tracer fields nmax = 180 ! 7200s / 0.4s per step = 18000 steps statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl index 4605e2d4e841..4a6a1edf527f 100644 --- a/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl +++ b/components/homme/test/reg_test/namelists/theta-fdc12-test3.nl @@ -6,7 +6,7 @@ theta_hydrostatic_mode = .false. rsplit = 1 ne = ${HOMME_TEST_NE} - qsize = 0 ! num tracer fields + qsize = 1 ! num tracer fields nmax = 180 ! 7200s / 0.4s per step = 18000 steps statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 From 5af86a4eea32ba7c2140dad9a7a07e9864bf920c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 7 Dec 2021 12:46:05 -0700 Subject: [PATCH 197/301] fix random fail --- components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp index 9d16e3806fbe..81a85796f2f7 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/compose_ut.cpp @@ -134,6 +134,7 @@ struct Session { p.dt_remap_factor = -1; p.params_set = true; p.theta_hydrostatic_mode = true; + p.rearth = PhysicalConstants::rearth0; const auto hyai = cmvdc(h.hybrid_ai); const auto hybi = cmvdc(h.hybrid_bi); @@ -155,7 +156,7 @@ struct Session { init_geometry_f90(); auto& geo = c.get(); - geo.m_rearth = PhysicalConstants::rearth0; + //geo.m_rearth = PhysicalConstants::rearth0; auto& sphop = c.create(); sphop.setup(geo, ref_FE); From 2af1292a4c33a27e82262cff3d25ba63efb6bb7c Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 7 Dec 2021 15:48:09 -0600 Subject: [PATCH 198/301] more on ftypes --- components/homme/src/share/cxx/prim_driver.cpp | 9 ++++++++- components/homme/src/share/cxx/prim_step.cpp | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index 22db5e690481..2d5e626c739f 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -106,17 +106,24 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np // Apply forcing. // to make it readable, duplicate some code + + //CAM, support only ftype2, _tracers are done in dp_coupling #if defined(CAM) if(params.ftype == ForcingAlg::FORCING_2) { apply_cam_forcing_dynamics(dt_remap); } + + //SCREAM, support only ftype2, nothing is done in dp layer #elif defined(SCREAM) if(params.ftype == ForcingAlg::FORCING_2) { if(nstep_iteration == 1) apply_cam_forcing_tracers(dt_remap); apply_cam_forcing_dynamics(dt_remap); } + + //standalone homme, support ftype0 and ftype2 + //ftype0 = ftype2 if dt_remap>=dt_tracer, but + //ftype0 != ftype2 for dt_remap Date: Tue, 7 Dec 2021 17:54:10 -0600 Subject: [PATCH 199/301] more for nsplit --- .../homme/src/share/cxx/SimulationParams.hpp | 7 ++- .../homme/src/share/cxx/prim_driver.cpp | 22 ++----- components/homme/src/share/cxx/prim_step.cpp | 9 ++- .../cxx/cxx_f90_interface_theta.cpp | 7 ++- .../src/theta-l_kokkos/prim_driver_mod.F90 | 63 ++++++++++++++----- .../src/theta-l_kokkos/theta_f2c_mod.F90 | 8 +-- 6 files changed, 74 insertions(+), 42 deletions(-) diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index 6c0d781682a1..ea21c25a4288 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -18,7 +18,7 @@ namespace Homme */ struct SimulationParams { - SimulationParams() : ftype(ForcingAlg::FORCING_OFF), params_set(false) {} + SimulationParams() : ftype(ForcingAlg::FORCING_OFF), params_set(false), nsplit(0) {} void print(); @@ -56,7 +56,9 @@ struct SimulationParams int hypervis_subcycle_tom; double hypervis_scaling; double nu_ratio1, nu_ratio2; //control balance between div and vort components in vector laplace - + int nsplit; + int nsplit_iteration; + // Use this member to check whether the struct has been initialized bool params_set; }; @@ -93,6 +95,7 @@ inline void SimulationParams::print () { printf (" disable_diagnostics: %s\n", (disable_diagnostics ? "yes" : "no")); printf (" theta_hydrostatic_mode: %s\n", (theta_hydrostatic_mode ? "yes" : "no")); printf (" prescribed_wind: %s\n", (prescribed_wind ? "yes" : "no")); + printf (" nsplit: %d\n", nsplit); printf ("\n**********************************************************\n"); } diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index 2d5e626c739f..065576fef1f8 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -22,7 +22,6 @@ namespace Homme void prim_step (const Real, const bool); void prim_step_flexible (const Real, const bool); void vertical_remap (const Real); -void apply_test_forcing (); void update_q (const int np1_qdp, const int np1); extern "C" { @@ -58,7 +57,8 @@ void initialize_dp3d_from_ps_c () { GPTLstop("tl-sc dp3d-from-ps"); } -void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np1, const int& next_output_step) +void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np1, + const int& next_output_step, const int& nsplit_iteration) { GPTLstart("tl-sc prim_run_subcycle_c"); @@ -66,6 +66,7 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np // Get simulation params SimulationParams& params = context.get(); + params.nsplit_iteration = nsplit_iteration; assert(params.params_set); const bool independent_time_steps = (params.transport_alg > 0 && @@ -116,7 +117,8 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np //SCREAM, support only ftype2, nothing is done in dp layer #elif defined(SCREAM) if(params.ftype == ForcingAlg::FORCING_2) { - if(nstep_iteration == 1) apply_cam_forcing_tracers(dt_remap); + if(params.nsplit_iteration == 1) + apply_cam_forcing_tracers(dt_remap*params.nsplit); apply_cam_forcing_dynamics(dt_remap); } @@ -188,20 +190,6 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np } // extern "C" -void apply_test_forcing () { - // Get simulation params - SimulationParams& params = Context::singleton().get(); - - switch (params.test_case) { - case TestCase::JW_BAROCLINIC: - break; // Do nothing - case TestCase::DCMIP2016_TEST2: - default: - Errors::runtime_abort("Test case not yet available in C++ build.\n", - Errors::err_not_implemented); - } -} - void update_q (const int np1_qdp, const int np1) { auto& context = Context::singleton(); diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index b05cfce9e148..3cb2f0e4b51c 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -19,7 +19,6 @@ namespace Homme void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnostics); void prim_advec_tracers_remap (const Real); void vertical_remap (const Real); -void apply_test_forcing (); static void set_tracer_transport_derived_values ( const SimulationParams& params, const Elements& elements, const TimeLevel& tl) @@ -123,7 +122,6 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { Elements& elements = context.get(); TimeLevel& tl = context.get(); - const auto dt_q = dt*params.dt_tracer_factor; const auto dt_remap = params.dt_remap_factor == 0 ? dt : dt*params.dt_remap_factor; tl.update_tracers_levels(params.dt_tracer_factor); @@ -132,6 +130,9 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { params.ftype == ForcingAlg::FORCING_2); bool apply_forcing; + const auto dt_q = dt * params.dt_tracer_factor; + const auto dt_q_nsplit = dt * params.dt_tracer_factor * params.nsplit; + //decide on tracer forcing //SCREAM, CAM -- support only ftype2 #if defined(CAM) @@ -145,7 +146,11 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { if (apply_forcing) { // Apply tracer forcings over tracer time step. +#ifdef SCREAM + apply_cam_forcing_tracers(dt_q_nsplit); +#else apply_cam_forcing_tracers(dt_q); +#endif } set_tracer_transport_derived_values(params, elements, tl); diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 4750881592ee..1b5fc431b2eb 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -43,7 +43,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const double& hypervis_scaling, const double& dcmip16_mu, const int& ftype, const int& theta_adv_form, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const bool& theta_hydrostatic_mode, const char** test_case, - const int& dt_remap_factor, const int& dt_tracer_factor) + const int& dt_remap_factor, const int& dt_tracer_factor, + const int& nsplit) { // Check that the simulation options are supported. This helps us in the future, since we // are currently 'assuming' some option have/not have certain values. As we support for more @@ -61,6 +62,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, Errors::check_option("init_simulation_params_c","nu",nu,0.0,Errors::ComparisonOp::GT); Errors::check_option("init_simulation_params_c","nu_div",nu_div,0.0,Errors::ComparisonOp::GT); Errors::check_option("init_simulation_params_c","theta_advection_form",theta_adv_form,{0,1}); + Errors::check_option("init_simulation_params_c","nsplit",nsplit,1,Errors::ComparisonOp::GE); // Get the simulation params struct SimulationParams& params = Context::singleton().create(); @@ -105,6 +107,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.transport_alg = transport_alg; params.theta_hydrostatic_mode = theta_hydrostatic_mode; params.dcmip16_mu = dcmip16_mu; + params.nsplit = nsplit; if (time_step_type==0) { params.time_step_type = TimeStepType::LF; } else if (time_step_type==1) { @@ -141,7 +144,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, if (ftype == -1) { params.ftype = ForcingAlg::FORCING_OFF; } else if (ftype == 0) { - params.ftype = ForcingAlg::FORCING_DEBUG; + params.ftype = ForcingAlg::FORCING_0; } else if (ftype == 2) { params.ftype = ForcingAlg::FORCING_2; } diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index ce6bc90dba9e..a39c0ee0c914 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -64,7 +64,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) use iso_c_binding, only : c_loc, c_ptr, c_bool, C_NULL_CHAR use theta_f2c_mod, only : init_reference_element_c, init_simulation_params_c, & init_time_level_c, init_hvcoord_c, init_elements_c - use time_mod, only : TimeLevel_t + use time_mod, only : TimeLevel_t, nsplit use hybvcoord_mod, only : hvcoord_t use control_mod, only : limiter_option, rsplit, qsplit, tstep_type, statefreq, & nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & @@ -107,7 +107,8 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) transport_alg, & LOGICAL(theta_hydrostatic_mode,c_bool), & c_loc(test_name), & - dt_remap_factor, dt_tracer_factor) + dt_remap_factor, dt_tracer_factor, & + nsplit) ! Initialize time level structure in C++ call init_time_level_c(tl%nm1, tl%n0, tl%np1, tl%nstep, tl%nstep0) @@ -346,7 +347,7 @@ subroutine prim_init_kokkos_functors (allocate_buffer) end subroutine prim_init_kokkos_functors - subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hvcoord,nsubstep) + subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hvcoord, nsplit_iteration) use iso_c_binding, only : c_int, c_ptr, c_loc use control_mod, only : qsplit, rsplit, statefreq, disable_diagnostics, & dt_remap_factor, dt_tracer_factor @@ -367,9 +368,6 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv use theta_f2c_mod, only : prim_run_subcycle_c, cxx_push_results_to_f90 #ifndef SCREAM use theta_f2c_mod, only : push_forcing_to_c -#endif -#if !defined(CAM) && !defined(SCREAM) - use test_mod, only : compute_test_forcing #endif ! ! Inputs @@ -382,7 +380,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep logical, intent(in) :: single_column type (TimeLevel_t), intent(inout) :: tl - integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit + integer, intent(in) :: nsplit_iteration ! = 1 .. nsplit ! ! Locals ! @@ -395,6 +393,9 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv integer :: n0_qdp, np1_qdp real(kind=real_kind) :: dt_remap, dt_q + if (nsplit_iteration < 1 .or. nsplit_iteration > nsplit) then + call abortmp ('nsplit_iteration out of range') + endif if (nets/=1 .or. nete/=nelemd) then call abortmp ('We don''t allow to call C routines from a horizontally threaded region') endif @@ -422,18 +423,16 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv call TimeLevel_Qdp(tl, dt_tracer_factor, n0_qdp, np1_qdp) -#if !defined(CAM) && !defined(SCREAM) ! Test forcing is only for standalone Homme (and only for some tests/configurations) if (compute_forcing_and_push_to_c) then - call compute_test_forcing(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) + call compute_test_forcing_dummy(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) call t_startf('push_to_cxx') call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & elem_derived_FPHI, elem_derived_FQ) call t_stopf('push_to_cxx') endif -#endif - call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep) + call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep,nsplit_iteration) ! Set final timelevels from C into Fortran structure tl%nstep = nstep_c @@ -441,7 +440,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv tl%n0 = n0_c + 1 tl%np1 = np1_c + 1 - call init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics) + call init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics,nsplit_iteration) if (push_to_f) then ! Set pointers to states @@ -545,13 +544,13 @@ subroutine init_logic_for_push_to_c() end subroutine init_logic_for_push_to_c - subroutine init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics) + subroutine init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnostics,nsplit_iter) use control_mod, only: test_with_forcing - use time_mod, only : timelevel_t + use time_mod, only : timelevel_t, nsplit type (TimeLevel_t), intent(in) :: tl - integer, intent(in) :: statefreq, nextOutputStep + integer, intent(in) :: statefreq, nextOutputStep, nsplit_iter logical, intent(in) :: compute_diagnostics #ifdef HOMMEXX_BENCHMARK_NOFORCING @@ -564,10 +563,44 @@ subroutine init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnost if (test_with_forcing) then push_to_f = .true. endif + if (nsplit_iter == nsplit) then + push_to_f = .true. + endif #endif end subroutine init_logic_for_push_to_f + subroutine compute_test_forcing_dummy(elem,hybrid,hvcoord,nt,ntQ,dt,nets,nete,tl) + use hybrid_mod, only : hybrid_t + use hybvcoord_mod, only : hvcoord_t + use time_mod, only : timelevel_t + use element_mod, only : element_t +#if !defined(CAM) && !defined(SCREAM) + use test_mod, only : compute_test_forcing +#endif + implicit none + type(element_t), intent(inout) :: elem(:) ! element array + type(hybrid_t), intent(in) :: hybrid ! hybrid parallel structure + type(hvcoord_t), intent(in) :: hvcoord + real(kind=real_kind),intent(in) :: dt + integer, intent(in) :: nets,nete,nt,ntQ + type(TimeLevel_t), intent(in) :: tl + +#if !defined(CAM) && !defined(SCREAM) + call compute_test_forcing(elem,hybrid,hvcoord,nt,ntQ,dt,nets,nete,tl) +#endif + + end subroutine compute_test_forcing_dummy + + + + + + + + + + end module diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 85b61befab38..e934a2a21e3d 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -14,12 +14,12 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & - dt_tracer_factor) bind(c) + dt_tracer_factor, nsplit) bind(c) use iso_c_binding, only: c_int, c_bool, c_double, c_ptr ! ! Inputs ! - integer(kind=c_int), intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type + integer(kind=c_int), intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type, nsplit integer(kind=c_int), intent(in) :: dt_remap_factor, dt_tracer_factor, transport_alg integer(kind=c_int), intent(in) :: state_frequency, qsize real(kind=c_double), intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, dcmip16_mu @@ -166,12 +166,12 @@ subroutine push_forcing_to_c(FM, FVTheta, FT, FPHI, FQ) bind(c) end subroutine push_forcing_to_c ! Run dycore for a full atm timesteps - subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step) bind(c) + subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step,nsplit_iter) bind(c) use iso_c_binding, only: c_int, c_double ! ! Inputs ! - integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step + integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step, nsplit_iter real (kind=c_double), intent(in) :: tstep end subroutine prim_run_subcycle_c From 3a1dcde5af2c77baf6296dc8f355fe84a8caddcd Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Tue, 7 Dec 2021 17:55:01 -0600 Subject: [PATCH 200/301] revert thetaxx name to theta-l_kokkos --- components/eam/bld/config_files/definition.xml | 2 +- components/eam/bld/configure | 6 +++--- components/eam/cime_config/config_component.xml | 2 +- .../testdefs/testmods_dirs/eam/hommexx/shell_commands | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/eam/bld/config_files/definition.xml b/components/eam/bld/config_files/definition.xml index eab43cf8a16c..3026e3aaacc5 100644 --- a/components/eam/bld/config_files/definition.xml +++ b/components/eam/bld/config_files/definition.xml @@ -29,7 +29,7 @@ Component interfaces: mct or esmf. Default: mct. Dynamics package: eul, sld, fv, or se. - + SE dynamical core option. diff --git a/components/eam/bld/configure b/components/eam/bld/configure index 4dd551982928..89a23d255d58 100755 --- a/components/eam/bld/configure +++ b/components/eam/bld/configure @@ -785,7 +785,7 @@ if (defined $opts{'dyn'}) { $cfg_ref->set('dyn_target', 'preqx'); } if (substr($opts{'dyn_target'},0,5) eq 'theta') { - # theta-l (f code) that thetaxx (cxx code) use 'theta-l' namelist defaults + # theta-l (f code) and theta-l_kokkos (cxx code) use 'theta-l' namelist defaults # ??????????????? or theta ? $cfg_ref->set('dyn_target', 'theta-l'); } @@ -2653,7 +2653,7 @@ sub write_filepath print $fh "$camsrcdir/homme/src/preqx/share\n"; print $fh "$camsrcdir/homme/src/share/compose\n"; } - elsif ($opts{'dyn_target'} eq 'thetaxx') { + elsif ($opts{'dyn_target'} eq 'theta-l_kokkos') { print $fh "$camsrcdir/homme/src/share/cxx\n"; print $fh "$camsrcdir/homme/src/share/cxx/mpi\n"; print $fh "$camsrcdir/homme/src/theta-l_kokkos\n"; @@ -2773,7 +2773,7 @@ sub write_cppdefs print $fh " -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE " } - if ($dyn eq "se" and $opts{"dyn_target"} eq "thetaxx") { + if ($dyn eq "se" and $opts{"dyn_target"} eq "theta-l_kokkos") { print $fh " -DKOKKOS_TARGET -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE " } diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 8bfbc82205ba..9112041106d6 100644 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -28,7 +28,7 @@ char - preqx,preqx_kokkos,preqx_acc,theta-l,thetaxx + preqx,preqx_kokkos,preqx_acc,theta-l,theta-l_kokkos theta-l build_component_cam env_build.xml diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands index 41f894c0b474..cb43c8c52f90 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands @@ -1 +1 @@ -./xmlchange CAM_TARGET=thetaxx +./xmlchange CAM_TARGET=theta-l_kokkos From 92332135ef21ee6bfc55462ad7204cfe2b3641e3 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Wed, 8 Dec 2021 09:14:57 -0600 Subject: [PATCH 201/301] Updated seaice spherical test cases for current MPAS mesh spec --- .../shared/mpas_seaice_velocity_solver_pwl.F | 24 +++---- .../testcases/advection/clean_grids.py | 2 +- .../testcases/advection/create_ics.py | 2 +- .../testcases/advection/get_testcase_data.py | 8 +-- .../testcases/advection/run_model.py | 2 +- .../spherical_operators_hist.py | 2 +- .../spherical_operators_map.py | 2 +- .../spherical_operators_scaling.py | 2 +- .../spherical_operators/strain/clean_grids.py | 6 +- .../spherical_operators/strain/create_ic.py | 2 +- .../strain/get_testcase_data.py | 8 +-- .../spherical_operators/strain/run_model.py | 2 +- .../spherical_operators/strain/strain_hist.py | 2 +- .../spherical_operators/strain/strain_map.py | 2 +- .../strain_stress_divergence/create_ic.py | 2 +- .../get_testcase_data.py | 8 +-- .../strain_stress_divergence/run_model.py | 2 +- .../strain_stress_divergence_hist.py | 2 +- .../strain_stress_divergence_map.py | 2 +- .../stress_divergence/create_ic.py | 2 +- .../stress_divergence/examine_mesh.py | 71 +++++++++++++++++++ .../stress_divergence/get_testcase_data.py | 8 +-- .../stress_divergence/run_model.py | 2 +- .../streams.seaice.stress_divergence | 4 ++ .../stress_divergence_hist.py | 2 +- .../stress_divergence_map.py | 2 +- 26 files changed, 124 insertions(+), 49 deletions(-) create mode 100644 components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/examine_mesh.py diff --git a/components/mpas-seaice/src/shared/mpas_seaice_velocity_solver_pwl.F b/components/mpas-seaice/src/shared/mpas_seaice_velocity_solver_pwl.F index 09da732bcdbe..23a2c5a80559 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_velocity_solver_pwl.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_velocity_solver_pwl.F @@ -161,8 +161,8 @@ subroutine seaice_init_velocity_solver_pwl(& do iSubCell = 1, nEdgesOnCell(iCell) iEdge = edgesOnCell(iSubCell,iCell) - iVertexOnCell1 = iSubCell - iVertexOnCell2 = seaice_wrapped_index(iSubCell + 1, nEdgesOnCell(iCell)) + iVertexOnCell1 = seaice_wrapped_index(iSubCell - 1, nEdgesOnCell(iCell)) + iVertexOnCell2 = iSubCell c = dvEdge(iEdge) a = sqrt((xLocal(iVertexOnCell1,iCell) - xPWLCentre)**2 + & @@ -185,8 +185,8 @@ subroutine seaice_init_velocity_solver_pwl(& ! calculate the linear basis on the sub triangle do iSubCell = 1, nEdgesOnCell(iCell) - iVertexOnCell1 = iSubCell - iVertexOnCell2 = seaice_wrapped_index(iSubCell + 1, nEdgesOnCell(iCell)) + iVertexOnCell1 = seaice_wrapped_index(iSubCell - 1, nEdgesOnCell(iCell)) + iVertexOnCell2 = iSubCell ! set up left hand matrix leftMatrix(1,1) = xLocal(iVertexOnCell1,iCell) - xPWLCentre @@ -240,12 +240,12 @@ subroutine seaice_init_velocity_solver_pwl(& subCellGradientU(iBasisVertex,iSubCell) = subBasisGradientU(iSubCell,3) * alphaPWL subCellGradientV(iBasisVertex,iSubCell) = subBasisGradientV(iSubCell,3) * alphaPWL - if (iSubCell == iBasisVertex) then + if (iSubCell == seaice_wrapped_index(iBasisVertex + 1, nEdgesOnCell(iCell))) then subCellGradientU(iBasisVertex,iSubCell) = subCellGradientU(iBasisVertex,iSubCell) + subBasisGradientU(iSubCell,1) subCellGradientV(iBasisVertex,iSubCell) = subCellGradientV(iBasisVertex,iSubCell) + subBasisGradientV(iSubCell,1) - else if (iSubCell == seaice_wrapped_index(iBasisVertex - 1, nEdgesOnCell(iCell))) then + else if (iSubCell == iBasisVertex) then subCellGradientU(iBasisVertex,iSubCell) = subCellGradientU(iBasisVertex,iSubCell) + subBasisGradientU(iSubCell,2) subCellGradientV(iBasisVertex,iSubCell) = subCellGradientV(iBasisVertex,iSubCell) + subBasisGradientV(iSubCell,2) @@ -262,7 +262,7 @@ subroutine seaice_init_velocity_solver_pwl(& do iGradientVertex = 1, nEdgesOnCell(iCell) iSubCell1 = iGradientVertex - iSubCell2 = seaice_wrapped_index(iGradientVertex - 1, nEdgesOnCell(iCell)) + iSubCell2 = seaice_wrapped_index(iGradientVertex + 1, nEdgesOnCell(iCell)) basisGradientU(iBasisVertex,iGradientVertex,iCell) = & 0.5_RKIND * (subCellGradientU(iBasisVertex,iSubCell1) + subCellGradientU(iBasisVertex,iSubCell2)) @@ -282,7 +282,7 @@ subroutine seaice_init_velocity_solver_pwl(& do iSubCell = 1, nEdgesOnCell(iCell) - if (iSubCell == iStressVertex .or. iSubCell == seaice_wrapped_index(iStressVertex - 1, nEdgesOnCell(iCell))) then + if (iSubCell == iStressVertex .or. iSubCell == seaice_wrapped_index(iStressVertex + 1, nEdgesOnCell(iCell))) then basisIntegral = ((alphaPWL + 1) * basisSubArea(iSubCell)) / 3.0_RKIND else basisIntegral = ( alphaPWL * basisSubArea(iSubCell)) / 3.0_RKIND @@ -308,18 +308,18 @@ subroutine seaice_init_velocity_solver_pwl(& do iSubCell = 1, nEdgesOnCell(iCell) ! determine stress subcell type - if (iSubCell == iStressVertex) then + if (iSubCell == seaice_wrapped_index(iStressVertex + 1, nEdgesOnCell(iCell))) then subCellTypeStress = 1 - else if (iSubCell == seaice_wrapped_index(iStressVertex - 1, nEdgesOnCell(iCell))) then + else if (iSubCell == iStressVertex) then subCellTypeStress = 2 else subCellTypeStress = 3 endif ! determine velocity subcell type - if (iSubCell == iVelocityVertex) then + if (iSubCell == seaice_wrapped_index(iVelocityVertex + 1, nEdgesOnCell(iCell))) then subCellTypeVelocity = 1 - else if (iSubCell == seaice_wrapped_index(iVelocityVertex - 1, nEdgesOnCell(iCell))) then + else if (iSubCell == iVelocityVertex) then subCellTypeVelocity = 2 else subCellTypeVelocity = 3 diff --git a/components/mpas-seaice/testing_and_setup/testcases/advection/clean_grids.py b/components/mpas-seaice/testing_and_setup/testcases/advection/clean_grids.py index b0d5ea99290f..30306b8ac0ab 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/advection/clean_grids.py +++ b/components/mpas-seaice/testing_and_setup/testcases/advection/clean_grids.py @@ -8,7 +8,7 @@ for nGrid in nGrids: - filein = Dataset("x1.%i.grid.nc" %(nGrid), "r") + filein = Dataset("grid.%i.nc" %(nGrid), "r") nCells = len(filein.dimensions["nCells"]) nVertices = len(filein.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/advection/create_ics.py b/components/mpas-seaice/testing_and_setup/testcases/advection/create_ics.py index 000f3e0839bb..d205bdca0767 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/advection/create_ics.py +++ b/components/mpas-seaice/testing_and_setup/testcases/advection/create_ics.py @@ -6,7 +6,7 @@ def create_ic_file(res, icType): # grid in - gridFilename = "x1.%s.grid.nc" %(res) + gridFilename = "grid.%s.nc" %(res) gridFile = Dataset(gridFilename, "r") nCells = len(gridFile.dimensions["nCells"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/advection/get_testcase_data.py b/components/mpas-seaice/testing_and_setup/testcases/advection/get_testcase_data.py index 9c514bec9e52..227dd1b31f42 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/advection/get_testcase_data.py +++ b/components/mpas-seaice/testing_and_setup/testcases/advection/get_testcase_data.py @@ -4,10 +4,10 @@ def get_testcase_data(): - filenames = ["x1.10242.grid.nc", - "x1.163842.grid.nc", - "x1.2562.grid.nc", - "x1.40962.grid.nc"] + filenames = ["grid.10242.nc", + "grid.163842.nc", + "grid.2562.nc", + "grid.40962.nc"] dirName = "https://web.lcrc.anl.gov/public/e3sm/mpas_standalonedata/mpas-seaice/testcases/strain_stress_divergence/" diff --git a/components/mpas-seaice/testing_and_setup/testcases/advection/run_model.py b/components/mpas-seaice/testing_and_setup/testcases/advection/run_model.py index 9f5d7044d9ed..55e28ba24cc1 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/advection/run_model.py +++ b/components/mpas-seaice/testing_and_setup/testcases/advection/run_model.py @@ -35,7 +35,7 @@ def run_model(): print(" Gridsize: ", gridSize) os.system("rm grid.nc ic.nc namelist.seaice") - os.system("ln -s x1.%i.grid.nc grid.nc" %(gridSize)) + os.system("ln -s grid.%i.nc grid.nc" %(gridSize)) os.system("ln -s ic_%s_%i.nc ic.nc" %(icType, gridSize)) if (advectionMethod == "IR"): diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_hist.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_hist.py index 14f5530cc895..680b9016fd37 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_hist.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_hist.py @@ -9,7 +9,7 @@ def strain_hist(axis): # grid - fileGrid = Dataset("./strain/x1.40962.grid.nc","r") + fileGrid = Dataset("./strain/grid.40962.nc","r") nCells = len(fileGrid.dimensions["nCells"]) nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_map.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_map.py index 248bcd461c38..c711c219aef9 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_map.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_map.py @@ -111,7 +111,7 @@ def plot_subfigure(axes, fig, nVertices, vertexDegree, cellsOnVertex, xCell, yCe def spherical_operators_map(): # grid - fileGrid = Dataset("./strain/x1.40962.grid.nc","r") + fileGrid = Dataset("./strain/grid.40962.nc","r") nCells = len(fileGrid.dimensions["nCells"]) nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_scaling.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_scaling.py index 75fe0abbc5c7..2bf706203624 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_scaling.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/spherical_operators_scaling.py @@ -589,7 +589,7 @@ def strain_scaling(axes, normType, label, xlabel, xMin, xMax, yMin): for resolution in resolutions: - filenameGrid = "./strain/x1.%i.grid.nc" %(resolution) + filenameGrid = "./strain/grid.%i.nc" %(resolution) filenameIC = "./strain/ic_%i.nc" %(resolution) filename = "./strain/output_%s_%i/output.2000.nc" %(dirname[method],resolution) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/clean_grids.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/clean_grids.py index a6f804789dbe..f209927a94fe 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/clean_grids.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/clean_grids.py @@ -8,7 +8,7 @@ for nGrid in nGrids: - filein = Dataset("x1.%i.grid.nc" %(nGrid), "r") + filein = Dataset("grid.%i.nc" %(nGrid), "r") nCells = len(filein.dimensions["nCells"]) nVertices = len(filein.dimensions["nVertices"]) @@ -24,7 +24,7 @@ filein.close() - fileout = Dataset("grid_in.%i.nc" %(nGrid),"w",format="NETCDF3_CLASSIC") + fileout = Dataset("grid1.%i.nc" %(nGrid),"w",format="NETCDF3_CLASSIC") fileout.ON_A_SPHERE = "YES" fileout.sphere_radius = 1.0 @@ -52,5 +52,5 @@ fileout.close() - cmd = "%s/mesh_tools/mesh_conversion_tools/MpasMeshConverter.x grid_in.%i.nc grid.%i.nc" %(mpas_tools_dir, nGrid,nGrid) + cmd = "%s/mesh_tools/mesh_conversion_tools/MpasMeshConverter.x grid1.%i.nc grid2.%i.nc" %(mpas_tools_dir, nGrid,nGrid) os.system(cmd) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/create_ic.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/create_ic.py index 24c29a2eb5cd..486bdc77ac65 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/create_ic.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/create_ic.py @@ -638,7 +638,7 @@ def create_ic(): print(" Gridsize: ", gridSize) # input - filenameIn = "x1.%i.grid.nc" %(gridSize) + filenameIn = "grid.%i.nc" %(gridSize) fileIn = Dataset(filenameIn,"r") diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/get_testcase_data.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/get_testcase_data.py index 9c514bec9e52..227dd1b31f42 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/get_testcase_data.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/get_testcase_data.py @@ -4,10 +4,10 @@ def get_testcase_data(): - filenames = ["x1.10242.grid.nc", - "x1.163842.grid.nc", - "x1.2562.grid.nc", - "x1.40962.grid.nc"] + filenames = ["grid.10242.nc", + "grid.163842.nc", + "grid.2562.nc", + "grid.40962.nc"] dirName = "https://web.lcrc.anl.gov/public/e3sm/mpas_standalonedata/mpas-seaice/testcases/strain_stress_divergence/" diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/run_model.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/run_model.py index 7eb6a108e50d..3aba089d7f00 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/run_model.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/run_model.py @@ -28,7 +28,7 @@ def run_model(): print(" Gridsize: ", gridSize) os.system("rm grid.nc ic.nc") - os.system("ln -s x1.%i.grid.nc grid.nc" %(gridSize)) + os.system("ln -s grid.%i.nc grid.nc" %(gridSize)) os.system("ln -s ic_%i.nc ic.nc" %(gridSize)) if (operatorMethod == "wachspress"): diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_hist.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_hist.py index 90f47db7547f..7a78fbeef045 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_hist.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_hist.py @@ -9,7 +9,7 @@ def strain_hist(): # grid - fileGrid = Dataset("x1.40962.grid.nc","r") + fileGrid = Dataset("grid.40962.nc","r") nCells = len(fileGrid.dimensions["nCells"]) nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_map.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_map.py index e277f2ce93c0..dfffc52c384e 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_map.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain/strain_map.py @@ -98,7 +98,7 @@ def plot_subfigure(axes, fig, nVertices, vertexDegree, cellsOnVertex, xCell, yCe def strain_map(): # grid - fileGrid = Dataset("x1.40962.grid.nc","r") + fileGrid = Dataset("grid.40962.nc","r") nCells = len(fileGrid.dimensions["nCells"]) nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/create_ic.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/create_ic.py index e41bbd763914..6657c6ff10f3 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/create_ic.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/create_ic.py @@ -653,7 +653,7 @@ def create_ic(): print(" Gridsize: ", gridSize) # input - filenameIn = "x1.%i.grid.nc" %(gridSize) + filenameIn = "grid.%i.nc" %(gridSize) fileIn = Dataset(filenameIn,"r") diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/get_testcase_data.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/get_testcase_data.py index 9c514bec9e52..227dd1b31f42 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/get_testcase_data.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/get_testcase_data.py @@ -4,10 +4,10 @@ def get_testcase_data(): - filenames = ["x1.10242.grid.nc", - "x1.163842.grid.nc", - "x1.2562.grid.nc", - "x1.40962.grid.nc"] + filenames = ["grid.10242.nc", + "grid.163842.nc", + "grid.2562.nc", + "grid.40962.nc"] dirName = "https://web.lcrc.anl.gov/public/e3sm/mpas_standalonedata/mpas-seaice/testcases/strain_stress_divergence/" diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/run_model.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/run_model.py index 4b6791e274b1..8561f6c45e2d 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/run_model.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/run_model.py @@ -25,7 +25,7 @@ def run_model(): print(" Gridsize: ", gridSize) os.system("rm grid.nc ic.nc") - os.system("ln -s x1.%i.grid.nc grid.nc" %(gridSize)) + os.system("ln -s grid.%i.nc grid.nc" %(gridSize)) os.system("ln -s ic_%i.nc ic.nc" %(gridSize)) if (operatorMethod == "wachspress"): diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_hist.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_hist.py index df68d7e6325a..e1ba8c7e9085 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_hist.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_hist.py @@ -9,7 +9,7 @@ def strain_stress_divergence_hist(): # grid - fileGrid = Dataset("x1.40962.grid.nc","r") + fileGrid = Dataset("grid.40962.nc","r") nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_map.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_map.py index 432f7cf6f56b..5bf622a7bbab 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_map.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/strain_stress_divergence/strain_stress_divergence_map.py @@ -98,7 +98,7 @@ def plot_subfigure(axes, fig, nVertices, vertexDegree, cellsOnVertex, xCell, yCe def strain_stress_divergence_map(): # grid - fileGrid = Dataset("x1.40962.grid.nc","r") + fileGrid = Dataset("grid.40962.nc","r") nCells = len(fileGrid.dimensions["nCells"]) nVertices = len(fileGrid.dimensions["nVertices"]) diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/create_ic.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/create_ic.py index f32a30c47b91..a1aa3bd54725 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/create_ic.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/create_ic.py @@ -653,7 +653,7 @@ def create_ic(): print(" Gridsize: ", gridSize) # input - filenameIn = "x1.%i.grid.nc" %(gridSize) + filenameIn = "grid.%i.nc" %(gridSize) fileIn = Dataset(filenameIn,"r") diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/examine_mesh.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/examine_mesh.py new file mode 100644 index 000000000000..23cd1b7ff686 --- /dev/null +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/examine_mesh.py @@ -0,0 +1,71 @@ +from netCDF4 import Dataset +from mpl_toolkits import mplot3d +import numpy as np +import matplotlib.pyplot as plt +import argparse + +parser = argparse.ArgumentParser(description='Plot vertices and edges of a cell') + +parser.add_argument('-i', dest='filenameIn', required=True, help='Input mesh file') +parser.add_argument('-c', dest='iCell', required=True, type=int, help='iCell to plot') + +args = parser.parse_args() + + +filein = Dataset(args.filenameIn,"r") + +nCells = len(filein.dimensions["nCells"]) +nVertices = len(filein.dimensions["nVertices"]) +nEdges = len(filein.dimensions["nEdges"]) + +xCell = filein.variables["xCell"][:] +yCell = filein.variables["yCell"][:] +zCell = filein.variables["zCell"][:] + +xVertex = filein.variables["xVertex"][:] +yVertex = filein.variables["yVertex"][:] +zVertex = filein.variables["zVertex"][:] + +xEdge = filein.variables["xEdge"][:] +yEdge = filein.variables["yEdge"][:] +zEdge = filein.variables["zEdge"][:] + +nEdgesOnCell = filein.variables["nEdgesOnCell"][:] + +edgesOnCell = filein.variables["edgesOnCell"][:] +verticesOnCell = filein.variables["verticesOnCell"][:] + +edgesOnCell[:] -= 1 +verticesOnCell[:] -= 1 + +#iCell = 123 +iCell = args.iCell + +colors = ["red","green","blue","orange","yellow","purple","magenta","cyan","teal"] + +fig = plt.figure() +ax = plt.axes(projection="3d") + +for iEdgeOnCell in range(0,nEdgesOnCell[iCell]): + + iVertexOnCell1 = iEdgeOnCell + iVertexOnCell2 = iEdgeOnCell + 1 + if (iVertexOnCell2 == nEdgesOnCell[iCell]): iVertexOnCell2 = 0 + + iVertex1 = verticesOnCell[iCell,iVertexOnCell1] + iVertex2 = verticesOnCell[iCell,iVertexOnCell2] + + iEdge = edgesOnCell[iCell,iEdgeOnCell] + + print(xVertex[iVertex1],yVertex[iVertex1],zVertex[iVertex1]) + print(xEdge [iEdge], yEdge [iEdge], zEdge [iEdge]) + print(xVertex[iVertex2],yVertex[iVertex2],zVertex[iVertex2]) + print() + + ax.scatter3D([xVertex[iVertex1]],[yVertex[iVertex1]],[zVertex[iVertex1]], c=colors[iEdgeOnCell], marker="+"); + ax.scatter3D([xEdge [iEdge]], [yEdge [iEdge]], [zEdge [iEdge]], c=colors[iEdgeOnCell], marker="."); + ax.scatter3D([xVertex[iVertex2]],[yVertex[iVertex2]],[zVertex[iVertex2]], c=colors[iEdgeOnCell], marker="x"); + +plt.show() + +filein.close() diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/get_testcase_data.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/get_testcase_data.py index 9c514bec9e52..227dd1b31f42 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/get_testcase_data.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/get_testcase_data.py @@ -4,10 +4,10 @@ def get_testcase_data(): - filenames = ["x1.10242.grid.nc", - "x1.163842.grid.nc", - "x1.2562.grid.nc", - "x1.40962.grid.nc"] + filenames = ["grid.10242.nc", + "grid.163842.nc", + "grid.2562.nc", + "grid.40962.nc"] dirName = "https://web.lcrc.anl.gov/public/e3sm/mpas_standalonedata/mpas-seaice/testcases/strain_stress_divergence/" diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/run_model.py b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/run_model.py index 5c249bc8a415..508a8c43acc9 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/run_model.py +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/run_model.py @@ -26,7 +26,7 @@ def run_model(): print(" Gridsize: ", gridSize) os.system("rm grid.nc ic.nc") - os.system("ln -s x1.%i.grid.nc grid.nc" %(gridSize)) + os.system("ln -s grid.%i.nc grid.nc" %(gridSize)) os.system("ln -s ic_%i.nc ic.nc" %(gridSize)) if (operatorMethod == "wachspress"): diff --git a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/streams.seaice.stress_divergence b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/streams.seaice.stress_divergence index 3bf8402c3dce..bdd7389b8721 100644 --- a/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/streams.seaice.stress_divergence +++ b/components/mpas-seaice/testing_and_setup/testcases/spherical_operators/stress_divergence/streams.seaice.stress_divergence @@ -57,6 +57,10 @@ + + + + Date: Wed, 8 Dec 2021 14:43:27 -0600 Subject: [PATCH 202/301] remove unused files --- .../theta-l_kokkos/cxx/ElementsBuffers.cpp | 74 ----------------- .../theta-l_kokkos/cxx/ElementsBuffers.hpp | 82 ------------------- 2 files changed, 156 deletions(-) delete mode 100644 components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.cpp delete mode 100644 components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.hpp diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.cpp b/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.cpp deleted file mode 100644 index e2fdaa11c9d9..000000000000 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/******************************************************************************** - * HOMMEXX 1.0: Copyright of Sandia Corporation - * This software is released under the BSD license - * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory - *******************************************************************************/ - -#include "ElementsBuffers.hpp" - -namespace Homme { - -void ElementsBuffers::init(const int num_elems) { - pressure = ExecViewManaged("Non-hydrostatic pressure", num_elems); - pi = ExecViewManaged("Hydrostatic Pressure at midpoints", num_elems); - pi_grad = ExecViewManaged("Hydrostatic Pressure gradient at midpoints", num_elems); - exner = ExecViewManaged("Exner pressure", num_elems); - v_i = ExecViewManaged("Horizontal velocity at interfaces", num_elems); - dp_i = ExecViewManaged("Pressure increments at interfaces", num_elems); - dpnh_dp_i = ExecViewManaged("Non-hydrostatic over hydrostatic pressure ratio", num_elems); - vdp = ExecViewManaged("(u,v)*dp", num_elems); - div_vdp = ExecViewManaged("Divergence of (u,v)*dp", num_elems); - ephi = ExecViewManaged("Energy (phi+0.5*|u|^2", num_elems); - temp_column = ExecViewManaged("A temporary column worth of data", num_elems); - omega_p = ExecViewManaged("Omega_P = omega/pressure = (Dp/Dt)/pressure", num_elems); - - vstar = ExecViewManaged("Buffer for (flux v)/dp", num_elems); - dpdissk = ExecViewManaged("dpdissk", num_elems); - // pressure_grad = ExecViewManaged("Gradient of pressure", num_elems); - // temperature_virt = ExecViewManaged("Virtual Temperature", num_elems); - // temperature_grad = ExecViewManaged("Gradient of temperature", num_elems); - // ephi = ExecViewManaged("Kinetic Energy + Geopotential Energy", num_elems); - // energy_grad = ExecViewManaged("Gradient of ephi", num_elems); - // vorticity = ExecViewManaged("Vorticity", num_elems); - - // ttens = ExecViewManaged("Temporary for temperature",num_elems); - // dptens = ExecViewManaged("Temporary for dp3d",num_elems); - // vtens = ExecViewManaged("Temporary for velocity",num_elems); - - // vstar = ExecViewManaged("buffer for (flux v)/dp", - // num_elems); - // dpdissk = ExecViewManaged( - // "dpdissk", num_elems); - - // preq_buf = ExecViewManaged("Preq Buffer", num_elems); - - // sdot_sum = ExecViewManaged("Sdot sum buffer", num_elems); - - // div_buf = ExecViewManaged("Divergence Buffer", - // num_elems); - // grad_buf = ExecViewManaged("Gradient Buffer", - // num_elems); - // curl_buf = ExecViewManaged("Vorticity Buffer", - // num_elems); - - // sphere_vector_buf = ExecViewManaged("laplacian vector Buffer", num_elems); - - // divergence_temp = ExecViewManaged("Divergence temporary", - // num_elems); - // vorticity_temp = ExecViewManaged("Vorticity temporary", - // num_elems); - // lapl_buf_1 = ExecViewManaged("Scalar laplacian Buffer", num_elems); - // lapl_buf_2 = ExecViewManaged("Scalar laplacian Buffer", num_elems); - // lapl_buf_3 = ExecViewManaged("Scalar laplacian Buffer", num_elems); - // v_vadv_buf = ExecViewManaged("v_vadv buffer", - // num_elems); - // t_vadv_buf = ExecViewManaged("t_vadv buffer", - // num_elems); - // eta_dot_dpdn_buf = ExecViewManaged("eta_dot_dpdpn buffer", - // num_elems); - - // kernel_start_times = ExecViewManaged("Start Times", num_elems); - // kernel_end_times = ExecViewManaged("End Times", num_elems); -} - -} // namespace Homme diff --git a/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.hpp b/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.hpp deleted file mode 100644 index 678f85fa3c56..000000000000 --- a/components/homme/src/theta-l_kokkos/cxx/ElementsBuffers.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/******************************************************************************** - * HOMMEXX 1.0: Copyright of Sandia Corporation - * This software is released under the BSD license - * See the file 'COPYRIGHT' in the HOMMEXX/src/share/cxx directory - *******************************************************************************/ - -#ifndef HOMMEXX_ELEMENTS_BUFFERS_HPP -#define HOMMEXX_ELEMENTS_BUFFERS_HPP - -#include "ElementsBuffersBase.hpp" - - -namespace Homme { - -// Elements-dependent buffers. -struct ElementsBuffers : public ElementsBuffersBase { - - ElementsBuffers() = default; - void init(const int num_elems); - - ExecViewUnmanaged pressure; - ExecViewUnmanaged pi; - ExecViewUnmanaged pi_grad; - ExecViewUnmanaged exner; - ExecViewUnmanaged v_i; - ExecViewUnmanaged dp_i; - ExecViewUnmanaged dpnh_dp_i; - ExecViewUnmanaged vdp; - ExecViewUnmanaged div_vdp; - ExecViewUnmanaged ephi; - ExecViewUnmanaged omega_p; - - ExecViewUnmanaged temp_column; - - // ExecViewManaged pressure_grad; - // ExecViewManaged temperature_virt; - // ExecViewManaged temperature_grad; - // ExecViewManaged omega_p; - // ExecViewManaged ephi; - // ExecViewManaged energy_grad; - // ExecViewManaged vorticity; - // ExecViewManaged ttens; - // ExecViewManaged dptens; - // ExecViewManaged vtens; - - - // Buffers for EulerStepFunctor - ExecViewUnmanaged vstar; - ExecViewUnmanaged dpdissk; - - // ExecViewManaged preq_buf; - // // sdot_sum is used in case rsplit=0 and in energy diagnostics - // // (not yet coded). - // ExecViewManaged sdot_sum; - // // Buffers for spherical operators - // ExecViewManaged div_buf; - // ExecViewManaged grad_buf; - // ExecViewManaged curl_buf; - - // ExecViewManaged sphere_vector_buf; - - // ExecViewManaged divergence_temp; - // ExecViewManaged vorticity_temp; - // ExecViewManaged lapl_buf_1; - // ExecViewManaged lapl_buf_2; - // ExecViewManaged lapl_buf_3; - - // // Buffers for vertical advection terms in V and T for case - // // of Eulerian advection, rsplit=0. These buffers are used in both - // // cases, rsplit>0 and =0. Some of these values need to be init-ed - // // to zero at the beginning of each RK stage. - // ExecViewManaged v_vadv_buf; - // ExecViewManaged t_vadv_buf; - // ExecViewManaged eta_dot_dpdn_buf; - - // ExecViewManaged kernel_start_times; - // ExecViewManaged kernel_end_times; -}; - -} // Homme - -#endif // HOMMEXX_ELEMENTS_BUFFERS_HPP From 516eb8012b90a4daf622b8586b984927433c1244 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Wed, 8 Dec 2021 14:45:16 -0600 Subject: [PATCH 203/301] fix building, add a timer, fix push logic --- components/eam/bld/configure | 2 +- components/homme/src/share/prim_driver_base.F90 | 4 ++++ .../homme/src/theta-l_kokkos/prim_driver_mod.F90 | 13 ++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/components/eam/bld/configure b/components/eam/bld/configure index 89a23d255d58..4b5cd9e48c06 100755 --- a/components/eam/bld/configure +++ b/components/eam/bld/configure @@ -2774,7 +2774,7 @@ sub write_cppdefs } if ($dyn eq "se" and $opts{"dyn_target"} eq "theta-l_kokkos") { - print $fh " -DKOKKOS_TARGET -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE " + print $fh " -DKOKKOS_TARGET -DMODEL_THETA_L -DHOMME_ENABLE_COMPOSE -DCOMPOSE_PORT" } if ($dyn eq "se" and $opts{"dyn_target"} eq "preqx_kokkos") { diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index a05530aee54c..0b65d090830b 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1593,6 +1593,8 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) sum_fq = 0 #endif + call t_startf("ApplyCAMForcing_tracers") + #ifdef MODEL_THETA_L if (dt_remap_factor==0) then adjust_ps=.true. ! stay on reference levels for Eulerian case @@ -1756,6 +1758,8 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) #endif + call t_stopf("ApplyCAMForcing_tracers") + end subroutine applyCAMforcing_tracers diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index a39c0ee0c914..f53d38970027 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -412,7 +412,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv compute_diagnostics = .false. endif - call init_logic_for_push_to_c() + call init_logic_for_push_to_c(nsplit_iteration) dt_q = dt*dt_tracer_factor if (dt_remap_factor == 0) then @@ -525,16 +525,23 @@ end subroutine setup_element_pointers !test with forcing (not performant): ! (always forcing and push to c) -> (subcycle) -> (always push to f) - subroutine init_logic_for_push_to_c() + subroutine init_logic_for_push_to_c(nsplit_iter) use control_mod, only: test_with_forcing + integer, intent(in) :: nsplit_iter compute_forcing_and_push_to_c = .false. ! Scream already computes all forcing using the same pointers ! stored in Hommexx, so the forcing is already up to date -#if defined(HOMMEXX_BENCHMARK_NOFORCING) || defined(SCREAM) +#if defined(HOMMEXX_BENCHMARK_NOFORCING) +#elif defined(SCREAM) + +#elif defined(CAM) + if (nsplit_iter == 1) then + compute_forcing_and_push_to_c = .true. + endif #else if(test_with_forcing)then compute_forcing_and_push_to_c = .true. From 963a9087d37e3efef65d009c88341b864093276b Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 8 Dec 2021 18:48:15 -0600 Subject: [PATCH 204/301] Remove DMS from ocean options in BGC compsets, not currently supported --- cime_config/allactive/config_compsets.xml | 38 +++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index da99c4ba68c6..2e90149f799c 100644 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -91,7 +91,7 @@ BGCEXP_CNTL_CNPRDCTC_1850 - 1850_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + 1850_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC @@ -101,12 +101,12 @@ BGCEXP_BCRC_CNPRDCTC_20TR - 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC BGCEXP_BCRC_CNPRDCTC_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC @@ -121,12 +121,12 @@ BGCEXP_BCRD_CNPRDCTC_20TR - 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRD BGCEXP_BCRD_CNPRDCTC_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD + SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRD @@ -141,12 +141,12 @@ BGCEXP_BDRC_CNPRDCTC_20TR - 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRC + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRC BGCEXP_BDRC_CNPRDCTC_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRC + SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRC @@ -161,12 +161,12 @@ BGCEXP_BDRD_CNPRDCTC_20TR - 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRD + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD BGCEXP_BDRD_CNPRDCTC_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRD + SSP585_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD @@ -181,7 +181,7 @@ BGCEXP_CNTL_CNPECACNT_1850 - 1850_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + 1850_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC @@ -191,12 +191,12 @@ BGCEXP_BCRC_CNPECACNT_20TR - 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC BGCEXP_BCRC_CNPECACNT_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC + SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRC @@ -211,12 +211,12 @@ BGCEXP_BCRD_CNPECACNT_20TR - 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD + 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRD BGCEXP_BCRD_CNPECACNT_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD + SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BCRD @@ -231,12 +231,12 @@ BGCEXP_BDRC_CNPECACNT_20TR - 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRC + 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRC BGCEXP_BDRC_CNPECACNT_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRC + SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRC @@ -251,12 +251,12 @@ BGCEXP_BDRD_CNPECACNT_20TR - 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRD + 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD BGCEXP_BDRD_CNPECACNT_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRD + SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD @@ -289,7 +289,7 @@ BGCEXP_BDRD_CNPECACNT_SSP585_CMIP6 - SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BDRD + SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD From 170450b982aed86ce45a0f9bc1580976f92430dc Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 10 Dec 2021 13:46:21 -0800 Subject: [PATCH 205/301] Update PE layouts on cori-knl for ne120pg2 F compsets Add S,M,L,X,T for MPASSI and update S,M,L,X,T for CICE [bfb] --- cime_config/allactive/config_pesall.xml | 259 ++++++++++++++++++------ 1 file changed, 192 insertions(+), 67 deletions(-) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3fc041f771c6..d86211bc33de 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -6589,129 +6589,254 @@ 0 - - cori-knl ne120 F-compset on 675 nodes, 64x2, sypd=1.95 - 64 - 128 + + cori-knl ne120pg2 F-compset with MPASSI on 1024 nodes, 16x4, sypd=2.2 + 16 + 256 - 43200 - 43200 - 43200 - 43200 - 43200 + 16384 + 16384 + 16384 + 16384 + 16384 64 64 - 43200 + 16384 - 2 + 4 + 4 + 4 + 4 + 1 + 1 + 1 + 1 + + + + cori-knl ne120pg2 F-compset with MPASSI on 338 nodes, 32x8, sypd=1.4 + 32 + 256 + + 10816 + 10816 + 10816 + 9600 + 9600 + 64 + 64 + 10816 + + + 8 2 + 2 + 2 + 1 + 1 + 1 + 1 + + + + cori-knl ne120pg2 F-compset with MPASSI on 169 nodes, 32x8, sypd=0.84 + 32 + 256 + + 5408 + 5408 + 5408 + 5200 + 5200 + 64 + 64 + 4096 + + + 8 + 4 + 4 + 4 + 1 + 1 + 1 + 1 + + + + cori-knl ne120pg2 F-compset with MPASSI on 85 nodes, 32x8, sypd=0.52 + 32 + 256 + + 2720 + 2720 + 2720 + 2560 + 2560 + 32 + 32 + 2048 + + + 8 + 4 + 4 + 4 + 1 + 1 + 1 + 1 + + + + cori-knl ne120pg2 F-compset with MPASSI on 43 nodes, 16x8, sypd=0.29 + 16 + 256 + + 688 + 688 + 688 + 640 + 640 + 16 + 16 + 688 + + + 16 + 16 + 16 + 16 + 1 + 1 + 1 + 1 + + + + cori-knl ne120pg2 F-compset with CICE on 1024 nodes, 16x4, sypd=2.4 + 16 + 256 + + 16384 + 16384 + 16384 + 16384 + 16384 + 64 + 64 + 16384 + + + 4 + 1 1 1 1 1 1 - 2 + 1 - cori-knl ne120 F-compset on 323 nodes, 67x4, sypd=1.18 - 67 - 268 + cori-knl ne120pg2 F-compset with CICE on 338 nodes, 64x4, sypd=1.6 + 64 + 256 - 21600 - 21600 - 21600 - 21600 - 21600 - 67 - 67 - 21600 + 21632 + 21632 + 21632 + 21632 + 21632 + 64 + 64 + 21632 4 - 4 + 2 1 1 1 1 1 - 4 + 1 - cori-knl ne120 F-compset on 162 nodes, 67x4, sypd=0.69 - 67 - 268 + cori-knl ne120pg2 F-compset with CICE on 169 nodes, 64x4, sypd=1.0 + 64 + 256 - 10800 - 10800 - 10800 - 10800 - 10800 - 67 - 67 - 10800 + 10816 + 10816 + 10816 + 10816 + 10816 + 64 + 64 + 10816 4 - 4 - 4 - 1 + 2 + 2 + 2 1 1 1 - 4 + 1 - cori-knl ne120 F-compset on 81 nodes, 67x4, sypd=0.35 - 67 - 268 + cori-knl ne120pg2 F-compset with CICE on 85 nodes, 64x4, sypd=0.55 + 64 + 256 - 5427 - 5427 - 5427 - 5427 - 5427 - 67 - 67 - 5427 + 5440 + 5440 + 5440 + 5440 + 5440 + 64 + 64 + 85 4 - 4 - 4 - 1 + 2 + 2 + 2 1 1 1 - 4 + 1 - cori-knl ne120 F-compset on 42 nodes, 67x4, sypd=0.19 - 67 + cori-knl ne120pg2 F-compset with CICE on 43 nodes, 64x4, sypd=0.31 + 64 268 - 2814 - 2814 - 2814 - 2814 - 2814 - 67 - 67 - 2814 + 2752 + 2752 + 2752 + 2752 + 2752 + 64 + 64 + 2752 4 2 2 - 1 + 2 1 1 1 - 4 + 1 From 379fbcca57e4b2508539c27c0e4f183b8370b540 Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 10 Dec 2021 15:10:59 -0800 Subject: [PATCH 206/301] For Perlmutter, Update modules for cuda after maint. --- cime_config/machines/config_machines.xml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 19d42bae713c..d8fb229a2ad7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -56,7 +56,7 @@ --> - Perlmutter at NERSC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) and 4 nvidia A100's. + Perlmutter at NERSC. Phase1 only: Each GPU node has single AMD EPYC 7713 64-Core (Milan) and 4 nvidia A100's. login Linux gnu,gnugpu,nvidia,nvidiagpu @@ -111,6 +111,7 @@ PrgEnv-gnu PrgEnv-nvidia cuda + cpe-cuda cudatoolkit craype-accel-nvidia80 craype-accel-host @@ -127,15 +128,19 @@ PrgEnv-nvidia - nvidia/21.7 + nvidia/21.9 - cuda + cpe-cuda + cudatoolkit + craype-accel-nvidia80 - cuda + cpe-cuda + cudatoolkit + craype-accel-nvidia80 @@ -147,13 +152,13 @@ - cray-libsci/21.08.1.2 + cray-libsci craype - cray-mpich/8.1.10 + cray-mpich/8.1.11 cray-hdf5-parallel/1.12.0.7 cray-netcdf-hdf5parallel/4.7.4.7 cray-parallel-netcdf/1.12.1.7 - cmake/3.20.5 + cmake/3.22.0 From c40a28cb46da4736ff1c14ed00f41b40764cfa6e Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 10 Dec 2021 16:40:48 -0800 Subject: [PATCH 207/301] Add allow_error="true" to try getting around module "error" messages. --- cime_config/machines/config_machines.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index d8fb229a2ad7..a0c5d7f6bc87 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -94,7 +94,7 @@ -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} - + /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh @@ -110,8 +110,6 @@ cray-parallel-netcdf PrgEnv-gnu PrgEnv-nvidia - cuda - cpe-cuda cudatoolkit craype-accel-nvidia80 craype-accel-host From 5d99869f2909d835d661c0d53cb2a00395b29744 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sat, 11 Dec 2021 22:33:04 -0600 Subject: [PATCH 208/301] ftype0 state for ne30 (tracer forcing frequency hardcoded) --- components/homme/src/share/cxx/prim_step.cpp | 23 ++++++++-- .../src/theta-l_kokkos/cxx/CamForcing.cpp | 12 +++++- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 17 ++++++++ .../cxx/cxx_f90_interface_theta.cpp | 7 +--- .../src/theta-l_kokkos/prim_driver_mod.F90 | 42 +++++++++++++++---- 5 files changed, 85 insertions(+), 16 deletions(-) diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index 3cb2f0e4b51c..dd544b84a302 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -135,18 +135,35 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { //decide on tracer forcing //SCREAM, CAM -- support only ftype2 + // + // + //PROPAGATE the same change to EUL xx part #if defined(CAM) - apply_forcing = 0; +// apply_forcing = (params.ftype == ForcingAlg::FORCING_2 && +// params.nsplit_iteration == 1 ); + apply_forcing = 1; #elif defined(SCREAM) apply_forcing = (params.ftype == ForcingAlg::FORCING_2 && - nstep_iteration == 1 ); + params.nsplit_iteration == 1 ); #else apply_forcing = forcing_0or2; #endif + +//printf("OG before-if: applyforcing %d \n",apply_forcing); + + if (apply_forcing) { + +//printf("OG param nsplit %d, param nsplit iter %d \n",params.nsplit,params.nsplit_iteration); +//std::cout<< "OG in appl forcing for tracers,dt_q " << std::to_string(dt_q)<<"\n"; +//std::cout<< "OG in appl forcing for tracers,dt_q_nsplit " << std::to_string(dt_q_nsplit)<<"\n"; + + +//PROPAGATE THIS TOO +//do not propagate derived FVtheta ? // Apply tracer forcings over tracer time step. -#ifdef SCREAM +#if defined(CAM) || defined(SCREAM) apply_cam_forcing_tracers(dt_q_nsplit); #else apply_cam_forcing_tracers(dt_q); diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index c713eae8e11a..544347a82a5c 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -19,7 +19,17 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, const TimeLevel& tl, const SimulationParams& p) { GPTLstart("ApplyCAMForcing_tracers"); - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, p.moisture); + +//original +// ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, p.moisture); + +//!!!! adjustment, +//call for ftype0 + CAM, or any ftype standalone homme + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, MoistDry::MOIST); + +//call for ftype2 + CAM +// ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, true, MoistDry::MOIST); + GPTLstop("ApplyCAMForcing_tracers"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 44f0b9ebba32..c5abc40bfe07 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -199,6 +199,10 @@ class ForcingFunctor constexpr int LAST_INT_PACK = ColInfo::LastPack; constexpr int LAST_INT_PACK_END = ColInfo::LastPackEnd; +//printf("OG in TagStates!!!!!!!"); + + + KernelVariables kv(team, m_tu_states); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), [&](const int idx) { @@ -220,6 +224,17 @@ class ForcingFunctor Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { +#if 0 +//printf("i %d,j %d",igp,jgp); +if(kv.ie == 0 && ilev == (NUM_LEV-1) && igp ==0 && jgp == 0){ + //for(int ii=0; ii< 4; ++ii) + // for (int jj=0; jj<4; ++jj){ + // auto u = Homme::subview(m_state.m_v,kv.ie,m_data.np1,0,ii,jj); +//printf("i %d,j %d,u after subcycle is %1.29e \n",ii,jj, u(0)[0]); +printf("OG fvtheta %1.15e , vtheta %1.15e \n",fvtheta(ilev)[7], vtheta(ilev)[7]); +} +#endif + vtheta(ilev) += m_dt*fvtheta(ilev); phi(ilev) += m_dt*fphi(ilev); @@ -243,6 +258,8 @@ class ForcingFunctor m_np1 = np1; m_np1_qdp = np1_qdp; m_adjustment = adjustment; + +//this should sit in SP m_moist = (moisture==MoistDry::MOIST); Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 1b5fc431b2eb..4cb5258403db 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -152,11 +152,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, // TODO Parse a fortran string and set this properly. For now, our code does // not depend on this except to throw an error in apply_test_forcing. std::string test_name(*test_case); - //if (test_name=="jw_baroclinic") { - params.test_case = TestCase::JW_BAROCLINIC; - //} else { - // Errors::runtime_abort("Error! Unknown test case '" + test_name + "'.\n"); - //} + //TEMP + params.test_case = TestCase::JW_BAROCLINIC; // Now this structure can be used safely params.params_set = true; diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index f53d38970027..7fe29bd0b04b 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -425,6 +425,9 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv ! Test forcing is only for standalone Homme (and only for some tests/configurations) if (compute_forcing_and_push_to_c) then + +!print *, 'OG IN PUSH FORCING', elem(1)%derived%fvtheta(1,1,72),elem_derived_fvtheta(1,1,72,1) + call compute_test_forcing_dummy(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) call t_startf('push_to_cxx') call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & @@ -470,7 +473,7 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv end subroutine prim_run_subcycle - +#if 0 subroutine setup_element_pointers (elem) use element_state, only : allocate_element_arrays, elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, elem_state_phis, & @@ -510,11 +513,12 @@ subroutine setup_element_pointers (elem) enddo end subroutine setup_element_pointers +#endif !the next 2 routines have logic for push to/from F and for forcing routine ! -!there are 3 cases: +!STANDALONE HOMME there are 3 cases: ! !performance: ! (no forcing, no push to c) -> (subcycle) -> (no push to f) @@ -559,18 +563,42 @@ subroutine init_logic_for_push_to_f(tl,statefreq,nextOutputStep,compute_diagnost type (TimeLevel_t), intent(in) :: tl integer, intent(in) :: statefreq, nextOutputStep, nsplit_iter logical, intent(in) :: compute_diagnostics + + logical :: time_for_homme_output + + time_for_homme_output = & + (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) #ifdef HOMMEXX_BENCHMARK_NOFORCING +!standalone homme, only benchmarks push_to_f = .false. -#else - - if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then + +#elif defined(SCREAM) +!SCREAM run, do nothing + +#elif defined(CAM) +!CAM run, push at the end of nsplit loop + if (nsplit_iter == nsplit) then push_to_f = .true. endif - if (test_with_forcing) then + +!CAM also needs some of homme output + !if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then + if ( time_for_homme_output ) then push_to_f = .true. endif - if (nsplit_iter == nsplit) then + +#else +!standalone homme, not benchmarks +!output + !if (MODULO(tl%nstep,statefreq)==0 .or. tl%nstep >= nextOutputStep .or. compute_diagnostics) then + if ( time_for_homme_output ) then + push_to_f = .true. + endif + +!push for standalone homme with forcing, test_with_forcing=false +!for most standalone homme tests + if (test_with_forcing) then push_to_f = .true. endif From 0c857cc909a7d1a1d83faca061621c4694e483a8 Mon Sep 17 00:00:00 2001 From: Alice Barthel Date: Mon, 13 Dec 2021 10:51:29 -0600 Subject: [PATCH 209/301] 1. corrected last layer thickness to partial, 2. fetch index_temp as best practice --- .../analysis_members/mpas_ocn_mixed_layer_heat_budget.F | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F index 6ede9e505a4a..748a53b6bca9 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F @@ -176,6 +176,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ integer, pointer :: nVertLevels, nCellsSolve integer :: iTracer, k, iCell, num_tracers integer, dimension(:), pointer :: maxLevelCell + integer, pointer :: index_temperature real(kind=RKIND), dimension(:, :, :), pointer :: & tracers, tracerTend @@ -222,7 +223,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ call mpas_pool_get_array(mixedLayerHeatBudgetAMPool, 'activeTracersTendML', activeTracersTendML) call mpas_pool_get_array(mixedLayerHeatBudgetAMPool, 'bruntVaisalaFreqML', bruntVaisalaFreqML) call mpas_pool_get_array(mixedLayerDepthsAMPool, 'dThreshMLD', dThreshMLD) - + call mpas_pool_get_dimension(tracersPool, 'index_temperature', index_temperature) call mpas_pool_get_array(tracersPool, 'activeTracers', tracers, timeLevel) call mpas_pool_get_array(tracerTendPool, 'activeTracersTend', tracerTend) @@ -255,7 +256,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ activeTracerForcingMLTend(iTracer, iCell) = activeTracerForcingMLTend(iTracer, iCell) + & activeTracerSurfaceFluxTendency(iTracer, k, iCell)*layerThickness(k, iCell) - if (iTracer == 1) THEN + if (iTracer == index_temperature) THEN activeTracerForcingMLTend(iTracer, iCell) = activeTracerForcingMLTend(iTracer, iCell) + & temperatureShortWaveTendency(k, iCell)*layerThickness(k, iCell) endif @@ -286,7 +287,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ activeTracerNonLocalTendency(iTracer, k, iCell)*difference activeTracerForcingMLTend(iTracer, iCell) = activeTracerForcingMLTend(iTracer, iCell) + & activeTracerSurfaceFluxTendency(iTracer, k, iCell)*difference - if (iTracer == 1) THEN + if (iTracer == index_temperature) THEN activeTracerForcingMLTend(iTracer, iCell) = activeTracerForcingMLTend(iTracer, iCell) + & temperatureShortWaveTendency(k, iCell)*difference endif @@ -305,7 +306,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ activeTracerNonLocalMLTend(iTracer, iCell) = activeTracerNonLocalMLTend(iTracer, iCell)/dThreshMLD(iCell) activeTracerForcingMLTend(iTracer, iCell) = activeTracerForcingMLTend(iTracer, iCell)/dThreshMLD(iCell) end do - bruntVaisalaFreqML(iCell) = bruntVaisalaFreqML(iCell) + BruntVaisalaFreqTop(k, iCell)*layerThickness(k, iCell) + bruntVaisalaFreqML(iCell) = bruntVaisalaFreqML(iCell) + BruntVaisalaFreqTop(k, iCell)*difference bruntVaisalaFreqML(iCell) = bruntVaisalaFreqML(iCell)/dThreshMLD(iCell) end do From 98af7781722f22423bab68dbbeee73cb2855f8ee Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Mon, 13 Dec 2021 11:12:43 -0600 Subject: [PATCH 210/301] Corrects coupling of ocean surface DON with sea ice DON. Replaces index_x2i_So_don1 with index_x2i_So_doc3 in ice_comp_mct. non-BFB with ocean and sea ice BGC active. BFB otherwise. Fixes issue #4698 --- components/mpas-seaice/driver/ice_comp_mct.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-seaice/driver/ice_comp_mct.F b/components/mpas-seaice/driver/ice_comp_mct.F index 20a3d00fb70d..fada4797f813 100644 --- a/components/mpas-seaice/driver/ice_comp_mct.F +++ b/components/mpas-seaice/driver/ice_comp_mct.F @@ -2064,7 +2064,7 @@ subroutine ice_import_mct(x2i_i, errorCode)!{{{ oceanDOCConc(2,i) = x2i_i % rAttr(index_x2i_So_doc2, n) * DOCPoolFractions(2) oceanDOCConc(3,i) = 0.0_RKIND oceanDICConc(1,i) = x2i_i % rAttr(index_x2i_So_dic1, n) - oceanDONConc(1,i) = x2i_i % rAttr(index_x2i_So_don1, n) * DOCPoolFractions(3) & + oceanDONConc(1,i) = x2i_i % rAttr(index_x2i_So_doc3, n) * DOCPoolFractions(3) & /carbonToNitrogenRatioDON(1) oceanNitrateConc(i) = x2i_i % rAttr(index_x2i_So_no3, n) oceanSilicateConc(i) = x2i_i % rAttr(index_x2i_So_sio3, n) From db532d13a9d820d1cf70602c1582d6327e30204d Mon Sep 17 00:00:00 2001 From: noel Date: Mon, 13 Dec 2021 09:17:49 -0800 Subject: [PATCH 211/301] whitespace cleanup --- cime_config/machines/config_machines.xml | 44 ++++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index a0c5d7f6bc87..9addd04ba049 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -89,12 +89,12 @@ --label -n {{ total_tasks }} -N {{ num_nodes }} - -c $SHELL{echo 128/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} - $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} + -c $SHELL{echo 128/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} + $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} - + /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh @@ -106,16 +106,16 @@ cray-hdf5-parallel - cray-netcdf-hdf5parallel + cray-netcdf-hdf5parallel cray-parallel-netcdf - PrgEnv-gnu - PrgEnv-nvidia - cudatoolkit - craype-accel-nvidia80 - craype-accel-host - perftools-base - perftools - darshan + PrgEnv-gnu + PrgEnv-nvidia + cudatoolkit + craype-accel-nvidia80 + craype-accel-host + perftools-base + perftools + darshan @@ -128,33 +128,33 @@ PrgEnv-nvidia nvidia/21.9 - + cpe-cuda cudatoolkit craype-accel-nvidia80 - + cpe-cuda cudatoolkit - craype-accel-nvidia80 + craype-accel-nvidia80 - + - craype-accel-host - + craype-accel-host + - craype-accel-host - + craype-accel-host + cray-libsci craype cray-mpich/8.1.11 cray-hdf5-parallel/1.12.0.7 - cray-netcdf-hdf5parallel/4.7.4.7 + cray-netcdf-hdf5parallel/4.7.4.7 cray-parallel-netcdf/1.12.1.7 cmake/3.22.0 @@ -163,7 +163,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 - + 1 1 From e5de53f4352cfec4e4efe5ae9f37c6d1b53caf7b Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 13 Dec 2021 11:48:24 -0600 Subject: [PATCH 212/301] reorg, bfb for fcase and homme --- components/homme/src/share/cxx/prim_step.cpp | 25 ++++++-------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index dd544b84a302..c22140ae6e24 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -131,28 +131,21 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { bool apply_forcing; const auto dt_q = dt * params.dt_tracer_factor; + //standalone homme nsplit=1 const auto dt_q_nsplit = dt * params.dt_tracer_factor * params.nsplit; //decide on tracer forcing - //SCREAM, CAM -- support only ftype2 - // - // //PROPAGATE the same change to EUL xx part -#if defined(CAM) -// apply_forcing = (params.ftype == ForcingAlg::FORCING_2 && -// params.nsplit_iteration == 1 ); - apply_forcing = 1; -#elif defined(SCREAM) - apply_forcing = (params.ftype == ForcingAlg::FORCING_2 && - params.nsplit_iteration == 1 ); +#if defined(CAM) || defined(SCREAM) + //CAM + xx supports only ftype2 and ftype0 + apply_forcing = + ( params.ftype == ForcingAlg::FORCING_0 || params.ftype == ForcingAlg::FORCING_2 ) && + params.nsplit_iteration == 1; +// apply_forcing = 1; #else apply_forcing = forcing_0or2; #endif - -//printf("OG before-if: applyforcing %d \n",apply_forcing); - - if (apply_forcing) { //printf("OG param nsplit %d, param nsplit iter %d \n",params.nsplit,params.nsplit_iteration); @@ -163,11 +156,7 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { //PROPAGATE THIS TOO //do not propagate derived FVtheta ? // Apply tracer forcings over tracer time step. -#if defined(CAM) || defined(SCREAM) apply_cam_forcing_tracers(dt_q_nsplit); -#else - apply_cam_forcing_tracers(dt_q); -#endif } set_tracer_transport_derived_values(params, elements, tl); From a17020f155fb443045b287812481713c8620747e Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 13 Dec 2021 14:53:04 -0600 Subject: [PATCH 213/301] fix ftype logic in xx, add a comment on fvtheta and fphi --- .../homme/src/share/cxx/HommexxEnums.hpp | 3 +-- .../homme/src/share/cxx/prim_driver.cpp | 21 +++++++------------ components/homme/src/share/cxx/prim_step.cpp | 14 ++++--------- .../cxx/cxx_f90_interface_theta.cpp | 2 ++ 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 0c0f6aad9c67..4ba0673e2720 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -40,8 +40,7 @@ enum class ComparisonOp { enum class ForcingAlg { FORCING_OFF, - FORCING_DEBUG, - FORCING_0, // Unsupported + FORCING_0, FORCING_1, // Unsupported FORCING_2, // TODO: Rename FORCING_1 and FORCING_2 to something more descriptive }; diff --git a/components/homme/src/share/cxx/prim_driver.cpp b/components/homme/src/share/cxx/prim_driver.cpp index 065576fef1f8..88dbcb7305a5 100644 --- a/components/homme/src/share/cxx/prim_driver.cpp +++ b/components/homme/src/share/cxx/prim_driver.cpp @@ -106,26 +106,21 @@ void prim_run_subcycle_c (const Real& dt, int& nstep, int& nm1, int& n0, int& np tl.update_tracers_levels(params.dt_tracer_factor); // Apply forcing. - // to make it readable, duplicate some code - - //CAM, support only ftype2, _tracers are done in dp_coupling -#if defined(CAM) - if(params.ftype == ForcingAlg::FORCING_2) { - apply_cam_forcing_dynamics(dt_remap); +#if defined(CAM) || defined(SCREAM) + //CAM and SCREAM, support only ftype0 and 2 + if (params.ftype == ForcingAlg::FORCING_0){ + apply_cam_forcing_tracers(dt_remap); } - - //SCREAM, support only ftype2, nothing is done in dp layer -#elif defined(SCREAM) - if(params.ftype == ForcingAlg::FORCING_2) { - if(params.nsplit_iteration == 1) + if (params.ftype == ForcingAlg::FORCING_2 && params.nsplit_iteration == 1 ){ apply_cam_forcing_tracers(dt_remap*params.nsplit); - apply_cam_forcing_dynamics(dt_remap); } + apply_cam_forcing_dynamics(dt_remap); + +#else //standalone homme, support ftype0 and ftype2 //ftype0 = ftype2 if dt_remap>=dt_tracer, but //ftype0 != ftype2 for dt_remap Date: Mon, 13 Dec 2021 14:54:49 -0600 Subject: [PATCH 214/301] switch bubble to ftype0 --- .../homme/test/reg_test/namelists/thetanh-moist-bubble.nl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl b/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl index 6583961704cc..5b310eb7b655 100644 --- a/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl +++ b/components/homme/test/reg_test/namelists/thetanh-moist-bubble.nl @@ -38,7 +38,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling hypervis_subcycle_tom = 1 hypervis_subcycle_q = 1 - se_ftype=2 + se_ftype=0 limiter_option = 9 planar_slice=.true. lx = 20000.0 From 2f079087e3a4059fbfc37d04a3adef781e4a515f Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 13 Dec 2021 15:43:46 -0600 Subject: [PATCH 215/301] remove hardcoding for adjustment --- .../homme/src/theta-l_kokkos/cxx/CamForcing.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 544347a82a5c..3e1a3f42cbc1 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -23,12 +23,16 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, //original // ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, p.moisture); -//!!!! adjustment, -//call for ftype0 + CAM, or any ftype standalone homme - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, MoistDry::MOIST); - -//call for ftype2 + CAM -// ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, true, MoistDry::MOIST); + bool adjustment; + if ( p.ftype == ForcingAlg::FORCING_0) adjustment = false; +#ifndef CAM +//this will change when ftype2 in homme changes + if ( p.ftype == ForcingAlg::FORCING_2) adjustment = false; +#endif +#ifdef CAM + if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; +#endif + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); GPTLstop("ApplyCAMForcing_tracers"); } From 35076fe40e9bb9d9974ca1683440e092f0fc45fb Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Mon, 13 Dec 2021 18:02:29 -0600 Subject: [PATCH 216/301] Removed Cf from sea ice constants --- .../src/column/constants/cice/ice_constants_colpkg.F90 | 1 - components/mpas-seaice/src/column/ice_colpkg.F90 | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-seaice/src/column/constants/cice/ice_constants_colpkg.F90 b/components/mpas-seaice/src/column/constants/cice/ice_constants_colpkg.F90 index a42413cacc09..18e678c907db 100644 --- a/components/mpas-seaice/src/column/constants/cice/ice_constants_colpkg.F90 +++ b/components/mpas-seaice/src/column/constants/cice/ice_constants_colpkg.F90 @@ -55,7 +55,6 @@ module ice_constants_colpkg cprho = cp_ocn*rhow ,&! for ocean mixed layer (J kg / K m^3) ! for ice strength - Cf = 17._dbl_kind ,&! ratio of ridging work to PE change in ridging Cp = 0.5_dbl_kind*gravit*(rhow-rhoi)*rhoi/rhow ,&! proport const for PE Pstar = 2.75e4_dbl_kind ,&! constant in Hibler strength formula ! (kstrength = 0) diff --git a/components/mpas-seaice/src/column/ice_colpkg.F90 b/components/mpas-seaice/src/column/ice_colpkg.F90 index bde30aa9e2eb..5a939941dea0 100644 --- a/components/mpas-seaice/src/column/ice_colpkg.F90 +++ b/components/mpas-seaice/src/column/ice_colpkg.F90 @@ -3514,8 +3514,9 @@ subroutine colpkg_ice_strength (ncat, & vicen, & strength) - use ice_constants_colpkg, only: p333, c0, c1, c2, Cf, Cp, Pstar, Cstar, & + use ice_constants_colpkg, only: p333, c0, c1, c2, Cp, Pstar, Cstar, & rhoi, puny + use ice_colpkg_shared, only: Cf use ice_mechred, only: asum_ridging, ridge_itd integer (kind=int_kind), intent(in) :: & From f41bbe6c267464ad26231c709fe7b8f283d7f1f1 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Mon, 13 Dec 2021 16:37:07 -0800 Subject: [PATCH 217/301] Cleanup - Removes commented out codes --- components/eam/bld/build-namelist | 4 ---- 1 file changed, 4 deletions(-) diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index 939aa61d102d..2c62ea028881 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -922,8 +922,6 @@ if ($co2_cycle) { # (user specification has higher precedence than the true value set above) if ($nl->get_value('co2_readflux_fuel') =~ /$TRUE/io) { add_default($nl, 'co2flux_fuel_file', 'sim_year'=>$sim_year); - # remove the line below for testing - # add_default($nl, 'co2flux_fuel_vrt_emis_file', 'sim_year'=>$sim_year); } #change the default value from '.true.' to '.false.' @@ -942,8 +940,6 @@ if ($co2_cycle) { # This should be the same file as the one in the aircraft_specifier file. # This is a workaround to get this filepath into the cam.input_data_list file # to allow the CESM scripts to obtain all required data for a run. - # remove the line below for the testing - # add_default($nl, 'aircraft_co2_file'); } } } From d791860450c0366bd55752a72d4f94ccec513315 Mon Sep 17 00:00:00 2001 From: Stephen Price Date: Mon, 13 Dec 2021 19:23:23 -0800 Subject: [PATCH 218/301] Update support for new Greenland 20km input file (now has muFriction and temperature interpolated from 1to10km input file). --- components/mpas-albany-landice/cime_config/buildnml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 58c2e2554e03..7000cb4c82e4 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -63,8 +63,8 @@ def buildnml(case, caseroot, compname): decomp_date += '20190326' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.gis20km': - grid_date += '20211119' - grid_prefix += 'gis20km' + grid_date += '20210824' + grid_prefix += 'gis_20km_r01' decomp_date += '150922' decomp_prefix += 'mpasli.graph.info.' elif glc_grid == 'mpas.ais20km': From 2e14457c12527e27ea5786676a462f57dfb15b58 Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 15 Dec 2021 10:37:45 -0800 Subject: [PATCH 219/301] After recent PR, correct PE layout for "S" size of ne102pg2 F cases using CICE [bfb] --- cime_config/allactive/config_pesall.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index d86211bc33de..5bd8e31f6b6c 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -6801,7 +6801,7 @@ 5440 64 64 - 85 + 5440 4 From cc97cda5a7cccabbd8aac11c71cd0d398b8f1a81 Mon Sep 17 00:00:00 2001 From: dqwu Date: Wed, 15 Dec 2021 15:41:45 -0600 Subject: [PATCH 220/301] Downgrading NetCDF lib from 4.8.1 to 4.8.0 for ANL GCE nodes It has been confirmed that some E3SM cases run on ANL GCE nodes with netcdf4p type and latest NetCDF 4.8.1 might return NC_EHDFERR from nc_enddef(). An issue has been created for NetCDF developers to fix. Before this issue is fixed in newer releases of NetCDF, we have to switch back to NetCDF 4.8.0, an older but more stable release. For consistency on GCE nodes, we downgrade the serial version of the NetCDF lib to 4.8.0 as well. --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2f5ee5f4d5cc..bd81f8649849 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1114,7 +1114,7 @@ $CIME_OUTPUT_ROOT/$CASE/bld - /nfs/gce/projects/climate/software/netcdf/4.8.1c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0 + /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0 @@ -1122,7 +1122,7 @@ /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/bin:$ENV{PATH} /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo /nfs/gce/projects/climate/software/hdf5/1.12.1/mpich-3.4.2/gcc-11.1.0 - /nfs/gce/projects/climate/software/netcdf/4.8.1c-4.3.1cxx-4.5.3f-parallel/mpich-3.4.2/gcc-11.1.0 + /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/pnetcdf/1.12.2/mpich-3.4.2/gcc-11.1.0 From e778eaf0dc745e599138d20b023b8d776bfc6e63 Mon Sep 17 00:00:00 2001 From: wadeburgess Date: Thu, 16 Dec 2021 08:20:18 -0700 Subject: [PATCH 221/301] Edit sandiatoss3 to change 'gpfs1' to 'gpfs' --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2f5ee5f4d5cc..5102448e2351 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1143,7 +1143,7 @@ fy210162 /projects/ccsm/timings .* - /gpfs1/$USER/acme_scratch/sandiatoss3 + /gpfs/$USER/acme_scratch/sandiatoss3 /projects/ccsm/inputdata /projects/ccsm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE From 657096ecadaa25f1602effaa0f5da2f3723061f6 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Thu, 16 Dec 2021 18:52:32 -0600 Subject: [PATCH 222/301] more logic around hard adjustment --- .../homme/src/theta-l_kokkos/cxx/CamForcing.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 3e1a3f42cbc1..1b59d4805f88 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -23,15 +23,19 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, //original // ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, p.moisture); - bool adjustment; + bool adjustment = false; + if ( p.ftype == ForcingAlg::FORCING_0) adjustment = false; -#ifndef CAM -//this will change when ftype2 in homme changes + +//standalone homme and ftype2 +#if !defined(CAM) && !defined(SCREAM) if ( p.ftype == ForcingAlg::FORCING_2) adjustment = false; #endif -#ifdef CAM +//CAM or SCREAM and ftype2 +#if defined(CAM) || defined(SCREAM) if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; #endif + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); GPTLstop("ApplyCAMForcing_tracers"); From a445407af8caf371a2b46878fa2609879d0a5b9a Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 17 Dec 2021 11:31:44 -0600 Subject: [PATCH 223/301] anvil/chrysalis threading and cprnc fix fixes to standalone HOMME: correct cprnc paths for anvil, chrysalis add cores-per-task srun option (otherwise HOMME threading tests have terrible performance) --- components/homme/cmake/machineFiles/anvil-bfb.cmake | 2 +- components/homme/cmake/machineFiles/anvil.cmake | 3 ++- components/homme/cmake/machineFiles/chrysalis-bfb.cmake | 2 +- components/homme/cmake/machineFiles/chrysalis.cmake | 2 +- components/homme/test/reg_test/run_tests/testing-utils.sh | 3 +++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/components/homme/cmake/machineFiles/anvil-bfb.cmake b/components/homme/cmake/machineFiles/anvil-bfb.cmake index d9d0328d9295..d46774646eb1 100644 --- a/components/homme/cmake/machineFiles/anvil-bfb.cmake +++ b/components/homme/cmake/machineFiles/anvil-bfb.cmake @@ -62,5 +62,5 @@ SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") SET (USE_QUEUING FALSE CACHE BOOL "") # for standalone HOMME builds: -SET (CPRNC_DIR /lcrc/group/e3sm/tools/cprnc CACHE FILEPATH "") +SET (CPRNC_DIR /lcrc/group/e3sm/soft/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/anvil.cmake b/components/homme/cmake/machineFiles/anvil.cmake index e03fb4c5b435..325a9caa280f 100644 --- a/components/homme/cmake/machineFiles/anvil.cmake +++ b/components/homme/cmake/machineFiles/anvil.cmake @@ -56,5 +56,6 @@ SET (USE_MPIEXEC "srun" CACHE STRING "") SET (USE_MPI_OPTIONS "-K --cpu_bind=cores" CACHE STRING "") SET (USE_QUEUING FALSE CACHE BOOL "") # for standalone HOMME builds: -SET (CPRNC_DIR /lcrc/group/e3sm/tools/cprnc CACHE FILEPATH "") +SET (CPRNC_DIR /lcrc/group/e3sm/soft/tools/cprnc CACHE FILEPATH "") + diff --git a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake index abbb1dadd9dc..ddeb7fa78111 100644 --- a/components/homme/cmake/machineFiles/chrysalis-bfb.cmake +++ b/components/homme/cmake/machineFiles/chrysalis-bfb.cmake @@ -40,7 +40,7 @@ SET (NetCDF_C_PATH "${NCCONFIG_OUTPUT}" CACHE STRING "") SET (USE_QUEUING FALSE CACHE BOOL "") # for standalone HOMME builds: -SET (CPRNC_DIR /lcrc/group/e3sm/soft/tools/cprnc CACHE FILEPATH "") +SET (CPRNC_DIR /lcrc/group/e3sm/tools/cprnc CACHE FILEPATH "") IF (${IFORT_RESULT} EQUAL 0) SET (HOMME_USE_MKL "TRUE" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/chrysalis.cmake b/components/homme/cmake/machineFiles/chrysalis.cmake index 12d6684e2e3a..a336809f6d33 100644 --- a/components/homme/cmake/machineFiles/chrysalis.cmake +++ b/components/homme/cmake/machineFiles/chrysalis.cmake @@ -40,7 +40,7 @@ SET (NetCDF_C_PATH "${NCCONFIG_OUTPUT}" CACHE STRING "") SET (USE_QUEUING FALSE CACHE BOOL "") # for standalone HOMME builds: -SET (CPRNC_DIR /lcrc/group/e3sm/soft/tools/cprnc CACHE FILEPATH "") +SET (CPRNC_DIR /lcrc/group/e3sm/tools/cprnc CACHE FILEPATH "") IF (${IFORT_RESULT} EQUAL 0) SET (HOMME_USE_MKL "TRUE" CACHE FILEPATH "") diff --git a/components/homme/test/reg_test/run_tests/testing-utils.sh b/components/homme/test/reg_test/run_tests/testing-utils.sh index b111caaae89b..62cc2f261ea6 100644 --- a/components/homme/test/reg_test/run_tests/testing-utils.sh +++ b/components/homme/test/reg_test/run_tests/testing-utils.sh @@ -485,6 +485,9 @@ createAllRunScriptsGeneric() { if [ -n "${OMP_NUM_TESTS}" -a "${RUN_OPENMP}" = ON ]; then echo "export OMP_NUM_THREADS=${OMP_NUMBER_THREADS}" >> $thisRunScript + if [ "${MPI_EXEC}" = "srun" ] ; then + echo "export SLURM_CPUS_PER_TASK=${OMP_NUMBER_THREADS}" >> $thisRunScript + fi echo "export OMP_STACKSIZE=128M" >> $thisRunScript echo "" >> $thisRunScript # new line for testNum in $(seq 1 ${OMP_NUM_TESTS}) From 8ab5faf299571ac919bf974e9869366b9644ecef Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 17 Dec 2021 10:22:35 -0800 Subject: [PATCH 224/301] updating to latest api 17 fates param file --- components/elm/bld/namelist_files/namelist_defaults.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 1291d3fdf855..6e75f5f97d6f 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -122,7 +122,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.16.1.0_12pft_c210630.nc +lnd/clm2/paramdata/fates_params_12pft_api17.c211116.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc From 9c0c5b366368a8e147aefb20855aa9714a5d6a8d Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 17 Dec 2021 10:24:53 -0800 Subject: [PATCH 225/301] adding endrun call after lai check --- components/elm/src/main/elmfates_interfaceMod.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index 20888ea72491..6dd7ad90c1d0 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -1165,7 +1165,8 @@ subroutine wrap_update_hlmfates_dyn(this, nc, bounds_clump, & if(use_fates_sp.and.abs(canopystate_inst%tlai_hist_patch(p) - & this%fates(nc)%bc_out(s)%tlai_pa(ifp)).gt.1e-09)then - write(iulog,*) 'fates lai not like hlm lai',tlai(p),this%fates(nc)%bc_out(s)%tlai_pa(ifp),ifp + write(iulog,*) 'fates lai not like hlm lai',tlai(p),this%fates(nc)%bc_out(s)%tlai_pa(ifp),ifp + call endrun(msg=errMsg(sourcefile, __LINE__)) endif frac_veg_nosno_alb(p) = this%fates(nc)%bc_out(s)%frac_veg_nosno_alb_pa(ifp) From d3a4a5c6c77a302b0fa32f98b36b7a782e157542 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 17 Dec 2021 17:06:58 -0800 Subject: [PATCH 226/301] sum the weights for nat veg across all topounits for a given gridcell --- .../elm/src/main/elmfates_interfaceMod.F90 | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index 6dd7ad90c1d0..b6700222fe84 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -498,8 +498,7 @@ subroutine init(this, bounds_proc ) use FatesParameterDerivedMod, only : param_derived use FatesInterfaceTypesMod, only : numpft_fates => numpft use elm_varsur, only : wt_nat_patch - - + use topounit_varcon , only: max_topounits, has_topounit implicit none @@ -511,6 +510,7 @@ subroutine init(this, bounds_proc ) integer :: nclumps ! Number of threads integer :: nc ! thread index integer :: s ! FATES site index + integer :: t ! topounit index (HLM) integer :: c ! HLM column index integer :: l ! HLM LU index integer :: g ! HLM grid index @@ -521,6 +521,7 @@ subroutine init(this, bounds_proc ) type(bounds_type) :: bounds_clump integer :: nmaxcol integer :: ndecomp + real(r8) :: wt_nat_patch_toposum ! Initialize the FATES communicators with the HLM ! This involves to stages @@ -637,6 +638,8 @@ subroutine init(this, bounds_proc ) ! --------------------------------------------------------------------------- g = col_pp%gridcell(c) + t = col_pp%topounit(c) + this%fates(nc)%sites(s)%lat = grc_pp%latdeg(g) this%fates(nc)%sites(s)%lon = grc_pp%londeg(g) @@ -644,7 +647,16 @@ subroutine init(this, bounds_proc ) this%fates(nc)%bc_in(s)%pft_areafrac(:)=0._r8 do m = natpft_lb,natpft_ub ft = m-natpft_lb - this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch(g,m) + + ! For now, sum the weights along all topounits for a given gridcell + wt_nat_patch_toposum = sum(wt_nat_patch(g,:,m)) + this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch_toposum + + !this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch(g,t,m) + + !write(iulog,*) 'elmfates: has_topounit: ',has_topounit + !write(iulog,*) 'elmfates: wt_nat_patch(g,1,m): ', wt_nat_patch(g,1,m) + !write(iulog,*) 'elmfates: sum wt_nat_patch: ', wt_nat_patch_toposum end do if(abs(sum(this%fates(nc)%bc_in(s)%pft_areafrac(natpft_lb:natpft_ub))-1.0_r8).gt.1.0e-9)then From 89ef6be670728ddc885613d6a66b6f713d3bb343 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 18 Dec 2021 12:34:11 -0800 Subject: [PATCH 227/301] thread bindings for knl and haswell, update cprnc path --- components/homme/cmake/machineFiles/cori-haswell.cmake | 5 +++-- components/homme/cmake/machineFiles/cori-knl.cmake | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/homme/cmake/machineFiles/cori-haswell.cmake b/components/homme/cmake/machineFiles/cori-haswell.cmake index a9c679d4401d..23be17dd2ed3 100644 --- a/components/homme/cmake/machineFiles/cori-haswell.cmake +++ b/components/homme/cmake/machineFiles/cori-haswell.cmake @@ -45,5 +45,6 @@ SET (USE_QUEUING FALSE CACHE BOOL "") SET (USE_MPIEXEC "srun" CACHE STRING "") -# temporary fix: -SET (USE_MPI_OPTIONS "-c 2 --cpu_bind=cores" CACHE STRING "") +SET (USE_MPI_OPTIONS "--cpu_bind=cores" CACHE STRING "") + +SET (CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/cori-knl.cmake b/components/homme/cmake/machineFiles/cori-knl.cmake index 301dd8955e93..c1204b5be01b 100644 --- a/components/homme/cmake/machineFiles/cori-knl.cmake +++ b/components/homme/cmake/machineFiles/cori-knl.cmake @@ -23,10 +23,9 @@ SET (USE_QUEUING FALSE CACHE BOOL "") SET (USE_MPIEXEC "srun" CACHE STRING "") -# temporary fix: -SET (USE_MPI_OPTIONS "-c 4 --cpu_bind=cores" CACHE STRING "") +SET (USE_MPI_OPTIONS "--cpu_bind=cores" CACHE STRING "") -SET (CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") +SET (CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc CACHE FILEPATH "") # by default, cori env loads haswell mod, do # module unload craype-haswell ; module load craype-mic-knl From fa890d1fcee91896585cdb3277b65b93e3cbbb5d Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 19 Dec 2021 12:23:18 -0800 Subject: [PATCH 228/301] tweaks to pass MPI arguments to unit test --- .../homme/cmake/machineFiles/compy.cmake | 21 +------------------ .../homme/test/unit_tests/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 21 deletions(-) mode change 100644 => 120000 components/homme/cmake/machineFiles/compy.cmake diff --git a/components/homme/cmake/machineFiles/compy.cmake b/components/homme/cmake/machineFiles/compy.cmake deleted file mode 100644 index 74f2c15de9a9..000000000000 --- a/components/homme/cmake/machineFiles/compy.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# O. Guba 08/2019 -#module purge - -SET(CMAKE_C_COMPILER "mpipgcc" CACHE STRING "") -SET(CMAKE_CXX_COMPILER "mpipgcxx" CACHE STRING "") -SET(CMAKE_Fortran_COMPILER "mpipgf90" CACHE STRING "") - -SET (WITH_PNETCDF FALSE CACHE FILEPATH "") - -SET (NETCDF_DIR "$ENV{NETCDF_ROOT}" CACHE STRING "") - -#is there a better way? without this linker does not see C netcdf library -SET (ADD_LINKER_FLAGS "-L/$ENV{MKLROOT}/lib/intel64 -L/$ENV{NETCDF_ROOT}/lib -lnetcdff -lnetcdf -lmkl_rt" CACHE STRING "") -SET (ADD_Fortran_FLAGS "-traceback" CACHE STRING "") - -SET (USE_MPIEXEC "srun" CACHE STRING "") -SET (USE_MPI_OPTIONS "--mpi=pmi2 --kill-on-bad-exit --cpu_bind=cores" CACHE STRING "") -SET (USE_QUEUING FALSE CACHE BOOL "") -# for standalone HOMME builds: -#SET (CPRNC_DIR /lcrc/group/acme/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/compy.cmake b/components/homme/cmake/machineFiles/compy.cmake new file mode 120000 index 000000000000..5d0d1a1a6dff --- /dev/null +++ b/components/homme/cmake/machineFiles/compy.cmake @@ -0,0 +1 @@ +compy-intel.cmake \ No newline at end of file diff --git a/components/homme/test/unit_tests/CMakeLists.txt b/components/homme/test/unit_tests/CMakeLists.txt index 7e67526c42c9..fb6060770859 100644 --- a/components/homme/test/unit_tests/CMakeLists.txt +++ b/components/homme/test/unit_tests/CMakeLists.txt @@ -9,7 +9,9 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con #IF (${NUM_CPUS} EQUAL 1) # ADD_TEST(${target_name}_test ${target_name}) #ELSE() - ADD_TEST(${target_name}_test ${USE_MPIEXEC} -n ${NUM_CPUS} ${MPI_OPTIONS} "./${target_name}") + set(TMP ${USE_MPI_OPTIONS}) + separate_arguments(TMP) + ADD_TEST(${target_name}_test ${USE_MPIEXEC} -n ${NUM_CPUS} ${TMP} ./${target_name}) #ENDIF() SET_TESTS_PROPERTIES(${target_name}_test PROPERTIES LABELS "unit") From 9ffdb63bdd4278ace02353a911bf323d10e33918 Mon Sep 17 00:00:00 2001 From: Wuyin Lin Date: Mon, 20 Dec 2021 16:12:04 -0800 Subject: [PATCH 229/301] New algorithm for determining tropopause and additional diagnostic output Add a new algorithm for determining tropopause: cold point parabolic. Output at additional levels in the stratosphere are also added. While the new algorithm is enabled during runtime, no additional variables are saved by default. [BFB] --- .../eam/src/physics/cam/cam_diagnostics.F90 | 69 +++-- components/eam/src/physics/cam/tropopause.F90 | 256 +++++++++++++++++- 2 files changed, 305 insertions(+), 20 deletions(-) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index cd0423ce2f49..60fb12d89d98 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -218,6 +218,7 @@ subroutine diag_init() call addfld ('Z200',horiz_only, 'A','m','Geopotential Z at 200 mbar pressure surface') call addfld ('Z100',horiz_only, 'A','m','Geopotential Z at 100 mbar pressure surface') call addfld ('Z050',horiz_only, 'A','m','Geopotential Z at 50 mbar pressure surface') + call addfld ('Z010',horiz_only, 'A','m','Geopotential Z at 10 mbar pressure surface') call addfld ('ZZ',(/ 'lev' /), 'A','m2','Eddy height variance' ) call addfld ('VZ',(/ 'lev' /), 'A','m2/s','Meridional transport of geopotential energy') @@ -301,6 +302,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') call addfld ('TUH',horiz_only, 'A','W/m', 'Total (vertically integrated) zonal MSE flux') @@ -352,6 +354,8 @@ subroutine diag_init() call addfld ('V250',horiz_only, 'A','m/s','Meridional wind at 250 mbar pressure surface') call addfld ('V200',horiz_only, 'A','m/s','Meridional wind at 200 mbar pressure surface') call addfld ('V100',horiz_only, 'A','m/s','Meridional wind at 100 mbar pressure surface') + call addfld ('V050',horiz_only, 'A','m/s','Meridional wind at 50 mbar pressure surface') + call addfld ('V010',horiz_only, 'A','m/s','Meridional wind at 10 mbar pressure surface') call addfld ('TT',(/ 'lev' /), 'A','K2','Eddy temperature variance' ) @@ -384,6 +388,8 @@ subroutine diag_init() call addfld ('Q400',horiz_only, 'A','kg/kg','Specific Humidity at 400 mbar pressure surface') call addfld ('Q300',horiz_only, 'A','kg/kg','Specific Humidity at 300 mbar pressure surface') call addfld ('Q100',horiz_only, 'A','kg/kg','Specific Humidity at 100 mbar pressure surface') + call addfld ('Q050',horiz_only, 'A','kg/kg','Specific Humidity at 050 mbar pressure surface') + call addfld ('Q010',horiz_only, 'A','kg/kg','Specific Humidity at 010 mbar pressure surface') call addfld ('T7001000',horiz_only, 'A','K','Temperature difference 700 mb - 1000 mb') call addfld ('TH7001000',horiz_only, 'A','K','Theta difference 700 mb - 1000 mb') @@ -979,6 +985,8 @@ subroutine diag_phys_writeout(state, psl) real(r8) ftem(pcols,pver) ! temporary workspace real(r8) ftem1(pcols,pver) ! another temporary workspace real(r8) ftem2(pcols,pver) ! another temporary workspace + real(r8) ftem4(pcols,pver) ! another temporary workspace + real(r8) ftem5(pcols,pver) ! another temporary workspace real(r8) psl_tmp(pcols) ! Sea Level Pressure real(r8) z3(pcols,pver) ! geo-potential height real(r8) p_surf(pcols) ! data interpolated to a pressure surface @@ -1107,6 +1115,10 @@ subroutine diag_phys_writeout(state, psl) call vertinterp(ncol, pcols, pver, state%pmid, 5000._r8, z3, p_surf) call outfld('Z050 ', p_surf, pcols, lchnk) end if + if (hist_fld_active('Z010')) then + call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, z3, p_surf) + call outfld('Z010 ', p_surf, pcols, lchnk) + end if ! ! Quadratic height fiels Z3*Z3 ! @@ -1314,19 +1326,20 @@ subroutine diag_phys_writeout(state, psl) end do call outfld ('TMQ ',ftem, pcols ,lchnk ) ! -! Mass of vertically integrated q flux +! Mass of vertically integrated water vapor flux ! - ftem(:ncol,:) = state%u(:ncol,:)*state%q(:ncol,:,1)*state%pdel(:ncol,:)*rga + ftem4(:ncol,:) = state%u(:ncol,:)*state%q(:ncol,:,1)*state%pdel(:ncol,:)*rga + ftem5(:ncol,:) = state%v(:ncol,:)*state%q(:ncol,:,1)*state%pdel(:ncol,:)*rga do k=2,pver - ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + ftem4(:ncol,1) = ftem4(:ncol,1) + ftem4(:ncol,k) + ftem5(:ncol,1) = ftem5(:ncol,1) + ftem5(:ncol,k) end do - call outfld ('TUQ ',ftem, pcols ,lchnk ) - ftem(:ncol,:) = state%v(:ncol,:)*state%q(:ncol,:,1)*state%pdel(:ncol,:)*rga - do k=2,pver - ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) - end do - call outfld ('TVQ ',ftem, pcols ,lchnk ) + ftem(:ncol,1) = sqrt( ftem4(:ncol,1)**2 + ftem5(:ncol,1)**2) + + call outfld ('TUQ ',ftem4, pcols ,lchnk ) + call outfld ('TVQ ',ftem5, pcols ,lchnk ) + call outfld ('TTQ ',ftem, pcols ,lchnk ) ! ! Mass of vertically integrated MSE flux @@ -1555,6 +1568,14 @@ subroutine diag_phys_writeout(state, psl) call vertinterp(ncol, pcols, pver, state%pmid, 10000._r8, state%q(1,1,1), p_surf) call outfld('Q100 ', p_surf, pcols, lchnk ) end if + if (hist_fld_active('Q050')) then + call vertinterp(ncol, pcols, pver, state%pmid, 5000._r8, state%q(1,1,1), p_surf) + call outfld('Q050 ', p_surf, pcols, lchnk ) + end if + if (hist_fld_active('Q010')) then + call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, state%q(1,1,1), p_surf) + call outfld('Q010 ', p_surf, pcols, lchnk ) + end if if (hist_fld_active('U1000')) then call vertinterp(ncol, pcols, pver, state%pmid, 100000._r8, state%u, p_surf) call outfld('U1000 ', p_surf, pcols, lchnk ) @@ -1615,6 +1636,10 @@ subroutine diag_phys_writeout(state, psl) call vertinterp(ncol, pcols, pver, state%pmid, 10000._r8, state%u, p_surf) call outfld('U100 ', p_surf, pcols, lchnk ) end if + if (hist_fld_active('U050')) then + call vertinterp(ncol, pcols, pver, state%pmid, 5000._r8, state%u, p_surf) + call outfld('U050 ', p_surf, pcols, lchnk ) + end if if (hist_fld_active('U010')) then call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, state%u, p_surf) call outfld('U010 ', p_surf, pcols, lchnk ) @@ -1675,6 +1700,19 @@ subroutine diag_phys_writeout(state, psl) call vertinterp(ncol, pcols, pver, state%pmid, 20000._r8, state%v, p_surf) call outfld('V200 ', p_surf, pcols, lchnk ) end if + if (hist_fld_active('V100')) then + call vertinterp(ncol, pcols, pver, state%pmid, 10000._r8, state%v, p_surf) + call outfld('V100 ', p_surf, pcols, lchnk ) + end if + if (hist_fld_active('V050')) then + call vertinterp(ncol, pcols, pver, state%pmid, 5000._r8, state%v, p_surf) + call outfld('V050 ', p_surf, pcols, lchnk ) + end if + if (hist_fld_active('V010')) then + call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, state%v, p_surf) + call outfld('V010 ', p_surf, pcols, lchnk ) + end if + if (hist_fld_active('U90M')) then call vertinterpz(ncol, pcols, pver, state%zm, 90._r8, state%u, p_surf) call outfld('U90M ', p_surf, pcols, lchnk ) @@ -1683,10 +1721,6 @@ subroutine diag_phys_writeout(state, psl) call vertinterpz(ncol, pcols, pver, state%zm, 90._r8, state%v, p_surf) call outfld('V90M ', p_surf, pcols, lchnk ) endif - if (hist_fld_active('V100')) then - call vertinterp(ncol, pcols, pver, state%pmid, 10000._r8, state%v, p_surf) - call outfld('V100 ', p_surf, pcols, lchnk ) - end if ftem(:ncol,:) = state%t(:ncol,:)*state%t(:ncol,:) call outfld('TT ',ftem ,pcols ,lchnk ) @@ -1836,11 +1870,6 @@ subroutine diag_phys_writeout(state, psl) call outfld('THE7001000 ', p_surf, pcols, lchnk ) end if - if (hist_fld_active('T010')) then - call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, state%t, p_surf) - call outfld('T010 ', p_surf, pcols, lchnk ) - end if - if (hist_fld_active('T250')) then call vertinterp(ncol, pcols, pver, state%pmid, 25000._r8, state%t, p_surf) call outfld('T250 ', p_surf, pcols, lchnk ) @@ -1857,6 +1886,10 @@ subroutine diag_phys_writeout(state, psl) call vertinterp(ncol, pcols, pver, state%pmid, 2500._r8, state%t, p_surf) call outfld('T025 ', p_surf, pcols, lchnk ) end if + if (hist_fld_active('T010')) then + call vertinterp(ncol, pcols, pver, state%pmid, 1000._r8, state%t, p_surf) + call outfld('T010 ', p_surf, pcols, lchnk ) + end if if (hist_fld_active('T005')) then call vertinterp(ncol, pcols, pver, state%pmid, 500._r8, state%t, p_surf) call outfld('T005 ', p_surf, pcols, lchnk ) diff --git a/components/eam/src/physics/cam/tropopause.F90 b/components/eam/src/physics/cam/tropopause.F90 index a9dd0d126a1a..95c5b489a412 100644 --- a/components/eam/src/physics/cam/tropopause.F90 +++ b/components/eam/src/physics/cam/tropopause.F90 @@ -21,6 +21,7 @@ module tropopause !--------------------------------------------------------------- use shr_kind_mod, only : r8 => shr_kind_r8 + use shr_const_mod, only : pi => shr_const_pi use ppgrid, only : pcols, pver, begchunk, endchunk use cam_abortutils, only : endrun use cam_logfile, only : iulog @@ -34,8 +35,11 @@ module tropopause private public :: tropopause_readnl, tropopause_init, tropopause_find, tropopause_output + public :: tropopause_findChemTrop public :: TROP_ALG_NONE, TROP_ALG_ANALYTIC, TROP_ALG_CLIMATE public :: TROP_ALG_STOBIE, TROP_ALG_HYBSTOB, TROP_ALG_TWMO, TROP_ALG_WMO + public :: TROP_ALG_CPP + public :: NOTFOUND save @@ -51,9 +55,10 @@ module tropopause integer, parameter :: TROP_ALG_TWMO = 5 ! WMO Definition, Reichler et al. [2003] integer, parameter :: TROP_ALG_WMO = 6 ! WMO Definition integer, parameter :: TROP_ALG_HYBSTOB = 7 ! Hybrid Stobie Algorithm + integer, parameter :: TROP_ALG_CPP = 8 ! Cold Point Parabolic - integer, parameter :: TROP_NALG = 7 ! Number of Algorithms - character,parameter :: TROP_LETTER(TROP_NALG) = (/ ' ', 'A', 'C', 'S', 'T', 'W', 'H' /) + integer, parameter :: TROP_NALG = 8 ! Number of Algorithms + character,parameter :: TROP_LETTER(TROP_NALG) = (/ ' ', 'A', 'C', 'S', 'T', 'W', 'H', 'F' /) ! unique identifier for output, don't use P ! These variables should probably be controlled by namelist entries. @@ -155,6 +160,13 @@ subroutine tropopause_init() call addfld('TROPP_DZ', (/ 'lev' /), 'A', 'm', 'Relalive Tropopause Height (primary)') call addfld('TROPP_PD', (/ 'lev' /), 'A', 'probability', 'Tropopause Distribution (primary)') call addfld('TROPP_FD', horiz_only, 'A', 'probability', 'Tropopause Found (primary)') + + call addfld('TROPF_P', horiz_only, 'A', 'Pa', 'Tropopause Pressure (cold point)', flag_xyfill=.True.) + call addfld('TROPF_T', horiz_only, 'A', 'K', 'Tropopause Temperature (cold point)', flag_xyfill=.True.) + call addfld('TROPF_Z', horiz_only, 'A', 'm', 'Tropopause Height (cold point)', flag_xyfill=.True.) + call addfld('TROPF_DZ', (/ 'lev' /), 'A', 'm', 'Relative Tropopause Height (cold point)', flag_xyfill=.True.) + call addfld('TROPF_PD', (/ 'lev' /), 'A', 'probability', 'Tropopause Distribution (cold point)') + call addfld('TROPF_FD', horiz_only, 'A', 'probability', 'Tropopause Found (cold point)') call addfld( 'hstobie_trop', (/ 'lev' /), 'I', 'fraction of model time', 'Lowest level with stratospheric chemsitry' ) call addfld( 'hstobie_linoz', (/ 'lev' /), 'I', 'fraction of model time', 'Lowest possible Linoz level' ) @@ -967,6 +979,124 @@ subroutine tropopause_wmo(pstate, tropLev, tropP, tropT, tropZ) return end subroutine tropopause_wmo + ! This routine searches for the cold point tropopause, and uses a parabolic + ! fit of the coldest point and two adjacent points to interpolate the cold point + ! between model levels. + subroutine tropopause_cpp(pstate, tropLev, tropP, tropT, tropZ) + + implicit none + + type(physics_state), intent(in) :: pstate + integer, intent(inout) :: tropLev(pcols) ! tropopause level index + real(r8), optional, intent(inout) :: tropP(pcols) ! tropopause pressure (Pa) + real(r8), optional, intent(inout) :: tropT(pcols) ! tropopause temperature (K) + real(r8), optional, intent(inout) :: tropZ(pcols) ! tropopause height (m) + + ! Local Variables + real(r8), parameter :: ztrop_low = 5000._r8 ! lowest tropopause level allowed (m) + real(r8), parameter :: ztrop_high = 25000._r8 ! highest tropopause level allowed (m) + + integer :: i + integer :: k, firstk, lastk + integer :: k2 + integer :: ncol ! number of columns in the chunk + integer :: lchnk ! chunk identifier + real(r8) :: tZ ! tropopause height (m) + real(r8) :: tmin + real(r8) :: f0, f1, f2 + real(r8) :: x0, x1, x2 + real(r8) :: c0, c1, c2 + real(r8) :: a, b, c + + ! Information about the chunk. + lchnk = pstate%lchnk + ncol = pstate%ncol + + ! Iterate over all of the columns. + do i = 1, ncol + + firstk = 0 + lastk = pver+1 + + ! Skip column in which the tropopause has already been found. + if (tropLev(i) == NOTFOUND) then + tmin = 1e6_r8 + + kloop: do k = pver-1, 2, -1 + + ! Skip levels below the minimum and stop if nothing is found + ! before the maximum. + if (pstate%zm(i, k) < ztrop_low) then + firstk = k + cycle kloop + else if (pstate%zm(i, k) > ztrop_high) then + lastk = k + exit kloop + end if + + ! Find the coldest point + if (pstate%t(i, k) < tmin) then + tropLev(i) = k + tmin = pstate%t(i,k) + end if + end do kloop + + ! If the minimum is at the edge of the search range, then don't + ! consider this to be a minima + if ((tropLev(i) >= (firstk-1)) .or. (tropLev(i) <= (lastk+1))) then + tropLev(i) = NOTFOUND + else + + ! If returning P, Z, or T, then do a parabolic fit using the + ! cold point and it its 2 surrounding points to interpolate + ! between model levels. + if (present(tropP) .or. present(tropZ) .or. present(tropT)) then + f0 = pstate%t(i, tropLev(i)-1) + f1 = pstate%t(i, tropLev(i)) + f2 = pstate%t(i, tropLev(i)+1) + + x0 = pstate%zm(i, tropLev(i)-1) + x1 = pstate%zm(i, tropLev(i)) + x2 = pstate%zm(i, tropLev(i)+1) + + c0 = (x0-x1)*(x0-x2) + c1 = (x1-x0)*(x1-x2) + c2 = (x2-x0)*(x2-x1) + + ! Determine the quadratic coefficients of: + ! T = a * z^2 - b*z + c + a = (f0/c0 + f1/c1 + f2/c2) + b = (f0/c0*(x1+x2) + f1/c1*(x0+x2) + f2/c2*(x0+x1)) + c = f0/c0*x1*x2 + f1/c1*x0*x2 + f2/c2*x0*x1 + + ! Find the altitude of the minimum temperature + tZ = 0.5_r8 * b / a + + ! The fit should be between the upper and lower points, + ! so skip the point if the fit fails. + if ((tZ >= x0) .or. (tZ <= x2)) then + tropLev(i) = NOTFOUND + else + ! Return the optional outputs + if (present(tropP)) then + tropP(i) = tropopause_interpolateP(pstate, i, tropLev(i), tZ) + end if + + if (present(tropT)) then + tropT(i) = a * tZ*tZ - b*tZ + c + end if + + if (present(tropZ)) then + tropZ(i) = tZ + end if + end if + end if + end if + end if + end do + + return + end subroutine tropopause_cpp ! Searches all the columns in the chunk and attempts to identify the tropopause. ! Two routines can be specifed, a primary routine which is tried first and a @@ -1021,6 +1151,62 @@ subroutine tropopause_find(pstate, tropLev, tropP, tropT, tropZ, primary, backup return end subroutine tropopause_find + ! Searches all the columns in the chunk and attempts to identify the "chemical" + ! tropopause. This is the lapse rate tropopause, backed up by the climatology + ! if the lapse rate fails to find the tropopause at pressures higher than a certain + ! threshold. This pressure threshold depends on latitude. Between 50S and 50N, + ! the climatology is used if the lapse rate tropopause is not found at P > 75 hPa. + ! At high latitude (poleward of 50), the threshold is increased to 125 hPa to + ! eliminate false events that are sometimes detected in the cold polar stratosphere. + ! + ! NOTE: This routine was adapted from code in chemistry.F90 and mo_gasphase_chemdr.F90. + subroutine tropopause_findChemTrop(pstate, tropLev, primary, backup) + + implicit none + + type(physics_state), intent(in) :: pstate + integer, optional, intent(in) :: primary ! primary detection algorithm + integer, optional, intent(in) :: backup ! backup detection algorithm + integer, intent(out) :: tropLev(pcols) ! tropopause level index + + ! Local Variable + real(r8), parameter :: rad2deg = 180._r8/pi ! radians to degrees conversion factor + real(r8) :: dlats(pcols) + integer :: i + integer :: ncol + integer :: backAlg + + ! First use the lapse rate tropopause. + ncol = pstate%ncol + call tropopause_find(pstate, tropLev, primary=primary, backup=TROP_ALG_NONE) + + ! Now check high latitudes (poleward of 50) and set the level to the + ! climatology if the level was not found or is at P <= 125 hPa. + dlats(:ncol) = pstate%lat(:ncol) * rad2deg ! convert to degrees + + if (present(backup)) then + backAlg = backup + else + backAlg = default_backup + end if + + do i = 1, ncol + if (abs(dlats(i)) > 50._r8) then + if (tropLev(i) .ne. NOTFOUND) then + if (pstate%pmid(i, tropLev(i)) <= 12500._r8) then + tropLev(i) = NOTFOUND + end if + end if + end if + end do + + ! Now use the backup algorithm + if ((backAlg /= TROP_ALG_NONE) .and. any(tropLev(:) == NOTFOUND)) then + call tropopause_findUsing(pstate, backAlg, tropLev) + end if + + return + end subroutine tropopause_findChemTrop ! Call the appropriate tropopause detection routine based upon the algorithm ! specifed. @@ -1058,6 +1244,9 @@ subroutine tropopause_findUsing(pstate, algorithm, tropLev, tropP, tropT, tropZ) case(TROP_ALG_WMO) call tropopause_wmo(pstate, tropLev, tropP, tropT, tropZ) + case(TROP_ALG_CPP) + call tropopause_cpp(pstate, tropLev, tropP, tropT, tropZ) + case default write(iulog, *) 'tropopause: Invalid detection algorithm (', algorithm, ') specified.' call endrun @@ -1066,6 +1255,48 @@ subroutine tropopause_findUsing(pstate, algorithm, tropLev, tropP, tropT, tropZ) return end subroutine tropopause_findUsing + ! This routine interpolates the pressures in the physics state to + ! find the pressure at the specified tropopause altitude. + function tropopause_interpolateP(pstate, icol, tropLev, tropZ) + + implicit none + + type(physics_state), intent(in) :: pstate + integer, intent(in) :: icol ! column being processed + integer, intent(in) :: tropLev ! tropopause level index + real(r8), optional, intent(in) :: tropZ ! tropopause pressure (m) + real(r8) :: tropopause_interpolateP + + ! Local Variables + real(r8) :: tropP ! tropopause pressure (Pa) + real(r8) :: dlogPdZ ! dlog(p)/dZ + + ! Interpolate the temperature linearly against log(P) + + ! Is the tropopause at the midpoint? + if (tropZ == pstate%zm(icol, tropLev)) then + tropP = pstate%pmid(icol, tropLev) + + else if (tropZ > pstate%zm(icol, tropLev)) then + + ! It is above the midpoint? Make sure we aren't at the top. + if (tropLev > 1) then + dlogPdZ = (log(pstate%pmid(icol, tropLev)) - log(pstate%pmid(icol, tropLev - 1))) / & + (pstate%zm(icol, tropLev) - pstate%zm(icol, tropLev - 1)) + tropP = pstate%pmid(icol, tropLev) + exp((tropZ - pstate%zm(icol, tropLev)) * dlogPdZ) + end if + else + + ! It is below the midpoint. Make sure we aren't at the bottom. + if (tropLev < pver) then + dlogPdZ = (log(pstate%pmid(icol, tropLev + 1)) - log(pstate%pmid(icol, tropLev))) / & + (pstate%zm(icol, tropLev + 1) - pstate%zm(icol, tropLev)) + tropP = pstate%pmid(icol, tropLev) + exp((tropZ - pstate%zm(icol, tropLev)) * dlogPdZ) + end if + end if + + tropopause_interpolateP = tropP + end function tropopause_interpolateP ! This routine interpolates the temperatures in the physics state to ! find the temperature at the specified tropopause pressure. @@ -1223,6 +1454,27 @@ subroutine tropopause_output(pstate) call outfld('TROPP_PD', tropPdf(:ncol, :), ncol, lchnk) call outfld('TROPP_FD', tropFound(:ncol), ncol, lchnk) + ! Find the tropopause using just the cold point algorithm. + call tropopause_find(pstate, tropLev, tropP=tropP, tropT=tropT, tropZ=tropZ, primary=TROP_ALG_CPP, backup=TROP_ALG_NONE) + + tropPdf(:,:) = 0._r8 + tropFound(:) = 0._r8 + tropDZ(:,:) = fillvalue + + do i = 1, ncol + if (tropLev(i) /= NOTFOUND) then + tropPdf(i, tropLev(i)) = 1._r8 + tropFound(i) = 1._r8 + tropDZ(i,:) = pstate%zm(i,:) - tropZ(i) + end if + end do + + call outfld('TROPF_P', tropP(:ncol), ncol, lchnk) + call outfld('TROPF_T', tropT(:ncol), ncol, lchnk) + call outfld('TROPF_Z', tropZ(:ncol), ncol, lchnk) + call outfld('TROPF_DZ', tropDZ(:ncol, :), ncol, lchnk) + call outfld('TROPF_PD', tropPdf(:ncol, :), ncol, lchnk) + call outfld('TROPF_FD', tropFound(:ncol), ncol, lchnk) ! If requested, do all of the algorithms. if (output_all) then From 5fd19d60b61d710e63da057f80d62720b582002f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 21 Dec 2021 18:05:17 -0700 Subject: [PATCH 230/301] Use lmod modules for mappy --- cime_config/machines/config_machines.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 8d30f7aefa1d..7a3598946692 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -860,12 +860,12 @@ - /usr/share/Modules/init/python.py - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/bin/modulecmd python - /usr/bin/modulecmd perl + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/env_modules_python.py + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/perl + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/sh + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/csh + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod python + /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod perl module module From 78b2f4db2603ca1d9b56eca1eeffaeff77ddbac9 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 21 Dec 2021 23:08:09 -0800 Subject: [PATCH 231/301] Reinstating p indexing accidentally removed This line was accidentally removed during an update --- components/elm/src/main/elmfates_interfaceMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index b6700222fe84..99d337f26399 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -814,7 +814,7 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst, & min(nlevsoil, canopystate_inst%altmax_lastyear_indx_col(c)) do ifp = 1, this%fates(nc)%sites(s)%youngest_patch%patchno - + p = ifp+col_pp%pfti(c) this%fates(nc)%bc_in(s)%t_veg24_pa(ifp) = & veg_es%t_veg24(p) From a3b22b99412a38fbbf597edde606727bd6f94229 Mon Sep 17 00:00:00 2001 From: mark-petersen Date: Wed, 22 Dec 2021 14:29:30 -0800 Subject: [PATCH 232/301] Update LANL scratch3 to scratch4 LANL /lustre/scratch3 will be decommisioned January 4, 2022. All data should now be written to /lustre/scratch4. --- cime_config/machines/config_machines.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7a3598946692..bdd1b54bcef7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2575,10 +2575,10 @@ openmpi,impi,mvapich climateacme /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/archive/$CASE - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc 4 e3sm_developer @@ -2655,10 +2655,10 @@ openmpi,impi,mvapich climateacme /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/archive/$CASE - /lustre/scratch3/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE + /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc 4 e3sm_developer From dd244ace5dd7069ce98b263d768f7a25f4dd0b70 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Thu, 23 Dec 2021 14:03:02 -0700 Subject: [PATCH 233/301] Make adjust_ps consistent with SCREAM Make setting of adjust_ps consistent with SCREAM repo. We want to allow adjust_ps=.false. for SCREAM, but retain adjust_ps=.true. for E3SM to remain BFB for v2. This change was originally made downstream in SCREAM, but causes conflicts between the two repos, and prevents EAM compsets from being BFB between the E3SM and SCREAM repos. Moving this change upstream, protected in an ifdef, allows the two repos to be consistent, and keeps EAM compsets BFB between the SCREAM and E3SM repos. This also sets the precedent for allowing changes upstream in E3SM that should apply only to SCREAM configurations, protected in ifdef SCREAM. To enable, we simply need to add the -cppdefs '-DSCREAM' flag to CAM_CONFIG_OPTS for SCREAM compsets. [BFB] --- components/homme/src/share/prim_driver_base.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index a05530aee54c..36c827abd481 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -1597,7 +1597,11 @@ subroutine applyCAMforcing_tracers(elem,hvcoord,np1,np1_qdp,dt,adjustment) if (dt_remap_factor==0) then adjust_ps=.true. ! stay on reference levels for Eulerian case else +#ifdef SCREAM + adjust_ps=.false. ! Lagrangian case can support adjusting dp3d or ps +#else adjust_ps=.true. ! Lagrangian case can support adjusting dp3d or ps +#endif endif #else adjust_ps=.true. ! preqx requires forcing to stay on reference levels From 962e16ec46fa4f01000d53f5ed3adaa8cb95c2d8 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Thu, 23 Dec 2021 15:41:45 -0800 Subject: [PATCH 234/301] adding fates satellite phenology test --- cime_config/tests.py | 1 + .../testmods_dirs/elm/fates_satphen/shell_commands | 3 +++ .../testmods_dirs/elm/fates_satphen/user_nl_elm | 13 +++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/shell_commands create mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/user_nl_elm diff --git a/cime_config/tests.py b/cime_config/tests.py index 5bef6fcc766a..0ee7cafb0459 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -24,6 +24,7 @@ "ERS_Ld20.f45_f45.IELMFATES.elm-fates", "SMS_Ld20.f45_f45.IELMFATES.elm-fates_rd", "SMS_Ld20.f45_f45.IELMFATES.elm-fates_eca", + "SMS_Ld30.f45_f45.IELMFATES.elm-fates_satphen", "ERS.f19_g16.I1850ELM.elm-betr", "ERS.f19_g16.I1850ELM.elm-vst", "ERS.f09_g16.I1850GSWCNPRDCTCBC.elm-vstrd", diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/shell_commands new file mode 100644 index 000000000000..a1d1444d7715 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/shell_commands @@ -0,0 +1,3 @@ +#!/bin/bash +if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi +./xmlchange NTHRDS=1 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/user_nl_elm new file mode 100644 index 000000000000..347ede09bfa4 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/user_nl_elm @@ -0,0 +1,13 @@ +use_cn = .false. +finidat = '' +hist_mfilt = 365 +hist_nhtfrq = -24 +hist_empty_htapes = .true. +fates_spitfire_mode = 0 +use_fates_fixed_biogeog = .true. +use_fates_nocomp = .true. +use_fates_sp = .true. +hist_fincl1 = 'NPP','GPP','BTRAN','H2OSOI','TLAI','LITTER_IN_ELEM','LITTER_OUT_ELEM', + 'FIRE_AREA','SCORCH_HEIGHT','FIRE_INTENSITY','FIRE_TFC_ROS','FIRE_FUEL_MEF', + 'FIRE_FUEL_BULKD','FIRE_FUEL_SAV','FIRE_NESTEROV_INDEX','PFTbiomass', + 'PFTleafbiomass','FIRE_ROS','WIND','AREA_TREES','AREA_PLANT','HR','AR','TSAI','HTOP' From c7a87082cda20f6fec3cfcde4d0a37b39204fe57 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 20 Apr 2021 07:45:28 -0600 Subject: [PATCH 235/301] dont try to turn on horiz/vert threads if no compiler support remove unecessary if statement in nu_div/nu logic remove unused namelist variable --- components/homme/src/share/namelist_mod.F90 | 10 ++++++++-- .../homme/src/theta-l/share/viscosity_theta.F90 | 12 ++---------- components/homme/test/reg_test/namelists/swtc2.nl | 1 - 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 2623adc27469..23f334a1dcd0 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -912,8 +912,14 @@ subroutine readnl(par) ! use maximum available: - if (NThreads == -1) NThreads = omp_get_max_threads() - + if (NThreads == -1) then +#if defined(HORIZ_OPENMP) || defined (COLUMN_OPENMP) + NThreads = omp_get_max_threads() +#else + NThreads = 1 +#endif + endif + ! sanity check on thread count ! HOMME will run if if nthreads > max, but gptl will print out GB of warnings. if (NThreads > omp_get_max_threads()) then diff --git a/components/homme/src/theta-l/share/viscosity_theta.F90 b/components/homme/src/theta-l/share/viscosity_theta.F90 index 227a34ebebcb..6e325b13af71 100644 --- a/components/homme/src/theta-l/share/viscosity_theta.F90 +++ b/components/homme/src/theta-l/share/viscosity_theta.F90 @@ -85,16 +85,8 @@ subroutine biharmonic_wk_theta(elem,stens,vtens,deriv,edgebuf,hybrid,nt,nets,net nu_ratio1=1 nu_ratio2=1 if (nu_div/=nu) then - if(hypervis_scaling /= 0) then - ! with the tensor, we cant seperate div and curl components. instead: - ! nu * (del V del ) * ( nu_ratio * grad(div) - curl(curl)) - nu_ratio1=(nu_div/nu) - nu_ratio2=1 - else - ! since operator is applied twice, take the sqrt - nu_ratio1=(nu_div/nu) - nu_ratio2=1 - endif + nu_ratio1=(nu_div/nu) + nu_ratio2=1 endif diff --git a/components/homme/test/reg_test/namelists/swtc2.nl b/components/homme/test/reg_test/namelists/swtc2.nl index 3dc92944bd6d..a6a7d55666c7 100644 --- a/components/homme/test/reg_test/namelists/swtc2.nl +++ b/components/homme/test/reg_test/namelists/swtc2.nl @@ -23,7 +23,6 @@ nu= 1e-9 nu_s= 1e-9 hypervis_order = 2 hypervis_subcycle = 2 -hypervis_power = 0 mesh_file = 'mountain_10_x2.g' hypervis_scaling = 3.2 / From 07546bd02eae0e16b1fa816ecdae124a21509b95 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 20 Apr 2021 08:02:56 -0600 Subject: [PATCH 236/301] remove hypervis_power restore correct var_coef behavoir remove hypervis power2 --- components/homme/src/preqx/derivative_mod.F90 | 2 +- .../homme/src/preqx_acc/derivative_mod.F90 | 14 +- .../homme/src/preqx_kokkos/derivative_mod.F90 | 2 +- components/homme/src/prim_movie_mod.F90 | 4 +- components/homme/src/share/control_mod.F90 | 15 +- .../homme/src/share/derivative_mod_base.F90 | 113 +--------------- .../homme/src/share/global_norms_mod.F90 | 128 +----------------- components/homme/src/share/namelist_mod.F90 | 22 +-- components/homme/src/share/sl_advection.F90 | 5 +- components/homme/src/sweqx/derivative_mod.F90 | 1 - .../src/theta-l/share/derivative_mod.F90 | 2 +- 11 files changed, 20 insertions(+), 288 deletions(-) diff --git a/components/homme/src/preqx/derivative_mod.F90 b/components/homme/src/preqx/derivative_mod.F90 index 6ceeca581a88..e01277904876 100644 --- a/components/homme/src/preqx/derivative_mod.F90 +++ b/components/homme/src/preqx/derivative_mod.F90 @@ -6,7 +6,7 @@ module derivative_mod use derivative_mod_base, only: derivative_t, subcell_integration, subcell_dss_fluxes, subcell_div_fluxes, & subcell_Laplace_fluxes, allocate_subcell_integration_matrix, derivinit, gradient, gradient_wk, vorticity, divergence, & gradient_sphere_wk_testcov, gradient_sphere_wk_testcontra, ugradv_sphere, vorticity_sphere, vorticity_sphere_diag, & - curl_sphere,curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, edge_flux_u_cg, & + curl_sphere,curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, & limiter_optim_iter_full, limiter_clip_and_sum, laplace_sphere_wk, divergence_sphere_wk, gradient_sphere, & divergence_sphere, laplace_z implicit none diff --git a/components/homme/src/preqx_acc/derivative_mod.F90 b/components/homme/src/preqx_acc/derivative_mod.F90 index 05ba15e4427d..223e9be7a256 100644 --- a/components/homme/src/preqx_acc/derivative_mod.F90 +++ b/components/homme/src/preqx_acc/derivative_mod.F90 @@ -7,7 +7,7 @@ module derivative_mod use derivative_mod_base, only: derivative_t, subcell_integration, subcell_dss_fluxes, subcell_div_fluxes, subcell_Laplace_fluxes, allocate_subcell_integration_matrix, & derivinit, & gradient_sphere_wk_testcov, gradient_sphere_wk_testcontra, ugradv_sphere, vorticity_sphere, vorticity_sphere_diag, curl_sphere, & - curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, edge_flux_u_cg, limiter_optim_iter_full, & + curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, limiter_optim_iter_full, & limiter_clip_and_sum, & laplace_sphere_wk, divergence_sphere_wk, gradient_sphere, divergence_sphere, laplace_z use kinds, only : real_kind, longdouble_kind @@ -17,14 +17,14 @@ module derivative_mod ! needed for spherical differential operators: use physical_constants, only : rrearth use element_mod, only : element_t - use control_mod, only : hypervis_scaling, hypervis_power + use control_mod, only : hypervis_scaling implicit none private public :: derivative_t, subcell_integration, subcell_dss_fluxes, subcell_div_fluxes, subcell_Laplace_fluxes, allocate_subcell_integration_matrix, & derivinit, & gradient_sphere_wk_testcov, gradient_sphere_wk_testcontra, ugradv_sphere, vorticity_sphere, vorticity_sphere_diag, curl_sphere, & - curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, edge_flux_u_cg, limiter_optim_iter_full, & + curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, limiter_optim_iter_full, & limiter_clip_and_sum, & laplace_sphere_wk, divergence_sphere_wk, gradient_sphere, divergence_sphere, laplace_z public :: laplace_sphere_wk_openacc @@ -36,7 +36,7 @@ module derivative_mod subroutine laplace_sphere_wk_openacc(s,grads,deriv,elem,var_coef,laplace,len,nets,nete,ntl,tl) use element_mod, only: element_t - use control_mod, only: hypervis_scaling, hypervis_power + use control_mod, only: hypervis_scaling implicit none !input: s = scalar !ouput: -< grad(PHI), grad(s) > = weak divergence of grad(s) @@ -58,11 +58,7 @@ subroutine laplace_sphere_wk_openacc(s,grads,deriv,elem,var_coef,laplace,len,net do j = 1 , np do i = 1 , np if (var_coef) then - if (hypervis_power/=0 ) then - ! scalar viscosity with variable coefficient - grads(i,j,1,k,ie) = grads(i,j,1,k,ie)*elem(ie)%variable_hyperviscosity(i,j) - grads(i,j,2,k,ie) = grads(i,j,2,k,ie)*elem(ie)%variable_hyperviscosity(i,j) - else if (hypervis_scaling /=0 ) then + if (hypervis_scaling /=0 ) then oldgrads = grads(i,j,:,k,ie) grads(i,j,1,k,ie) = sum(oldgrads(:)*elem(ie)%tensorVisc(i,j,1,:)) grads(i,j,2,k,ie) = sum(oldgrads(:)*elem(ie)%tensorVisc(i,j,2,:)) diff --git a/components/homme/src/preqx_kokkos/derivative_mod.F90 b/components/homme/src/preqx_kokkos/derivative_mod.F90 index 6ceeca581a88..e01277904876 100644 --- a/components/homme/src/preqx_kokkos/derivative_mod.F90 +++ b/components/homme/src/preqx_kokkos/derivative_mod.F90 @@ -6,7 +6,7 @@ module derivative_mod use derivative_mod_base, only: derivative_t, subcell_integration, subcell_dss_fluxes, subcell_div_fluxes, & subcell_Laplace_fluxes, allocate_subcell_integration_matrix, derivinit, gradient, gradient_wk, vorticity, divergence, & gradient_sphere_wk_testcov, gradient_sphere_wk_testcontra, ugradv_sphere, vorticity_sphere, vorticity_sphere_diag, & - curl_sphere,curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, edge_flux_u_cg, & + curl_sphere,curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, & limiter_optim_iter_full, limiter_clip_and_sum, laplace_sphere_wk, divergence_sphere_wk, gradient_sphere, & divergence_sphere, laplace_z implicit none diff --git a/components/homme/src/prim_movie_mod.F90 b/components/homme/src/prim_movie_mod.F90 index 7e5c12508f6e..34279bbfb1dd 100644 --- a/components/homme/src/prim_movie_mod.F90 +++ b/components/homme/src/prim_movie_mod.F90 @@ -21,7 +21,7 @@ module prim_movie_mod use cube_mod, only : cube_assemble use control_mod, only : test_case, runtype, geometry, & restartfreq, & - integration, hypervis_power, qsplit + integration, qsplit use common_io_mod, only : & output_start_time, & output_end_time, & @@ -547,8 +547,6 @@ subroutine prim_movie_output(elem, tl, hvcoord, par) st=1 do ie=1,nelemd vartmp(:,:,1) = elem(ie)%variable_hyperviscosity(:,:) - ! scale back to a length scale - if (hypervis_power /= 0 ) vartmp(:,:,1)=vartmp(:,:,1)**(2d0/hypervis_power) en=st+elem(ie)%idxp%NumUniquePts-1 call UniquePoints(elem(ie)%idxP,vartmp(:,:,1),var2d(st:en)) st=en+1 diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 03bbd31f1fd3..ed86327db6f2 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -171,23 +171,16 @@ module control_mod integer, public :: psurf_vis = 0 ! 0 = use laplace on eta surfaces ! 1 = use (approx.) laplace on p surfaces - real (kind=real_kind), public :: hypervis_power=0 ! if not 0, use variable hyperviscosity based on element area real (kind=real_kind), public :: hypervis_scaling=0 ! use tensor hyperviscosity !three types of hyper viscosity are supported right now: ! (1) const hv: nu * del^2 del^2 - ! (2) scalar hv: nu(lat,lon) * del^2 del^2 - ! (3) tensor hv, nu * ( \div * tensor * \grad ) * del^2 + ! (2) tensor hv, nu * ( \div * tensor * \grad ) * del^2 ! - ! (1) default: hypervis_power=0, hypervis_scaling=0 - ! (2) Original version for var-res grids. (M. Levy) - ! scalar coefficient within each element - ! hypervisc_scaling=0 - ! set hypervis_power>0 and set fine_ne, max_hypervis_courant - ! (3) tensor HV var-res grids + ! (1) hypervis_scaling=0 + ! (2) tensor HV var-res grids ! tensor within each element: - ! set hypervis_scaling > 0 (typical values would be 3.2 or 4.0) - ! hypervis_power=0 + ! set hypervis_scaling > 0 (typical values would be 3.0) ! (\div * tensor * \grad) operator uses cartesian laplace ! diff --git a/components/homme/src/share/derivative_mod_base.F90 b/components/homme/src/share/derivative_mod_base.F90 index e6f795e4869a..314081a58d56 100644 --- a/components/homme/src/share/derivative_mod_base.F90 +++ b/components/homme/src/share/derivative_mod_base.F90 @@ -10,7 +10,7 @@ module derivative_mod_base use quadrature_mod, only : quadrature_t, gauss, gausslobatto,legendre, jacobi use parallel_mod, only : abortmp use element_mod, only : element_t - use control_mod, only : hypervis_scaling, hypervis_power + use control_mod, only : hypervis_scaling use physical_constants, only : scale_factor_inv, laplacian_rigid_factor implicit none @@ -84,7 +84,6 @@ module derivative_mod_base ! public :: laplace_eta public :: laplace_z public :: element_boundary_integral - public :: edge_flux_u_cg public :: limiter_optim_iter_full public :: limiter_clip_and_sum @@ -911,102 +910,6 @@ end function element_boundary_integral - function edge_flux_u_cg( v,p,pedges, deriv, elem, u_is_contra) result(result) -! -! -! input: v = velocity in contra or lat-lon coordinates (CONTINUIOUS) -! p = scalar on this element -! pedges = scalar edge data from neighbor elements -! -! ouput: result(i,j) = contour integral of PHI_ij * pstar * v dot normal -! where PHI_ij = cardinal function at i,j GLL point -! pstar = centered or other flux -! - real(kind=real_kind), intent(in) :: v(np,np,2) - real(kind=real_kind), intent(in) :: p(np,np) - real(kind=real_kind), intent(in) :: pedges(0:np+1,0:np+1) - type (derivative_t), intent(in) :: deriv - type (element_t), intent(in) :: elem - real(kind=real_kind) :: result(np,np) - logical :: u_is_contra - - ! Local - real(kind=real_kind) :: ucontra(np,np,2) ! in lat-lon coordinates - real(kind=real_kind) :: flux,pstar - integer i,j - - - result=0 - - - if (u_is_contra) then - ucontra=v - else - ! latlon->contra - do j=1,np - do i=1,np - ucontra(i,j,1)=(elem%Dinv(i,j,1,1)*v(i,j,1) + elem%Dinv(i,j,1,2)*v(i,j,2)) - ucontra(i,j,2)=(elem%Dinv(i,j,2,1)*v(i,j,1) + elem%Dinv(i,j,2,2)*v(i,j,2)) - enddo - enddo - endif -#if 0 - ! centered - do i=1,np - j=1 - pstar=(pedges(i,0) + p(i,j) ) /2 - flux = -pstar*ucontra(i,j,2)*( deriv%Mvv_twt(i,i)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - - j=np - pstar=(pedges(i ,np+1) + p(i,j) ) /2 - flux = pstar*ucontra(i,j,2)* ( deriv%Mvv_twt(i,i)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - enddo - - do j=1,np - i=1 - pstar=(pedges(0 ,j ) + p(i,j) )/2 - flux = -pstar*ucontra(i,j,1)* ( deriv%Mvv_twt(j,j)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - - i=np - pstar=(pedges(np+1,j ) + p(i,j) ) /2 - flux = pstar*ucontra(i,j,1)* ( deriv%Mvv_twt(j,j)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - end do -#else - ! upwind - do i=1,np - j=1 - pstar=p(i,j) - if (ucontra(i,j,2)>0) pstar=pedges(i,0) - flux = -pstar*ucontra(i,j,2)*( deriv%Mvv_twt(i,i)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - - j=np - pstar=p(i,j) - if (ucontra(i,j,2)<0) pstar=pedges(i,np+1) - flux = pstar*ucontra(i,j,2)* ( deriv%Mvv_twt(i,i)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - enddo - - do j=1,np - i=1 - pstar=p(i,j) - if (ucontra(i,j,1)>0) pstar=pedges(0,j) - flux = -pstar*ucontra(i,j,1)* ( deriv%Mvv_twt(j,j)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - - i=np - pstar=p(i,j) - if (ucontra(i,j,1)<0) pstar=pedges(np+1,j) - flux = pstar*ucontra(i,j,1)* ( deriv%Mvv_twt(j,j)*elem%metdet(i,j)*scale_factor_inv) - result(i,j)=result(i,j)+flux - end do -#endif - - end function edge_flux_u_cg !DIR$ ATTRIBUTES FORCEINLINE :: vorticity_sphere @@ -1189,17 +1092,11 @@ function laplace_sphere_wk(s,deriv,elem,var_coef) result(laplace) grads=gradient_sphere(s,deriv,elem%Dinv) if (var_coef) then - if (hypervis_power/=0 ) then - ! scalar viscosity with variable coefficient - grads(:,:,1) = grads(:,:,1)*elem%variable_hyperviscosity(:,:) - grads(:,:,2) = grads(:,:,2)*elem%variable_hyperviscosity(:,:) - else if (hypervis_scaling /=0 ) then + if (hypervis_scaling /=0 ) then ! tensor hv, (3) oldgrads=grads do j=1,np do i=1,np -!JMD grads(i,j,1) = sum(oldgrads(i,j,:)*elem%tensorVisc(i,j,1,:)) -!JMD grads(i,j,2) = sum(oldgrads(i,j,:)*elem%tensorVisc(i,j,2,:)) grads(i,j,1) = oldgrads(i,j,1)*elem%tensorVisc(i,j,1,1) + & oldgrads(i,j,2)*elem%tensorVisc(i,j,1,2) grads(i,j,2) = oldgrads(i,j,1)*elem%tensorVisc(i,j,2,1) + & @@ -1326,12 +1223,6 @@ function vlaplace_sphere_wk_contra(v,deriv,elem,var_coef,nu_ratio) result(laplac div=divergence_sphere(v,deriv,elem) vor=vorticity_sphere(v,deriv,elem) - if (var_coef .and. hypervis_power/=0 ) then - ! scalar viscosity with variable coefficient - div = div*elem%variable_hyperviscosity(:,:) - vor = vor*elem%variable_hyperviscosity(:,:) - endif - if (present(nu_ratio)) div = nu_ratio*div laplace = gradient_sphere_wk_testcov(div,deriv,elem) - & diff --git a/components/homme/src/share/global_norms_mod.F90 b/components/homme/src/share/global_norms_mod.F90 index e3539f1d3304..87835ad2f264 100644 --- a/components/homme/src/share/global_norms_mod.F90 +++ b/components/homme/src/share/global_norms_mod.F90 @@ -257,7 +257,7 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) use reduction_mod, only : ParallelMin,ParallelMax use physical_constants, only : scale_factor_inv, scale_factor,dd_pi - use control_mod, only : nu, nu_q, nu_div, hypervis_order, nu_top, hypervis_power, & + use control_mod, only : nu, nu_q, nu_div, hypervis_order, nu_top, & fine_ne, max_hypervis_courant, hypervis_scaling, dcmip16_mu,dcmip16_mu_s,dcmip16_mu_q use control_mod, only : tstep_type use parallel_mod, only : abortmp, global_shared_buf, global_shared_sum @@ -337,121 +337,7 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) min_max_dx=ParallelMin(min_max_dx,hybrid) -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! SCALAR, RESOLUTION-AWARE HYPERVISCOSITY -! this block of code initializes the variable_hyperviscsoity() array -! based on largest length scale in each element and user specified scaling -! it then limits the coefficient if the user specifed a max CFL -! this limiting is based on the smallest length scale of each element -! since that controls the CFL. -! Mike Levy -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - if (hypervis_power /= 0) then - - min_hypervis = 1d99 - max_hypervis = 0 - avg_hypervis = 0 - - - max_unif_dx = min_max_dx ! use this for average resolution, unless: -! viscosity in namelist specified for smallest element: - if (fine_ne>0) then - ! viscosity in namelist specified for regions with a resolution - ! equivilant to a uniform grid with ne=fine_ne - if (np /= 4 ) call abortmp('ERROR: setting fine_ne only supported with NP=4') - max_unif_dx = (111.28*30)/dble(fine_ne) ! in km - endif - -! -! note: if L = eigenvalue of metinv, then associated length scale (km) is -! dx = 1.0d0/( sqrt(L)*0.5d0*dble(np-1)*rrearth*1000.0d0) -! -! for viscosity *tensor*, we take at each point: -! nu1 = nu*(dx1/max_unif_dx)**3.2 dx1 associated with eigenvalue 1 -! nu2 = nu*(dx2/max_unif_dx)**3.2 dx2 associated with eigenvalue 2 -! with this approach: -! - with this formula, no need to adjust for CFL violations -! - if nu comes from a 3.2 scaling that is stable for coarse and fine resolutions, -! this formulat will be stable. -! - gives the correct answer in long skinny rectangles: -! large viscosity in the long direction, small viscosity in the short direction -! -! - - normDinv_hypervis = 0 - - do ie=nets,nete - ! variable viscosity based on map from ulatlon -> ucontra - - ! dx_long - elem(ie)%variable_hyperviscosity = sqrt((elem(ie)%dx_long/max_unif_dx) ** hypervis_power) - elem(ie)%hv_courant = dtnu*(elem(ie)%variable_hyperviscosity(1,1)**2) * & - (lambda_vis**2) * ((scale_factor_inv*elem(ie)%normDinv)**4) - - ! Check to see if this is stable - if (elem(ie)%hv_courant.gt.max_hypervis_courant) then - stable_hv = sqrt( max_hypervis_courant / & - ( dtnu * (lambda_vis)**2 * (scale_factor_inv*elem(ie)%normDinv)**4 ) ) - -#if 0 - ! Useful print statements for debugging the adjustments to hypervis - print*, "Adjusting hypervis on elem ", elem(ie)%GlobalId - print*, "From ", nu*elem(ie)%variable_hyperviscosity(1,1)**2, " to ", nu*stable_hv - print*, "Difference = ", nu*(/elem(ie)%variable_hyperviscosity(1,1)**2-stable_hv/) - print*, "Factor of ", elem(ie)%variable_hyperviscosity(1,1)**2/stable_hv - print*, " " -#endif - -! make sure that: elem(ie)%hv_courant <= max_hypervis_courant - elem(ie)%variable_hyperviscosity = stable_hv - elem(ie)%hv_courant = dtnu*(stable_hv**2) * (lambda_vis)**2 * (scale_factor_inv*elem(ie)%normDinv)**4 - end if - normDinv_hypervis = max(normDinv_hypervis, elem(ie)%hv_courant/dtnu) - - min_hypervis = min(min_hypervis, elem(ie)%variable_hyperviscosity(1,1)) - max_hypervis = max(max_hypervis, elem(ie)%variable_hyperviscosity(1,1)) - global_shared_buf(ie,1) = elem(ie)%variable_hyperviscosity(1,1) - end do - - min_hypervis = ParallelMin(min_hypervis, hybrid) - max_hypervis = ParallelMax(max_hypervis, hybrid) - call wrap_repro_sum(nvars=1, comm=hybrid%par%comm) - avg_hypervis = global_shared_sum(1)/dble(nelem) - - normDinv_hypervis = ParallelMax(normDinv_hypervis, hybrid) - - ! apply DSS (aka assembly procedure) to variable_hyperviscosity (makes continuous) - call initEdgeBuffer(hybrid%par,edgebuf,elem,1) - do ie=nets,nete - zeta(:,:,ie) = elem(ie)%variable_hyperviscosity(:,:)*elem(ie)%spheremp(:,:) - call edgeVpack(edgebuf,zeta(1,1,ie),1,0,ie) - end do - call bndry_exchangeV(hybrid,edgebuf) - do ie=nets,nete - call edgeVunpack(edgebuf,zeta(1,1,ie),1,0,ie) - elem(ie)%variable_hyperviscosity(:,:) = zeta(:,:,ie)*elem(ie)%rspheremp(:,:) - end do - call FreeEdgeBuffer(edgebuf) - - ! replace hypervis w/ bilinear based on continuous corner values - do ie=nets,nete - noreast = elem(ie)%variable_hyperviscosity(np,np) - nw = elem(ie)%variable_hyperviscosity(1,np) - se = elem(ie)%variable_hyperviscosity(np,1) - sw = elem(ie)%variable_hyperviscosity(1,1) - do i=1,np - x = gp%points(i) - do j=1,np - y = gp%points(j) - elem(ie)%variable_hyperviscosity(i,j) = 0.25d0*( & - (1.0d0-x)*(1.0d0-y)*sw + & - (1.0d0-x)*(y+1.0d0)*nw + & - (x+1.0d0)*(1.0d0-y)*se + & - (x+1.0d0)*(y+1.0d0)*noreast) - end do - end do - end do - else if (hypervis_scaling/=0) then + if (hypervis_scaling/=0) then ! tensorHV. New eigenvalues are the eigenvalues of the tensor V ! formulas here must match what is in cube_mod.F90 ! for tensorHV, we scale out the rearth dependency @@ -472,7 +358,6 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) ! Oksana Guba !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if (hypervis_scaling /= 0) then -#if 1 call initEdgeBuffer(hybrid%par,edgebuf,elem,1) do rowind=1,2 do colind=1,2 @@ -489,10 +374,8 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) enddo !rowind enddo !colind call FreeEdgeBuffer(edgebuf) -#endif !IF BILINEAR MAP OF V NEEDED -#if 1 do rowind=1,2 do colind=1,2 ! replace hypervis w/ bilinear based on continuous corner values @@ -515,7 +398,6 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) end do enddo !rowind enddo !colind -#endif endif deallocate(gp%points) deallocate(gp%weights) @@ -576,12 +458,6 @@ subroutine print_cfl(elem,hybrid,nets,nete,dtnu) if(dcmip16_mu_s>0)write(iulog,'(a,f10.2,a)') 'dcmip16_mu_s viscosity CFL: dt < S*', & 1.0d0/(dcmip16_mu_s*((scale_factor_inv*max_normDinv)**2)*lambda_vis),'s' - if (hypervis_power /= 0) then - write(iulog,'(a,3e11.4)')'Hyperviscosity (dynamics): ave,min,max = ', & - nu*(/avg_hypervis**2,min_hypervis**2,max_hypervis**2/) -! print*, 'fine_ne = ', fine_ne -! print*, 'Using max_unif_dx = ', max_unif_dx - end if write(iulog,*) 'tstep_type = ',tstep_type end if diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 23f334a1dcd0..44eba15c728c 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -74,7 +74,6 @@ module namelist_mod disable_diagnostics, & ! use to disable diagnostics for timing reasons psurf_vis, & hypervis_order, & - hypervis_power, & hypervis_subcycle, & hypervis_subcycle_tom,& hypervis_subcycle_q, & @@ -291,7 +290,6 @@ subroutine readnl(par) dcmip16_pbl_type,& psurf_vis, & hypervis_order, & - hypervis_power, & hypervis_subcycle, & hypervis_subcycle_tom, & hypervis_subcycle_q, & @@ -801,7 +799,6 @@ subroutine readnl(par) call MPI_bcast(disable_diagnostics,1,MPIlogical_t,par%root,par%comm,ierr) call MPI_bcast(psurf_vis,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_order,1,MPIinteger_t ,par%root,par%comm,ierr) - call MPI_bcast(hypervis_power,1,MPIreal_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_scaling,1,MPIreal_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_subcycle,1,MPIinteger_t ,par%root,par%comm,ierr) call MPI_bcast(hypervis_subcycle_tom,1,MPIinteger_t ,par%root,par%comm,ierr) @@ -1021,18 +1018,7 @@ subroutine readnl(par) laplacian_rigid_factor = rrearth end if ! if plane -!logic around different hyperviscosity options - if (hypervis_power /= 0) then - if (hypervis_scaling /= 0) then - print *,'Both hypervis_power and hypervis_scaling are nonzero.' - print *,'(1) Set hypervis_power=1, hypervis_scaling=0 for HV based on an element area.' - print *,'(2) Set hypervis_power=0 and hypervis_scaling=1 for HV based on a tensor.' - print *,'(3) Set hypervis_power=0 and hypervis_scaling=0 for constant HV.' - call abortmp("Error: hypervis_power>0 and hypervis_scaling>0") - endif - endif - - if (topology == "plane" .and. .not. (hypervis_power == 0 .and. hypervis_scaling > 0)) then + if (topology == "plane" .and. hypervis_scaling==0) then call abortmp("Error: planar grids require the use of tensor HV") end if @@ -1199,11 +1185,7 @@ subroutine readnl(par) write(iulog,*)"readnl: runtype = ",runtype write(iulog,*)"readnl: se_fv_phys_remap_alg = ",se_fv_phys_remap_alg - if (hypervis_power /= 0)then - write(iulog,*)"Variable scalar hyperviscosity: hypervis_power=",hypervis_power - write(iulog,*)"max_hypervis_courant = ", max_hypervis_courant - write(iulog,*)"Equivalent ne in fine region = ", fine_ne - elseif(hypervis_scaling /=0)then + if(hypervis_scaling /=0)then write(iulog,*)"Tensor hyperviscosity: hypervis_scaling=",hypervis_scaling else write(iulog,*)"Constant (hyper)viscosity used." diff --git a/components/homme/src/share/sl_advection.F90 b/components/homme/src/share/sl_advection.F90 index f2c5c2b35ed7..af337eaf2bad 100644 --- a/components/homme/src/share/sl_advection.F90 +++ b/components/homme/src/share/sl_advection.F90 @@ -137,7 +137,7 @@ end subroutine sl_init1 subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_option_out, & cdr_check, geometry_type) bind(c) use control_mod, only: semi_lagrange_hv_q, hypervis_subcycle_q, semi_lagrange_cdr_check, & - hypervis_power, nu_q, hypervis_scaling, limiter_option, geometry + nu_q, hypervis_scaling, limiter_option, geometry use iso_c_binding, only: c_int, c_double real(c_double), intent(out) :: nu_q_out, hv_scaling @@ -153,9 +153,6 @@ subroutine sl_get_params(nu_q_out, hv_scaling, hv_q, hv_subcycle_q, limiter_opti geometry_type = 0 ! sphere if (trim(geometry) == "plane") geometry_type = 1 - ! Assert some things. - if (nu_q > 0 .and. hv_q > 0 .and. hypervis_power /= 0) & - call abortmp('SL transport: hypervis_power /= 0') end subroutine sl_get_params subroutine prim_advec_tracers_remap_ALE(elem, deriv, hvcoord, hybrid, dt, tl, nets, nete) diff --git a/components/homme/src/sweqx/derivative_mod.F90 b/components/homme/src/sweqx/derivative_mod.F90 index 277234869f0c..c953f714853c 100644 --- a/components/homme/src/sweqx/derivative_mod.F90 +++ b/components/homme/src/sweqx/derivative_mod.F90 @@ -25,7 +25,6 @@ module derivative_mod curl_sphere_wk_testcov, & vlaplace_sphere_wk, & element_boundary_integral, & - edge_flux_u_cg, & limiter_optim_iter_full, & limiter_clip_and_sum, & laplace_sphere_wk, & diff --git a/components/homme/src/theta-l/share/derivative_mod.F90 b/components/homme/src/theta-l/share/derivative_mod.F90 index a2efb73b5b84..452a2b6f7b63 100644 --- a/components/homme/src/theta-l/share/derivative_mod.F90 +++ b/components/homme/src/theta-l/share/derivative_mod.F90 @@ -6,7 +6,7 @@ module derivative_mod use derivative_mod_base, only: derivative_t, subcell_integration, subcell_dss_fluxes, subcell_div_fluxes, subcell_Laplace_fluxes, allocate_subcell_integration_matrix, & derivinit, gradient, gradient_wk, vorticity, divergence, & gradient_sphere_wk_testcov, gradient_sphere_wk_testcontra, ugradv_sphere, vorticity_sphere, vorticity_sphere_diag, curl_sphere, & - curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, edge_flux_u_cg, limiter_optim_iter_full, limiter_clip_and_sum,& + curl_sphere_wk_testcov, vlaplace_sphere_wk, element_boundary_integral, limiter_optim_iter_full, limiter_clip_and_sum,& laplace_sphere_wk, divergence_sphere_wk, gradient_sphere, divergence_sphere, laplace_z, get_deriv implicit none end module derivative_mod From 7a1ab5f89540a8de2bc8e7816de09f0205f7982f Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 20 Apr 2021 11:29:40 -0500 Subject: [PATCH 237/301] remove unused qn0 argument --- .../src/arkode/theta-l/arkode_interface.F90 | 19 ++- .../homme/src/arkode/theta-l/arkode_mod.F90 | 45 ++------ .../homme/src/theta-l/share/imex_mod.F90 | 7 +- .../src/theta-l/share/prim_advance_mod.F90 | 108 +++++++++--------- 4 files changed, 74 insertions(+), 105 deletions(-) diff --git a/components/homme/src/arkode/theta-l/arkode_interface.F90 b/components/homme/src/arkode/theta-l/arkode_interface.F90 index 35af5e3ac64b..f28d61c0ca14 100644 --- a/components/homme/src/arkode/theta-l/arkode_interface.F90 +++ b/components/homme/src/arkode/theta-l/arkode_interface.F90 @@ -43,7 +43,7 @@ ! Modified by Christopher J. Vogl (LLNL) 2020 !================================================================= -function farkifun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) & +function farkifun(t, y, ydot, hvcoord, hybrid, deriv, imex, dt, eta_ave_w) & result(ierr) !----------------------------------------------------------------- ! Description: farkifun provides the implicit portion of the right @@ -75,7 +75,6 @@ function farkifun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) type(hvcoord_t), intent(in) :: hvcoord type(hybrid_t), intent(in) :: hybrid type(derivative_t), intent(in) :: deriv - integer, intent(in) :: qn0 integer, intent(in) :: imex real(real_kind), intent(in) :: dt real(real_kind), intent(in) :: eta_ave_w @@ -106,7 +105,7 @@ function farkifun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) bval = 1.d0 ! The function call to compute_andor_apply_rhs is as follows: - ! compute_andor_apply_rhs(np1, nm1, n0, qn0, dt2, elem, hvcoord, hybrid, & + ! compute_andor_apply_rhs(np1, nm1, n0, dt2, elem, hvcoord, hybrid, & ! deriv, nets, nete, compute_diagnostics, eta_ave_w, scale1, scale2, scale3) ! ! This call returns the following: @@ -121,7 +120,7 @@ function farkifun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) ! DSS is the averaging procedure for the active and inactive nodes ! - call compute_andor_apply_rhs(ydot%tl_idx, ydot%tl_idx, y%tl_idx, qn0, & + call compute_andor_apply_rhs(ydot%tl_idx, ydot%tl_idx, y%tl_idx, & 1.d0, y%elem, hvcoord, hybrid, deriv, y%nets, y%nete, & .false., bval*eta_ave_w, scale1, scale2, 0.d0) @@ -135,7 +134,7 @@ end function farkifun !================================================================= -function farkefun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) & +function farkefun(t, y, ydot, hvcoord, hybrid, deriv, imex, dt, eta_ave_w) & result(ierr) !----------------------------------------------------------------- ! Description: farkefun provides the explicit portion of the right @@ -167,7 +166,6 @@ function farkefun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) type(hvcoord_t), intent(in) :: hvcoord type(hybrid_t), intent(in) :: hybrid type(derivative_t), intent(in) :: deriv - integer, intent(in) :: qn0 integer, intent(in) :: imex real(real_kind), intent(in) :: dt real(real_kind), intent(in) :: eta_ave_w @@ -198,7 +196,7 @@ function farkefun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) bval = 1.d0 ! The function call to compute_andor_apply_rhs is as follows: - ! compute_andor_apply_rhs(np1, nm1, n0, qn0, dt2, elem, hvcoord, hybrid, & + ! compute_andor_apply_rhs(np1, nm1, n0, dt2, elem, hvcoord, hybrid, & ! deriv, nets, nete, compute_diagnostics, eta_ave_w, scale1, scale2, scale3) ! ! This call returns the following: @@ -213,7 +211,7 @@ function farkefun(t, y, ydot, hvcoord, hybrid, deriv, qn0, imex, dt, eta_ave_w) ! DSS is the averaging procedure for the active and inactive nodes ! - call compute_andor_apply_rhs(ydot%tl_idx, ydot%tl_idx, y%tl_idx, qn0, & + call compute_andor_apply_rhs(ydot%tl_idx, ydot%tl_idx, y%tl_idx, & 1.d0, y%elem, hvcoord, hybrid, deriv, y%nets, y%nete, & .false., bval*eta_ave_w, scale1, scale2, 0.d0) @@ -243,7 +241,7 @@ function FColumnSolSolve(b, t, y, gamma, x) result(ierr) !----------------------------------------------------------------- !======= Inclusions =========== - use arkode_mod, only: get_hvcoord_ptr, get_qn0 + use arkode_mod, only: get_hvcoord_ptr use control_mod, only: theta_hydrostatic_mode use eos, only: pnh_and_exner_from_eos use imex_mod, only: get_dirk_jacobian @@ -282,11 +280,10 @@ function FColumnSolSolve(b, t, y, gamma, x) result(ierr) real (kind=real_kind) :: dpnh_dp_i(np,np,nlevp) real (kind=real_kind) :: exner(np,np,nlev) real (kind=real_kind) :: b1(nlev,np,np) - integer :: qn0, i, j, k, ie, info(np,np), Ipiv(nlev,np,np) + integer :: i, j, k, ie, info(np,np), Ipiv(nlev,np,np) !======= Internals ============ call get_hvcoord_ptr(hvcoord) - call get_qn0(qn0) ! set return value to success ierr = 0 diff --git a/components/homme/src/arkode/theta-l/arkode_mod.F90 b/components/homme/src/arkode/theta-l/arkode_mod.F90 index b3fdcdca9d1c..f6f2489f2635 100644 --- a/components/homme/src/arkode/theta-l/arkode_mod.F90 +++ b/components/homme/src/arkode/theta-l/arkode_mod.F90 @@ -59,7 +59,7 @@ module arkode_mod type(N_Vector), pointer :: ptr end type N_VectorPtr - public :: parameter_list, evolve_solution, get_hvcoord_ptr, get_qn0 + public :: parameter_list, evolve_solution, get_hvcoord_ptr public :: update_nonlinear_stats, finalize_nonlinear_stats save @@ -72,7 +72,7 @@ module arkode_mod type(N_VectorPtr) :: y(3) type(c_funptr) :: efun_ptr, ifun_ptr real(real_kind) :: dt_save, eta_ave_w_save, rout(40) - integer :: imex_save, qn0_save + integer :: imex_save logical :: initialized = .false. ! variables used for nonlinear solver stats, not necessary for use of ARKode integer :: total_nonlinear_iterations = 0 @@ -190,29 +190,9 @@ end subroutine get_hvcoord_ptr !================================================================= - subroutine get_qn0(qn0) - !----------------------------------------------------------------- - ! Description: obtains current qn0 value - ! Arguments: - ! qn0 - (int, output) qn0 value - !----------------------------------------------------------------- - - !======= Declarations ========= - implicit none - - ! calling variables - integer, intent(out) :: qn0 - - !======= Internals ============ - qn0 = qn0_save - - return - end subroutine get_qn0 - - !================================================================= function evolve_solution(elem, nets, nete, deriv, hvcoord, hybrid, & - dt, eta_ave_w, n0, np1, qn0, arkode_parameters, & + dt, eta_ave_w, n0, np1, arkode_parameters, & table_set) result(ierr) !----------------------------------------------------------------- ! Description: resets internal ARKode solution without memory allocation @@ -228,7 +208,6 @@ function evolve_solution(elem, nets, nete, deriv, hvcoord, hybrid, & ! eta_ave_w - (real, input) average flux value ! n0 - (int, input) timelevel holding current solution ! np1 - (int, input) timelevel for evolved solution - ! qn0 - (int, input) timelevel for tracer mass ! arkode_parameters - (parameter, input) object for arkode parameters ! table_set - (butcher_table_set, input) Butcher table(s) !----------------------------------------------------------------- @@ -255,7 +234,7 @@ function evolve_solution(elem, nets, nete, deriv, hvcoord, hybrid, & type(parameter_list), intent(in) :: arkode_parameters type(butcher_table_set), intent(in) :: table_set real(real_kind), intent(in) :: dt, eta_ave_w - integer, intent(in) :: nets, nete, n0, np1, qn0 + integer, intent(in) :: nets, nete, n0, np1 type(element_t), intent(inout) :: elem(:) ! local variables @@ -274,14 +253,13 @@ function evolve_solution(elem, nets, nete, deriv, hvcoord, hybrid, & dt_save = dt eta_ave_w_save = eta_ave_w imex_save = table_set%imex - qn0_save = qn0 hybrid_ptr => hybrid deriv_ptr => deriv hvcoord_ptr => hvcoord ! Initialize or reinitialize ARKode if (.not.initialized) then - call initialize(elem, nets, nete, hybrid%par, n0, qn0, tstart, & + call initialize(elem, nets, nete, hybrid%par, n0, tstart, & arkode_parameters, table_set) initialized = .true. else @@ -371,7 +349,7 @@ end subroutine reinitialize !================================================================= - subroutine initialize(elem, nets, nete, par, n0, qn0, tstart, & + subroutine initialize(elem, nets, nete, par, n0, tstart, & arkode_parameters, table_set) !----------------------------------------------------------------- ! Description: allocates memory for and initializes ARKode @@ -381,7 +359,6 @@ subroutine initialize(elem, nets, nete, par, n0, qn0, tstart, & ! nete - (int, input) ending index for elem array ! par - (obj*, input) parallel object ! n0 - (int, input) timelevel holding current solution - ! qn0 - (int, input) timelevel for tracer mass ! tstart - (real, input) time to start ARKode solve at ! arkode_parameters - (parameter, input) object for arkode parameters ! table_set - (butcher_table_set, input) object for Butcher table(s) @@ -413,7 +390,7 @@ subroutine initialize(elem, nets, nete, par, n0, qn0, tstart, & type(parameter_list), target, intent(in) :: arkode_parameters type(butcher_table_set), target, intent(in) :: table_set real(real_kind), intent(in) :: tstart - integer, intent(in) :: nets, nete, n0, qn0 + integer, intent(in) :: nets, nete, n0 type(element_t), intent(inout) :: elem(:) ! local variables @@ -706,14 +683,14 @@ function efun(t, sunvec_y, sunvec_ydot, data) result(ierr) bind(C) type(NVec_t), pointer :: y => NULL() type(NVec_t), pointer :: ydot => NULL() real(real_kind) :: dt, eta_ave_w, bval, cval, scale1, scale2 - integer :: imex, qn0, ie + integer :: imex, ie integer :: farkefun !======= Internals ============ y => FN_VGetContent(sunvec_y) ydot => FN_VGetContent(sunvec_ydot) - ierr = farkefun(t, y, ydot, hvcoord_ptr, hybrid_ptr, deriv_ptr, qn0_save, & + ierr = farkefun(t, y, ydot, hvcoord_ptr, hybrid_ptr, deriv_ptr, & imex_save, dt_save, eta_ave_w_save) end function efun @@ -749,14 +726,14 @@ function ifun(t, sunvec_y, sunvec_ydot, data) result(ierr) bind(C) type(NVec_t), pointer :: y => NULL() type(NVec_t), pointer :: ydot => NULL() real(real_kind) :: dt, eta_ave_w, bval, cval, scale1, scale2 - integer :: imex, qn0, ie + integer :: imex, ie integer :: farkifun !======= Internals ============ y => FN_VGetContent(sunvec_y) ydot => FN_VGetContent(sunvec_ydot) - ierr = farkifun(t, y, ydot, hvcoord_ptr, hybrid_ptr, deriv_ptr, qn0_save, & + ierr = farkifun(t, y, ydot, hvcoord_ptr, hybrid_ptr, deriv_ptr, & imex_save, dt_save, eta_ave_w_save) end function ifun diff --git a/components/homme/src/theta-l/share/imex_mod.F90 b/components/homme/src/theta-l/share/imex_mod.F90 index 84efbbe614dd..6ebfebfbb917 100644 --- a/components/homme/src/theta-l/share/imex_mod.F90 +++ b/components/homme/src/theta-l/share/imex_mod.F90 @@ -58,7 +58,7 @@ end subroutine compute_gwphis - subroutine compute_stage_value_dirk(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2,qn0,elem,hvcoord,hybrid,& + subroutine compute_stage_value_dirk(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,itercount,itererr,verbosity_in) !=================================================================================== ! this subroutine solves a stage value equation for a DIRK method which takes the form @@ -89,7 +89,7 @@ subroutine compute_stage_value_dirk(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2,qn0,el ! !=================================================================================== - integer, intent(in) :: nm1,n0,np1,qn0,nets,nete + integer, intent(in) :: nm1,n0,np1,nets,nete real (kind=real_kind), intent(in) :: dt2 integer :: itercount real (kind=real_kind) :: itererr @@ -541,7 +541,7 @@ subroutine test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) real (kind=real_kind) :: norminfJ0(np,np) real (kind=real_kind) :: dt,epsie,jacerrorvec(6),minjacerr - integer :: k,ie,qn0,i,j + integer :: k,ie,i,j minjacerr=0 if (hybrid%masterthread) write(iulog,*)'Running IMEX Jacobian unit test...' do ie=nets,nete @@ -552,7 +552,6 @@ subroutine test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) vtheta_dp(:,:,:) = elem(ie)%state%vtheta_dp(:,:,:,tl%n0) phi_i(:,:,:) = elem(ie)%state%phinh_i(:,:,:,tl%n0) phis(:,:) = elem(ie)%state%phis(:,:) - call TimeLevel_Qdp(tl, qsplit, qn0) call pnh_and_exner_from_eos(hvcoord,vtheta_dp,dp3d,phi_i,& pnh,exner,dpnh_dp_i,pnh_i_out=pnh_i) diff --git a/components/homme/src/theta-l/share/prim_advance_mod.F90 b/components/homme/src/theta-l/share/prim_advance_mod.F90 index 181689099eb4..0bbf611106c0 100644 --- a/components/homme/src/theta-l/share/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advance_mod.F90 @@ -101,7 +101,7 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co real (kind=real_kind) :: ahat3,ahat4,ahat5,ahat6,dhat1,dhat2,dhat3,dhat4 real (kind=real_kind) :: gamma,delta,ap,aphat,dhat5,offcenter - integer :: ie,nm1,n0,np1,nstep,qsplit_stage,k, qn0 + integer :: ie,nm1,n0,np1,nstep,qsplit_stage,k integer :: n,i,j,maxiter #ifdef ARKODE @@ -117,8 +117,6 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co np1 = tl%np1 nstep = tl%nstep - ! get timelevel for accessing tracer mass Qdp() to compute virtual temperature - call TimeLevel_Qdp(tl, qsplit, qn0) ! compute current Qdp() timelevel ! integration = "explicit" ! @@ -158,23 +156,23 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co if (tstep_type==1) then ! RK2 ! forward euler to u(dt/2) = u(0) + (dt/2) RHS(0) (store in u(np1)) - call compute_andor_apply_rhs(np1,n0,n0,qn0,dt/2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,n0,dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) ! leapfrog: u(dt) = u(0) + dt RHS(dt/2) (store in u(np1)) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w,1.d0,1.d0,1.d0) else if (tstep_type==4) then ! explicit table from IMEX-KG254 method - call compute_andor_apply_rhs(np1,n0,n0,qn0,dt/4,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,n0,dt/4,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/6,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt/6,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) - call compute_andor_apply_rhs(np1,n0,np1,qn0,3*dt/8,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,3*dt/8,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w*1d0,1.d0,1.d0,1.d0) @@ -182,16 +180,16 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co else if (tstep_type==5) then ! Ullrich 3nd order 5 stage: CFL=sqrt( 4^2 -1) = 3.87 ! u1 = u0 + dt/5 RHS(u0) (save u1 in timelevel nm1) - call compute_andor_apply_rhs(nm1,n0,n0,qn0,dt/5,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(nm1,n0,n0,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,eta_ave_w/4,1.d0,1.d0,1.d0) ! u2 = u0 + dt/5 RHS(u1) - call compute_andor_apply_rhs(np1,n0,nm1,qn0,dt/5,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,nm1,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) ! u3 = u0 + dt/3 RHS(u2) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/3,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) ! u4 = u0 + 2dt/3 RHS(u3) - call compute_andor_apply_rhs(np1,n0,np1,qn0,2*dt/3,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,2*dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,1.d0,1.d0) ! compute (5*u1/4 - u0/4) in timelevel nm1: do ie=nets,nete @@ -207,7 +205,7 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co - elem(ie)%state%phinh_i(:,:,1:nlev,n0) )/4 enddo ! u5 = (5*u1/4 - u0/4) + 3dt/4 RHS(u4) - call compute_andor_apply_rhs(np1,nm1,np1,qn0,3*dt/4,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,nm1,np1,3*dt/4,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,3*eta_ave_w/4,1.d0,1.d0,1.d0) ! final method is the same as: ! u5 = u0 + dt/4 RHS(u0)) + 3dt/4 RHS(u4) @@ -216,33 +214,33 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co a1=0d0 a2=1-a1 dt2=dt/4 - call compute_andor_apply_rhs(np1,n0,n0,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,n0,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=dt/6 - call compute_andor_apply_rhs(nm1,n0,np1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(nm1,n0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,nm1,a2*dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,nm1,a2*dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=3*dt/8 - call compute_andor_apply_rhs(np1,n0,nm1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,nm1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=dt/2 - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,a1*dt2,np1,a2*dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w*1d0,1.d0,0d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,a1*dt,np1,a2*dt,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,a1*dt,np1,a2*dt,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u0 saved in elem(n0) ! u2 saved in elem(nm1) @@ -256,16 +254,16 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co aphat = 0.5d0-offcenter dhat3 = 0.5d0+offcenter - call compute_andor_apply_rhs(np1,n0,n0,qn0,dt/2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,n0,dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,0d0,1d0,0d0,1d0) ! aphat/ap,1d0) - call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt/2,np1,dhat3*dt/2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt/2,np1,dhat3*dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1d0,0d0,1d0) - call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt/2,np1,dhat3*dt/2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt/2,np1,dhat3*dt/2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! introduce 1st order offcentering @@ -273,41 +271,41 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co aphat = 0.5d0-offcenter dhat3 = 0.5d0+offcenter - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w,1d0,0d0,1d0) - call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt,np1,dhat3*dt,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,aphat*dt,np1,dhat3*dt,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) else if (tstep_type==9) then ! KGU5-3 (3rd order) with IMEX backward euler (2nd order) ! ! u1 = u0 + dt/5 RHS(u0) (save u1 in timelevel nm1) - call compute_andor_apply_rhs(nm1,n0,n0,qn0,dt/5,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(nm1,n0,n0,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,eta_ave_w/4,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,nm1,dt/5,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,nm1,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u2 = u0 + dt/5 RHS(u1) - call compute_andor_apply_rhs(np1,n0,nm1,qn0,dt/5,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,nm1,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt/5,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt/5,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u3 = u0 + dt/3 RHS(u2) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/3,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt/3,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u4 = u0 + 2dt/3 RHS(u3) - call compute_andor_apply_rhs(np1,n0,np1,qn0,2*dt/3,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,2*dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,2*dt/3,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,2*dt/3,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u5 = u1 + dt 3/4 RHS(u4) - call compute_andor_apply_rhs(np1,nm1,np1,qn0,3*dt/4,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,nm1,np1,3*dt/4,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,3*eta_ave_w/4,1.d0,0.d0,1.d0) ! u(np1) = [u1 + 3dt/4 RHS(u4)] + 1/4 (u1 - u0) STABLE do ie=nets,nete @@ -330,43 +328,43 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co a1=5*dt/18 a2=dt/36 ! 5/18 - 1/4 (due to the 1/4*u1 added above) a3=8*dt/18 - call compute_stage_value_dirk(nm1,a2,n0,a1,np1,a3,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,a2,n0,a1,np1,a3,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) else if (tstep_type==10) then ! KG5(2nd order CFL=4) + optimized dt2=dt/4 - call compute_andor_apply_rhs(nm1,n0,n0,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(nm1,n0,n0,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,nm1,dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,nm1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=dt/6 - call compute_andor_apply_rhs(np1,n0,nm1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,nm1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=3*dt/8 - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) dt2=dt/2 - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt2,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,0d0,1.d0,0.d0,1.d0) - call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,0d0,n0,0d0,np1,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) - call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + call compute_andor_apply_rhs(np1,n0,np1,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w*1d0,1.d0,0d0,1.d0) a1=.24362d0 a2=.34184d0 a3=1-(a1+a2) - call compute_stage_value_dirk(nm1,a2*dt,n0,a1*dt,np1,a3*dt,qn0,elem,hvcoord,hybrid,& + call compute_stage_value_dirk(nm1,a2*dt,n0,a1*dt,np1,a3*dt,elem,hvcoord,hybrid,& deriv,nets,nete,maxiter,itertol) ! u0 saved in elem(n0) ! u1 saved in elem(nm1) @@ -466,7 +464,7 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co ! use ARKode solver to evolve solution ierr = evolve_solution(elem, nets, nete, deriv, hvcoord, hybrid, & - dt, eta_ave_w, n0, np1, qn0, arkode_parameters, & + dt, eta_ave_w, n0, np1, arkode_parameters, & arkode_table_set) if (ierr /= 0) then call abortmp('ARKode evolve failed') @@ -1018,7 +1016,7 @@ end subroutine advance_physical_vis !============================ stiff and or non-stiff ============================================ - subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& + subroutine compute_andor_apply_rhs(np1,nm1,n0,dt2,elem,hvcoord,hybrid,& deriv,nets,nete,compute_diagnostics,eta_ave_w,scale1,scale2,scale3) ! =================================== ! compute the RHS, accumulate into u(np1) and apply DSS @@ -1030,11 +1028,9 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& ! accomodated. For example, setting nm1=np1=n0 this routine will ! take a forward euler step, overwriting the input with the output. ! - ! qn0 = timelevel used to access Qdp() in order to compute virtual Temperature - ! ! =================================== - integer, intent(in) :: np1,nm1,n0,qn0,nets,nete + integer, intent(in) :: np1,nm1,n0,nets,nete real*8, intent(in) :: dt2 logical, intent(in) :: compute_diagnostics type (hvcoord_t), intent(in) :: hvcoord From 3db07e8aaa2274fc1a50c406c2c9d063dd04fed5 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 20 Apr 2021 11:34:15 -0500 Subject: [PATCH 238/301] dont run a state check when scale3=0 --- components/homme/src/theta-l/share/prim_advance_mod.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/homme/src/theta-l/share/prim_advance_mod.F90 b/components/homme/src/theta-l/share/prim_advance_mod.F90 index 0bbf611106c0..313f174485a1 100644 --- a/components/homme/src/theta-l/share/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advance_mod.F90 @@ -1751,6 +1751,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,dt2,elem,hvcoord,hybrid,& enddo ! check for layer spacing <= 1m + if (scale3 /= 0) then do k=1,nlev do j=1,np do i=1,np @@ -1762,6 +1763,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,dt2,elem,hvcoord,hybrid,& enddo enddo enddo + endif #endif endif if (scale3 /= 0) then From e8d6a1901ef62ff6357ee6e4a771bedf17d211a6 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 24 Dec 2021 15:49:13 -0700 Subject: [PATCH 239/301] remove obsolete HOMME_IMEX_MKLSOLVE code --- .../homme/src/theta-l/share/imex_mod.F90 | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/components/homme/src/theta-l/share/imex_mod.F90 b/components/homme/src/theta-l/share/imex_mod.F90 index 6ebfebfbb917..65da7f1dca19 100644 --- a/components/homme/src/theta-l/share/imex_mod.F90 +++ b/components/homme/src/theta-l/share/imex_mod.F90 @@ -16,7 +16,6 @@ module imex_mod use control_mod, only: theta_hydrostatic_mode, qsplit use perf_mod, only: t_startf, t_stopf #ifdef HOMMEXX_BFB_TESTING - use bfb_mod, only: tridiag_diagdom_bfb_a1x1 use iso_c_binding, only: c_loc #endif @@ -128,15 +127,6 @@ subroutine compute_stage_value_dirk(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2,elem,h integer :: i,j,k,l,ie,nt integer :: nsafe,verbosity -!#define HOMME_IMEX_MKLSOLVE -#ifdef HOMME_IMEX_MKLSOLVE - real (kind=real_kind) :: JacDt(nlev,np,np) , JacLt(nlev-1,np,np) - real (kind=real_kind) :: JacUt(nlev-1,np,np), JacU2t(nlev-2,np,np) - real (kind=real_kind) :: Ipiv(nlev,np,np) - real (kind=real_kind) :: Fnt(np,np,nlev), xt(nlev,np,np) - integer :: info(np,np) -#endif - #undef NEWTONCOND #ifdef NEWTONCOND real*8 :: DLANGT ! external @@ -283,44 +273,7 @@ subroutine compute_stage_value_dirk(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2,elem,h x(:,:,1:nlev) = -Fn(:,:,1:nlev) -#ifdef HOMME_IMEX_MKLSOLVE - do j=1,np - do i=1,np - do k = 1,nlev-1 - JacLt(k,i,j) = JacL(i,j,k) - JacUt(k,i,j) = JacU(i,j,k) - end do - do k = 1,nlev - JacDt(k,i,j) = JacD(i,j,k) - xt(k,i,j) = x(i,j,k) - end do -# ifdef HOMMEXX_BFB_TESTING - ! Note: the C function is designed to accept both single and double precision, - ! so we need to pass also the size of a real (last argument) - call tridiag_diagdom_bfb_a1x1(nlev, JacLt(:,i,j), jacDt(:,i,j), jacUt(:,i,j), & - xt(:,i,j),INT(SIZEOF(JacL)/SIZE(JacL),4)) -# else - -#ifdef NEWTONCOND - ! nlev condition number: 500e3 with phi, 850e3 with dphi - anorm=DLANGT('1-norm', nlev, JacLt(:,i,j),JacDt(:,i,j),JacUt(:,i,j)) - call DGTTRF(nlev, JacLt(:,i,j), JacDt(:,i,j),JacUt(:,i,j),JacU2t(:,i,j), Ipiv(:,i,j), info ) - call DGTCON('1',nlev,JacLt(:,i,j),JacDt(:,i,j),JacUt(:,i,j),JacU2t(:,i,j),Ipiv(:,i,j),& - ANORM, RCOND, WORK, IWORK, info2 ) -#else - call DGTTRF(nlev, JacLt(:,i,j), JacDt(:,i,j),JacUt(:,i,j),JacU2t(:,i,j), Ipiv(:,i,j), info(i,j) ) -#endif - ! Tridiagonal solve - call DGTTRS( 'N', nlev,1, JacLt(:,i,j), JacDt(:,i,j), JacUt(:,i,j), JacU2t(:,i,j), Ipiv(:,i,j),xt(:,i,j), nlev, info(i,j) ) - do k = 1,nlev - x(i,j,k) = xt(k,i,j) - end do -#endif - end do - end do -#else call solve_strict_diag_dominant_tridiag(JacL, JacD, JacU, x) -#endif do k = 1,nlev-1 dphi(:,:,k) = dphi_n0(:,:,k) + & From d680cb8ae162631f6ad4cc5b1ac5d529e29183e1 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 24 Dec 2021 15:49:45 -0700 Subject: [PATCH 240/301] add comments about EOS needing to return dpnh_i=1 at surface --- components/homme/src/theta-l/share/eos.F90 | 4 ++-- components/homme/src/theta-l/share/prim_advance_mod.F90 | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/components/homme/src/theta-l/share/eos.F90 b/components/homme/src/theta-l/share/eos.F90 index 66084a7a5ea5..faad7ba744e2 100644 --- a/components/homme/src/theta-l/share/eos.F90 +++ b/components/homme/src/theta-l/share/eos.F90 @@ -188,9 +188,9 @@ subroutine pnh_and_exner_from_eos2(hvcoord,vtheta_dp,dp3d,dphi,pnh,exner,& ! surface boundary condition pnh_i determined by w equation to enforce ! w b.c. This is computed in the RHS calculation. Here, we use ! an approximation (hydrostatic) so that dpnh/dpi = 1 + ! DO NOT CHANGE this approximation. it is required by + ! compute_andor_apply_rhs() pnh_i(:,:,nlevp) = pnh(:,:,nlev) + dp3d(:,:,nlev)/2 - ! extrapolote NH perturbation: - !pnh_i(:,:,nlevp) = pi_i(:,:,nlevp) + (3*(pnh(:,:,nlev)-pi(:,:,nlev)) - (pnh(:,:,nlev-1)-pi(:,:,nlev-1)) )/2 ! compute d(pnh)/d(pi) at interfaces diff --git a/components/homme/src/theta-l/share/prim_advance_mod.F90 b/components/homme/src/theta-l/share/prim_advance_mod.F90 index 313f174485a1..6544125810bd 100644 --- a/components/homme/src/theta-l/share/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/share/prim_advance_mod.F90 @@ -1143,7 +1143,8 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,dt2,elem,hvcoord,hybrid,& enddo endif #endif - + ! this routine will set dpnh_dp_i(nlevp)=1 - a very good approximation, that will + ! then be corrected below, after the DSS. call pnh_and_exner_from_eos(hvcoord,vtheta_dp,dp3d,phi_i,pnh,exner,dpnh_dp_i,caller='CAAR') dp3d_i(:,:,1) = dp3d(:,:,1) From f56c25d2ec2da7c62ed8ee99ca0c74618d2d497f Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 24 Dec 2021 15:50:48 -0700 Subject: [PATCH 241/301] remove extraneous lines from some CMakeLists.txt files These cmake files define macros that are used by other executables. The should only contain the setup macro and a call to createTestExec() Any other commands wont be run by the other executables --- components/homme/src/theta-l/CMakeLists.txt | 1 - components/homme/src/tool/CMakeLists.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/components/homme/src/theta-l/CMakeLists.txt b/components/homme/src/theta-l/CMakeLists.txt index 2bad213f0b25..aa35eb39e68e 100644 --- a/components/homme/src/theta-l/CMakeLists.txt +++ b/components/homme/src/theta-l/CMakeLists.txt @@ -128,4 +128,3 @@ thetal_setup() ############################################################################ createTestExec(theta-l theta-l ${PREQX_NP} ${PREQX_NC} ${PREQX_PLEV} ${PREQX_USE_PIO} ${PREQX_USE_ENERGY} ${QSIZE_D}) -TARGET_LINK_LIBRARIES(theta-l dl) diff --git a/components/homme/src/tool/CMakeLists.txt b/components/homme/src/tool/CMakeLists.txt index 8398d0cba47e..ab80d2577447 100644 --- a/components/homme/src/tool/CMakeLists.txt +++ b/components/homme/src/tool/CMakeLists.txt @@ -72,4 +72,3 @@ tool_setup() ############################################################################ createTestExec(homme_tool tool ${PREQX_NP} ${PREQX_NC} ${PREQX_PLEV} ${PREQX_USE_PIO} ${PREQX_USE_ENERGY} ${QSIZE_D}) -TARGET_LINK_LIBRARIES(homme_tool dl) From 3ed833cbd7c22879489908d56ee1837719e8e69a Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 24 Dec 2021 16:36:46 -0700 Subject: [PATCH 242/301] fix wrong dimensions --- components/homme/src/theta-l/element_state.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/theta-l/element_state.F90 b/components/homme/src/theta-l/element_state.F90 index 7d57dc03a73d..ab3e37d040be 100644 --- a/components/homme/src/theta-l/element_state.F90 +++ b/components/homme/src/theta-l/element_state.F90 @@ -80,7 +80,7 @@ module element_state real (kind=real_kind) :: dp_ref(np,np,nlev) ! ref states based on PHIS real (kind=real_kind) :: theta_ref(np,np,nlev) real (kind=real_kind) :: phi_ref(np,np,nlevp) - real (kind=real_kind) :: lap_p_wk(np,np,nlevp) + real (kind=real_kind) :: lap_p_wk(np,np,nlev) end type derived_state_t From 4849b2e59d8b479f572b509ec187217beda5c42c Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 24 Dec 2021 16:37:06 -0700 Subject: [PATCH 243/301] typo in python parsing script --- components/homme/test/held_suarez0/parseE3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/homme/test/held_suarez0/parseE3.py b/components/homme/test/held_suarez0/parseE3.py index eb6c9408c64d..32637cc9d7df 100755 --- a/components/homme/test/held_suarez0/parseE3.py +++ b/components/homme/test/held_suarez0/parseE3.py @@ -84,8 +84,8 @@ def lookfor1(fid,key1,key2="",allow_eof=0): mumin.extend([float(str[0])]) mumax.extend([float(str[3])]) - # look for TS. return all zeros if we find ps first: - str = lookfor1(sys.stdin,"TS=","ps=",0) + # look for TBOT. return all zeros if we find ps first: + str = lookfor1(sys.stdin,"TBOT=","ps=",0) str=str.split() Tmin.extend([float(str[0])]) Tmax.extend([float(str[1])]) @@ -185,7 +185,7 @@ def lookfor1(fid,key1,key2="",allow_eof=0): plt.savefig("mu.png") plt.figure() - print ('plotting TS..std min,max=%f %f' % (np.std(Tmin),np.std(Tmax))) + print ('plotting TBOT..std min,max=%f %f' % (np.std(Tmin),np.std(Tmax))) print ('min,max=%f %f' % (min(Tmin),max(Tmax))) legend1=("min: %.3f std: %.4f" % (min(Tmin),np.std(Tmin)) ) plt.plot(time,Tmin,label=legend1) @@ -194,7 +194,7 @@ def lookfor1(fid,key1,key2="",allow_eof=0): plt.axis([min(time), max(min(time)+200,max(time)+10), 0,400]) plt.grid(True) plt.legend() - plt.savefig("TS.png") + plt.savefig("TBOT.png") if showplot: plt.show() From 9a8d580e489529127448015309c9620cdde0b951 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 21 Feb 2021 10:22:59 -0600 Subject: [PATCH 244/301] reorg vertical remap code and add new options vert_remap_q_alg=0 unchanged vert_remap_q_alg=1 unchanged vert_remap_q_alg=2 removed vert_remap_q_alg=3 removed (it was identical to 1) vert_remap_q_alg=10 unchanged vert_remap_q_alg=11 new: q_alg=10 with out limiters, for research [BFB] --- .../homme/src/preqx/share/vertremap_mod.F90 | 6 +- components/homme/src/share/control_mod.F90 | 11 +-- components/homme/src/share/vertremap_base.F90 | 90 +++++++++---------- .../homme/src/theta-l/share/vertremap_mod.F90 | 4 +- 4 files changed, 55 insertions(+), 56 deletions(-) diff --git a/components/homme/src/preqx/share/vertremap_mod.F90 b/components/homme/src/preqx/share/vertremap_mod.F90 index 3d3777751c41..bf75d16566f3 100644 --- a/components/homme/src/preqx/share/vertremap_mod.F90 +++ b/components/homme/src/preqx/share/vertremap_mod.F90 @@ -114,7 +114,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) ttmp(:,:,:,1)=ttmp(:,:,:,1)*dp_star call t_startf('vertical_remap1_1') - call remap1(ttmp,np,1,dp_star,dp) + call remap1(ttmp,np,1,dp_star,dp,vert_remap_q_alg) call t_stopf('vertical_remap1_1') elem(ie)%state%t(:,:,:,np1)=ttmp(:,:,:,1)/dp @@ -123,7 +123,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) ttmp(:,:,:,2)=elem(ie)%state%v(:,:,2,:,np1)*dp_star call t_startf('vertical_remap1_2') - call remap1(ttmp,np,2,dp_star,dp) + call remap1(ttmp,np,2,dp_star,dp,vert_remap_q_alg) call t_stopf('vertical_remap1_2') elem(ie)%state%v(:,:,1,:,np1)=ttmp(:,:,:,1)/dp @@ -141,7 +141,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) if (qsize>0 .and. np1_qdp > 0) then call t_startf('vertical_remap1_3') - call remap1(elem(ie)%state%Qdp(:,:,:,:,np1_qdp),np,qsize,dp_star,dp) + call remap1(elem(ie)%state%Qdp(:,:,:,:,np1_qdp),np,qsize,dp_star,dp,vert_remap_q_alg) call t_stopf('vertical_remap1_3') !dir$ simd diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 03bbd31f1fd3..495afe8f30b7 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -72,13 +72,14 @@ module control_mod ! interspace a lf-trapazoidal step every LFTfreq leapfrogs ! 0 = disabled -! vert_remap_q_alg: -1 remap without monotone filter, used for some test cases -! 0 default value, Zerroukat monotonic splines +! vert_remap_q_alg: -1 PPM remap without monotone filter, used for some test cases +! 0 Zerroukat monotonic splines ! 1 PPM vertical remap with mirroring at the boundaries ! (solid wall bc's, high-order throughout) -! 2 PPM vertical remap without mirroring at the boundaries -! (no bc's enforced, first-order at two cells bordering top and bottom boundaries) - integer, public :: vert_remap_q_alg = 0 +! 2 PPM, switching to piecewise constant at boundaries +! 10 PPM with linear extrapolation at boundaries, with column limiter +! 11 PPM with unlimited linear extrapolation ab boundaries + integer, public :: vert_remap_q_alg = 0 ! tracers ! advect theta 0: conservation form ! 1: expanded divergence form (less noisy, non-conservative) diff --git a/components/homme/src/share/vertremap_base.F90 b/components/homme/src/share/vertremap_base.F90 index da52b54bd7b1..631a947cfc98 100644 --- a/components/homme/src/share/vertremap_base.F90 +++ b/components/homme/src/share/vertremap_base.F90 @@ -33,7 +33,6 @@ module vertremap_base use hybvcoord_mod, only : hvcoord_t use perf_mod, only : t_startf, t_stopf ! _EXTERNAL use parallel_mod, only : abortmp, parallel_t - use control_mod, only : vert_remap_q_alg ! options: ! remap1 ! remap any field, splines, monotone @@ -75,7 +74,7 @@ end subroutine remap_calc_grids -subroutine remap1(Qdp,nx,qsize,dp1,dp2) +subroutine remap1(Qdp,nx,qsize,dp1,dp2,remap_alg) ! remap 1 field ! input: Qdp field to be remapped (NOTE: MASS, not MIXING RATIO) ! dp1 layer thickness (source) @@ -84,7 +83,7 @@ subroutine remap1(Qdp,nx,qsize,dp1,dp2) ! output: remaped Qdp, conserving mass, monotone on Q=Qdp/dp ! implicit none - integer, intent(in) :: nx,qsize + integer, intent(in) :: nx,qsize,remap_alg real (kind=real_kind), intent(inout) :: Qdp(nx,nx,nlev,qsize) real (kind=real_kind), intent(in) :: dp1(nx,nx,nlev),dp2(nx,nx,nlev) ! ======================== @@ -100,16 +99,16 @@ subroutine remap1(Qdp,nx,qsize,dp1,dp2) lt1,lt2,lt3,t0,t1,t2,t3,t4,tm,tp,ie,i,ilev,j,jk,k,q logical :: abrtf=.false. - q = vert_remap_q_alg - if ( (q.ne.-1) .and. (q.ne.0) .and. (q.ne.1) .and. (q.ne.2) .and. (q.ne.3) .and. (q.ne.10) )& - call abortmp('Bad vert_remap_q_alg value. Use -1, 0, 1, 2, 3, or 10.') + q = remap_alg + if ( (q.ne.-1) .and. (q.ne.0) .and. (q.ne.1) .and. (q.ne.10) .and. (q.ne.11) )& + call abortmp('Bad remap_alg value. Use -1, 0, 1, 10 or 11.') - if (vert_remap_q_alg == -1) then + if (remap_alg == -1) then call remap1_nofilter(qdp,nx,qsize,dp1,dp2) return endif - if (vert_remap_q_alg == 1 .or. vert_remap_q_alg == 2 .or. vert_remap_q_alg == 3 .or. vert_remap_q_alg == 10) then - call remap_Q_ppm(qdp,nx,qsize,dp1,dp2) + if (remap_alg >= 1) then + call remap_Q_ppm(qdp,nx,qsize,dp1,dp2,remap_alg) return endif @@ -510,11 +509,12 @@ end subroutine remap1_nofilter !This uses the exact same model and reference grids and data as remap_Q, but it interpolates !using PPM instead of splines. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! vert_remap_q_alg == 1 means mirrored values in ghost cells (i.e., no flux) -!! vert_remap_q_alg == 2 means piecewise constant in 2 cells near boundaries (don't use ghost cells) -!! vert_remap_q_alg == 3 means UKMO splines ghost cells (copy last value into all ghost cells) +!! remap_alg == 1 means mirrored values in ghost cells (i.e., no flux) +!! remap_alg == 2 means 1st order reconstruction in ghost cells +!! remap_alg == 10 linear extrapolation with global bounds preservation +!! remap_alg == 11 linear extrapolation !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) +subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,remap_alg) ! remap 1 field ! input: Qdp field to be remapped (NOTE: MASS, not MIXING RATIO) ! dp1 layer thickness (source) @@ -522,9 +522,8 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) ! ! output: remaped Qdp, conserving mass ! - use control_mod, only : vert_remap_q_alg implicit none - integer,intent(in) :: nx,qsize + integer,intent(in) :: nx,qsize, remap_alg real (kind=real_kind), intent(inout) :: Qdp(nx,nx,nlev,qsize) real (kind=real_kind), intent(in) :: dp1(nx,nx,nlev),dp2(nx,nx,nlev) ! Local Variables @@ -560,7 +559,7 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) !It makes sure there's an old interface value below the domain that is larger. pin(nlev+1) = pio(nlev+1) !The total mass in a column does not change. !Therefore, the pressure of that mass cannot either. - !Fill in the ghost regions with mirrored values. if vert_remap_q_alg is defined, this is of no consequence. + !Fill in the ghost regions with mirrored values. if remap_alg is defined, this is of no consequence. do k = 1 , gs dpo(1 -k) = dpo( k) dpo(nlev+k) = dpo(nlev+1-k) @@ -599,6 +598,7 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) !From here, we loop over tracers for only those portions which depend on tracer data, which includes PPM limiting and !mass accumulation + do q = 1 , qsize !Accumulate the old mass up to old grid cell interface locations to simplify integration !during remapping. Also, divide out the grid spacing so we're working with actual tracer @@ -610,26 +610,28 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) masso(k+1) = masso(k) + ao(k) !Accumulate the old mass. This will simplify the remapping ao(k) = ao(k) / dpo(k) !Divide out the old grid spacing because we want the tracer mixing ratio, not mass. enddo - !Fill in ghost values. Ignored if vert_remap_q_alg == 2 - if (vert_remap_q_alg >= 1 .and. vert_remap_q_alg <= 3) then + !Fill in ghost values. + if (remap_alg==10) then + ext(1) = minval(ao(1:nlev)) + ext(2) = maxval(ao(1:nlev)) + call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), 1,ext(1), ext(2)) + call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& + ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), 1, ext(1), ext(2)) + else if (remap_alg==11) then + call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), 0,ext(1), ext(2)) + call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& + ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), 0, ext(1), ext(2)) + else + !values for gs>1 are not used + !For remap_alg == 2 values might be used in computations that are then overwritten, + ! so they should be set but wont effect answer do k = 1 , gs - if (vert_remap_q_alg == 3) then - ao(1 -k) = ao(1) - ao(nlev+k) = ao(nlev) - elseif (vert_remap_q_alg == 2 .or. vert_remap_q_alg == 1)then !Ignored if vert_remap_q_alg == 2 - ao(1 -k) = ao( k) - ao(nlev+k) = ao(nlev+1-k) - endif - enddo - elseif (vert_remap_q_alg == 10) then - ext(1) = minval(ao(1:nlev)) - ext(2) = maxval(ao(1:nlev)) - call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), ext(1), ext(2)) - call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& - ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), ext(1), ext(2)) - endif + ao(1 -k) = ao(1) + ao(nlev+k) = ao(nlev) + enddo + endif !Compute monotonic and conservative PPM reconstruction over every cell - coefs(:,:) = compute_ppm( ao , ppmdx ) + coefs(:,:) = compute_ppm( ao , ppmdx, remap_alg ) !Compute tracer values on the new grid by integrating from the old cell bottom to the new !cell interface to form a new grid mass accumulation. Taking the difference between !accumulation at successive interfaces gives the mass inside each cell. Since Qdp is @@ -680,11 +682,11 @@ end function compute_ppm_grids !This computes a limited parabolic interpolant using a net 5-cell stencil, but the stages of computation are broken up into 3 stages -function compute_ppm( a , dx ) result(coefs) - use control_mod, only: vert_remap_q_alg +function compute_ppm( a , dx, remap_alg ) result(coefs) implicit none real(kind=real_kind), intent(in) :: a ( -1:nlev+2) !Cell-mean values real(kind=real_kind), intent(in) :: dx (10, 0:nlev+1) !grid spacings + integer :: remap_alg real(kind=real_kind) :: coefs(0:2, nlev ) !PPM coefficients (for parabola) real(kind=real_kind) :: ai (0:nlev ) !fourth-order accurate, then limited interface values real(kind=real_kind) :: dma(0:nlev+1) !An expression from Collela's '84 publication @@ -726,13 +728,6 @@ function compute_ppm( a , dx ) result(coefs) coefs(2,j) = 3. * (-2. * a(j) + ( al + ar )) enddo - !If vert_remap_q_alg == 2, use piecewise constant in the boundaries, and don't use ghost cells. - if (vert_remap_q_alg == 2) then - coefs(0,1:2) = a(1:2) - coefs(1:2,1:2) = 0. - coefs(0,nlev-1:nlev) = a(nlev-1:nlev) - coefs(1:2,nlev-1:nlev) = 0.D0 - endif end function compute_ppm !=======================================================================================================! @@ -778,12 +773,13 @@ subroutine binary_search(pio, pivot, k) end subroutine binary_search - subroutine linextrap(dx1,dx2,dx3,dx4,y1,y2,y3,y4,lo,hi) + subroutine linextrap(dx1,dx2,dx3,dx4,y1,y2,y3,y4,limit,lo,hi) real(kind=real_kind), intent(in) :: dx1,dx2,dx3,dx4,y1,y2,lo,hi real(kind=real_kind), intent(out) :: y3,y4 real(kind=real_kind) :: den,num,a real(kind=real_kind) :: z3,z4 + integer :: limit ! In exact arithmetic, the following is equivalent to ! x1 = half*dx1 @@ -806,8 +802,10 @@ subroutine linextrap(dx1,dx2,dx3,dx4,y1,y2,y3,y4,lo,hi) a = num/den y4 = (1-a)*y1 + a*y2 - y3 = max(lo, min(hi, y3)) - y4 = max(lo, min(hi, y4)) + if (limit==1) then + y3 = max(lo, min(hi, y3)) + y4 = max(lo, min(hi, y4)) + endif end subroutine linextrap diff --git a/components/homme/src/theta-l/share/vertremap_mod.F90 b/components/homme/src/theta-l/share/vertremap_mod.F90 index 8d5831184f9d..dc536908d859 100644 --- a/components/homme/src/theta-l/share/vertremap_mod.F90 +++ b/components/homme/src/theta-l/share/vertremap_mod.F90 @@ -123,7 +123,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) !ttmp(:,:,:,5)=ttmp(:,:,:,5) !*dp_star call t_startf('vertical_remap1_1') - call remap1(ttmp,np,5,dp_star,dp) + call remap1(ttmp,np,5,dp_star,dp,vert_remap_q_alg) call t_stopf('vertical_remap1_1') elem(ie)%state%v(:,:,1,:,np1)=ttmp(:,:,:,1)/dp @@ -154,7 +154,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) if (qsize>0 .and. np1_qdp > 0) then call t_startf('vertical_remap1_3') - call remap1(elem(ie)%state%Qdp(:,:,:,:,np1_qdp),np,qsize,dp_star,dp) + call remap1(elem(ie)%state%Qdp(:,:,:,:,np1_qdp),np,qsize,dp_star,dp,vert_remap_q_alg) call t_stopf('vertical_remap1_3') !dir$ simd From 790bc2c28696272df60912f1ccc669cabdaaa162 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 26 Dec 2021 17:40:57 -0600 Subject: [PATCH 245/301] add remap alg option to SL code --- components/homme/src/share/sl_advection.F90 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/homme/src/share/sl_advection.F90 b/components/homme/src/share/sl_advection.F90 index f2c5c2b35ed7..8f360d10faf4 100644 --- a/components/homme/src/share/sl_advection.F90 +++ b/components/homme/src/share/sl_advection.F90 @@ -9,7 +9,8 @@ module sl_advection use element_mod, only : element_t use hybvcoord_mod, only : hvcoord_t use time_mod, only : TimeLevel_t, TimeLevel_Qdp - use control_mod, only : integration, test_case, hypervis_order, transport_alg, limiter_option + use control_mod, only : integration, test_case, hypervis_order, transport_alg, limiter_option,& + vert_remap_q_alg use edge_mod, only : edgevpack_nlyr, edgevunpack_nlyr, edge_g use edgetype_mod, only : EdgeDescriptor_t, EdgeBuffer_t use hybrid_mod, only : hybrid_t @@ -328,7 +329,8 @@ subroutine calc_trajectory(elem, deriv, hvcoord, hybrid, dt, tl, & deriv, dp_tol, elem(ie)%derived%divdp) wr(:,:,:,1) = elem(ie)%derived%vn0(:,:,1,:)*elem(ie)%state%dp3d(:,:,:,tl%np1) wr(:,:,:,2) = elem(ie)%derived%vn0(:,:,2,:)*elem(ie)%state%dp3d(:,:,:,tl%np1) - call remap1(wr, np, 2, elem(ie)%state%dp3d(:,:,:,tl%np1), elem(ie)%derived%divdp) + call remap1(wr, np, 2, elem(ie)%state%dp3d(:,:,:,tl%np1), elem(ie)%derived%divdp,& + vert_remap_q_alg) elem(ie)%derived%vn0(:,:,1,:) = wr(:,:,:,1)/elem(ie)%derived%divdp elem(ie)%derived%vn0(:,:,2,:) = wr(:,:,:,2)/elem(ie)%derived%divdp end do @@ -1063,7 +1065,7 @@ subroutine sl_vertically_remap_tracers(hybrid, elem, nets, nete, tl, dt_q) end if #endif call remap1(elem(ie)%state%Qdp(:,:,:,:,np1_qdp), np, qsize, elem(ie)%derived%divdp, & - elem(ie)%state%dp3d(:,:,:,tl%np1)) + elem(ie)%state%dp3d(:,:,:,tl%np1),vert_remap_q_alg) do q = 1,qsize elem(ie)%state%Q(:,:,:,q) = elem(ie)%state%Qdp(:,:,:,q,np1_qdp)/ & elem(ie)%state%dp3d(:,:,:,tl%np1) From a417b5495e5a743707cddd3054cbd49c170dfef6 Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 29 Dec 2021 15:30:01 -0800 Subject: [PATCH 246/301] remove redundant entry --- components/eam/cime_config/config_compsets.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index e9009dd7ed27..acbcc57a6418 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -271,13 +271,6 @@ 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV - - - - F20TR-MMFXX-CMIP6 - 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV - - From 2d1dd1fd57ce7ce5a35fa4f6e70b6faf89b5e16c Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 29 Dec 2021 15:32:18 -0800 Subject: [PATCH 247/301] clean up use case file for MMF AMIP --- .../use_cases/20TR_MMF-1mom_CMIP6.xml | 55 +------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml index 681cb2d7d86f..32dbab8537e0 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml @@ -4,16 +4,7 @@ .true. - - - - - - .false. - - - atm/cam/solar/Solar_1850-2299_input4MIPS_c20181106.nc @@ -41,55 +32,11 @@ CYCLICAL 01 - - - - - - - - --> - - - - - - - - - - - - --> - - - - - - - - - - - - - - - - - --> - - - - atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc SERIAL - - - - + 'H2O2', 'H2SO4', 'SO2' From 37c20c2f79a1a0c48c08dc603eb9cc1cf57e7778 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 1 Jan 2022 14:21:21 -0800 Subject: [PATCH 248/301] Removes BPRP compset, adds Land-atm compsets and addresses comments [BFB] - Bit-For-Bit --- cime_config/allactive/config_compsets.xml | 35 ++++++++----------- components/eam/bld/build-namelist | 3 -- .../cime_config/config_component.xml | 3 -- .../cime_config/config_component_e3sm.xml | 2 -- 4 files changed, 15 insertions(+), 28 deletions(-) diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index c31d6420ed0a..3aee1eb1bc1c 100644 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -99,11 +99,6 @@ 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BCRC - - BGCEXP_BPRP_CNPRDCTC_1850 - 1850_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP - - BGCEXP_BCRC_CNPRDCTC_20TR 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC @@ -124,16 +119,6 @@ SSP585SOI_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BCRC - - BGCEXP_BPRP_CNPRDCTC_20TR - 20TR_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP - - - - BGCEXP_BPRP_CNPRDCTC_SSP585_CMIP6 - SSP585_CAM5%CMIP6_CLM45%CNPRDCTCBC_MPASCICE%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BPRP - - BGCEXP_BCRD_CNPRDCTC_20TR 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRD @@ -204,6 +189,11 @@ 1850SOI_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BCRC + + BGCEXP_LNDATM_CNORDCTC_1850 + 1850_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV_BGC%LNDATM + + BGCEXP_BCRC_CNPECACNT_20TR 20TR_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECOOIDMS_MOSART_SGLC_SWAV_BGC%BCRC @@ -284,6 +274,16 @@ SSP585SOI_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BDRD + + BGCEXP_LNDATM_CNORDCTC_20TR + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV_BGC%LNDATM + + + + BGCEXP_LNDATM_CNORDCTC_20TR + 20TR_EAM%CMIP6_ELM%CNPRDCTCBC_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV_BGC%LNDATM + + @@ -312,11 +312,6 @@ 20TRSOI_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BDRD - - BGCEXP_BDRD_CNPECACNT_SSP585_CMIP6S - SSP585SOI_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI_MPASO_MOSART_SGLC_SWAV_BGC%BDRD - - diff --git a/components/eam/bld/build-namelist b/components/eam/bld/build-namelist index 2c62ea028881..b897386fecab 100755 --- a/components/eam/bld/build-namelist +++ b/components/eam/bld/build-namelist @@ -937,9 +937,6 @@ if ($co2_cycle) { add_default($nl, 'aircraft_datapath'); add_default($nl, 'aircraft_type'); - # This should be the same file as the one in the aircraft_specifier file. - # This is a workaround to get this filepath into the cam.input_data_list file - # to allow the CESM scripts to obtain all required data for a run. } } } diff --git a/components/mpas-ocean/cime_config/config_component.xml b/components/mpas-ocean/cime_config/config_component.xml index 71a9f91ab4f2..9133731b3332 100644 --- a/components/mpas-ocean/cime_config/config_component.xml +++ b/components/mpas-ocean/cime_config/config_component.xml @@ -88,17 +88,14 @@ diagnostic constant constant - prognostic bcrc bcrd bdrc bdrd - prognostic bcrc bcrd bdrc bdrd - prognostic case_comp env_case.xml diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 0e44028d6c83..7ffb7e1cb83f 100644 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -262,12 +262,10 @@ CO2A CO2A_OI CO2A_OI - CO2C CO2C CO2C CO2C CO2C - CO2C_OI CO2C_OI CO2C_OI CO2C_OI From 740620f5603941a6eafa50433a92158fc820df35 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Sat, 1 Jan 2022 14:33:40 -0800 Subject: [PATCH 249/301] Adds 20TR and 1850 tests for BGC --- cime_config/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 2424643a9260..08588bd53805 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -68,6 +68,8 @@ "PEM_Ln5.ne4_oQU240.F2010", "SMS_D_Ln5.ne4_oQU240.F2010.eam-cosplite_nhtfrq5", "SMS_Ln1.ne4_oQU240.F2010.eam-chem_pp", + "SMS_Ln5.ne30pg2_EC30to60E2r2.BGCEXP_LNDATM_CNORDCTC_20TR", + "SMS_Ln5.ne30pg2_EC30to60E2r2.BGCEXP_LNDATM_CNORDCTC_1850", "SMS_D_Ln5.ne4_oQU240.F2010.eam-clubb_sp", "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgp", "ERS_Ld5.ne4_oQU240.F2010.eam-rrtmgpxx", From 4c6b9f980944811101b4df5bccd352a380ffd38e Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 27 Dec 2021 18:38:58 -0600 Subject: [PATCH 250/301] update comments --- components/homme/src/share/control_mod.F90 | 6 ++---- components/homme/src/share/vertremap_base.F90 | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 495afe8f30b7..a41abc5b4445 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -74,11 +74,9 @@ module control_mod ! vert_remap_q_alg: -1 PPM remap without monotone filter, used for some test cases ! 0 Zerroukat monotonic splines -! 1 PPM vertical remap with mirroring at the boundaries -! (solid wall bc's, high-order throughout) -! 2 PPM, switching to piecewise constant at boundaries +! 1 PPM vertical remap with constant extension at the boundaries ! 10 PPM with linear extrapolation at boundaries, with column limiter -! 11 PPM with unlimited linear extrapolation ab boundaries +! 11 PPM with unlimited linear extrapolation at boundaries integer, public :: vert_remap_q_alg = 0 ! tracers ! advect theta 0: conservation form diff --git a/components/homme/src/share/vertremap_base.F90 b/components/homme/src/share/vertremap_base.F90 index 631a947cfc98..d9e8e16ce344 100644 --- a/components/homme/src/share/vertremap_base.F90 +++ b/components/homme/src/share/vertremap_base.F90 @@ -622,9 +622,6 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,remap_alg) call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), 0, ext(1), ext(2)) else - !values for gs>1 are not used - !For remap_alg == 2 values might be used in computations that are then overwritten, - ! so they should be set but wont effect answer do k = 1 , gs ao(1 -k) = ao(1) ao(nlev+k) = ao(nlev) From eba902c83a3343b3fa919dab021a4f0220ca7978 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Wed, 29 Dec 2021 00:08:08 -0600 Subject: [PATCH 251/301] update test case to test q_alg=11 --- components/homme/test/reg_test/namelists/thetah-test22.nl | 1 - components/homme/test/reg_test/namelists/thetanh-TC.nl | 2 ++ components/homme/test/reg_test/namelists/thetanh-test22.nl | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/homme/test/reg_test/namelists/thetah-test22.nl b/components/homme/test/reg_test/namelists/thetah-test22.nl index 680d2dde9340..c51af9d8b997 100644 --- a/components/homme/test/reg_test/namelists/thetah-test22.nl +++ b/components/homme/test/reg_test/namelists/thetah-test22.nl @@ -17,7 +17,6 @@ tstep = 0.1 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 1 ! 1 => default method - smooth = 0.05 ! timestep smooting nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 nu_s = 3.2e7 nu_p = 3.2e7 diff --git a/components/homme/test/reg_test/namelists/thetanh-TC.nl b/components/homme/test/reg_test/namelists/thetanh-TC.nl index 1b7e9f0ab859..22b242ab51e4 100644 --- a/components/homme/test/reg_test/namelists/thetanh-TC.nl +++ b/components/homme/test/reg_test/namelists/thetanh-TC.nl @@ -17,6 +17,8 @@ tstep_type = 9 rsplit = 2 qsplit = 2 + vert_remap_q_alg = 10 + vert_remap_u_alg = 11 nu = 3e16 ! default= 1e15*(ne30/ne8)**3.2 = 6.9e16 nu_s = 3e16 nu_p = 3e16 diff --git a/components/homme/test/reg_test/namelists/thetanh-test22.nl b/components/homme/test/reg_test/namelists/thetanh-test22.nl index f3892b575c4e..fa3db72ae512 100644 --- a/components/homme/test/reg_test/namelists/thetanh-test22.nl +++ b/components/homme/test/reg_test/namelists/thetanh-test22.nl @@ -17,7 +17,6 @@ tstep = 0.1 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 1 ! 1 => default method - smooth = 0.05 ! timestep smooting nu = 3.2e7 ! reduced planet hyperviscosity hv/500^3 nu_s = 3.2e7 nu_p = 3.2e7 From de9d2d0a0368357af0459f616d5f3b3f339f17be Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 2 Jan 2022 12:17:46 -0700 Subject: [PATCH 252/301] add vert_remap_u_alg option --- components/homme/src/share/control_mod.F90 | 1 + components/homme/src/share/namelist_mod.F90 | 10 ++++++++-- components/homme/src/theta-l/share/vertremap_mod.F90 | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index a41abc5b4445..ff8ce8861020 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -78,6 +78,7 @@ module control_mod ! 10 PPM with linear extrapolation at boundaries, with column limiter ! 11 PPM with unlimited linear extrapolation at boundaries integer, public :: vert_remap_q_alg = 0 ! tracers + integer, public :: vert_remap_u_alg = -2 ! remap for dynamics. default -2 means inherit vert_remap_q_alg ! advect theta 0: conservation form ! 1: expanded divergence form (less noisy, non-conservative) diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 2623adc27469..f7a50f9e09ac 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -101,6 +101,7 @@ module namelist_mod hv_theta_correction, & hv_theta_thresh, & vert_remap_q_alg, & + vert_remap_u_alg, & se_fv_phys_remap_alg, & timestep_make_subcycle_parameters_consistent @@ -310,6 +311,7 @@ subroutine readnl(par) hv_theta_correction, & hv_theta_thresh, & vert_remap_q_alg, & + vert_remap_u_alg, & se_fv_phys_remap_alg @@ -781,6 +783,7 @@ subroutine readnl(par) call MPI_bcast(hv_theta_correction,1, MPIinteger_t, par%root,par%comm,ierr) call MPI_bcast(hv_theta_thresh,1, MPIreal_t, par%root,par%comm,ierr) call MPI_bcast(vert_remap_q_alg,1, MPIinteger_t, par%root,par%comm,ierr) + call MPI_bcast(vert_remap_u_alg,1, MPIinteger_t, par%root,par%comm,ierr) call MPI_bcast(fine_ne, 1, MPIinteger_t, par%root,par%comm,ierr) call MPI_bcast(max_hypervis_courant,1,MPIreal_t, par%root,par%comm,ierr) @@ -940,7 +943,9 @@ subroutine readnl(par) endif endif #endif - + ! set defautl for dynamics remap + if (vert_remap_u_alg == -2) vert_remap_u_alg = vert_remap_q_alg + ! more thread error checks: #ifdef HORIZ_OPENMP if(par%masterproc) write(iulog,*)'-DHORIZ_OPENMP enabled' @@ -1173,8 +1178,9 @@ subroutine readnl(par) if (hv_ref_profiles==0 .and. hv_theta_correction==1) then call abortmp("hv_theta_correction=1 requires hv_ref_profiles=1 or 2") endif - + write(iulog,*)"readnl: vert_remap_q_alg = ",vert_remap_q_alg + write(iulog,*)"readnl: vert_remap_u_alg = ",vert_remap_u_alg #ifdef CAM write(iulog,*)"readnl: se_nsplit = ", NSPLIT write(iulog,*)"readnl: se_tstep = ", tstep diff --git a/components/homme/src/theta-l/share/vertremap_mod.F90 b/components/homme/src/theta-l/share/vertremap_mod.F90 index dc536908d859..b14a413d64bc 100644 --- a/components/homme/src/theta-l/share/vertremap_mod.F90 +++ b/components/homme/src/theta-l/share/vertremap_mod.F90 @@ -11,7 +11,7 @@ module vertremap_mod use element_mod, only : element_t use perf_mod, only : t_startf, t_stopf ! _EXTERNAL use parallel_mod, only : abortmp, parallel_t - use control_mod, only : vert_remap_q_alg + use control_mod, only : vert_remap_q_alg,vert_remap_u_alg use eos, only : phi_from_eos implicit none private @@ -123,7 +123,7 @@ subroutine vertical_remap(hybrid,elem,hvcoord,dt,np1,np1_qdp,nets,nete) !ttmp(:,:,:,5)=ttmp(:,:,:,5) !*dp_star call t_startf('vertical_remap1_1') - call remap1(ttmp,np,5,dp_star,dp,vert_remap_q_alg) + call remap1(ttmp,np,5,dp_star,dp,vert_remap_u_alg) call t_stopf('vertical_remap1_1') elem(ie)%state%v(:,:,1,:,np1)=ttmp(:,:,:,1)/dp From 3053da27b7d69f35357ba3347847bf39b2c04969 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 2 Jan 2022 17:42:55 -0600 Subject: [PATCH 253/301] remove hypervis_power (more) --- .../test_execs/share_kokkos_ut/sphere_op_interface.F90 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 b/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 index 30bb747d60c8..3f11db4d8b27 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_interface.F90 @@ -206,7 +206,7 @@ subroutine laplace_sphere_wk_c_callable(s,dvv,dinv,spheremp,tensorVisc,& ! ! Inputs ! - use control_mod, only: hypervis_power, hypervis_scaling + use control_mod, only: hypervis_scaling real(kind=real_kind), intent(in) :: s(np,np) real(kind=real_kind), intent(in) :: dvv(np, np) real(kind=real_kind), intent(in) :: dinv(np, np, 2, 2) @@ -224,7 +224,6 @@ subroutine laplace_sphere_wk_c_callable(s,dvv,dinv,spheremp,tensorVisc,& scale_factor_inv = rrearth !redefining params from control_mod, not the usual homme practice, but... - hypervis_power = hvpower hypervis_scaling = hvscaling deriv%dvv = dvv elem%Dinv = Dinv @@ -271,7 +270,7 @@ end subroutine laplace_simple_c_callable subroutine vlaplace_sphere_wk_cartesian_c_callable(v, dvv, dinv, spheremp, & tensorVisc, vec_sph2cart, hvpower, hvscaling, var_coef, laplace) bind(c) use derivative_mod_base, only : vlaplace_sphere_wk_cartesian - use control_mod, only : hypervis_power, hypervis_scaling + use control_mod, only : hypervis_scaling ! ! Inputs @@ -294,7 +293,6 @@ subroutine vlaplace_sphere_wk_cartesian_c_callable(v, dvv, dinv, spheremp, & laplacian_rigid_factor = rrearth !redefining params from control_mod, not the usual homme practice, but... - hypervis_power = hvpower hypervis_scaling = hvscaling deriv%dvv = dvv elem%Dinv = Dinv From 745a96e877220a8e218237e80e379203202cdf94 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 2 Jan 2022 17:48:57 -0600 Subject: [PATCH 254/301] check for ncl in runall script --- components/homme/dcmip_tests/runall.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/homme/dcmip_tests/runall.sh b/components/homme/dcmip_tests/runall.sh index 64f099f3a860..710e75ef78ad 100755 --- a/components/homme/dcmip_tests/runall.sh +++ b/components/homme/dcmip_tests/runall.sh @@ -66,6 +66,13 @@ if [ "$#" -lt "2" ]; then echo "./homme.sh /path/to/output/dir {submit,status,latex} testno" exit 1 fi + +if ! which ncl >&/dev/null > /dev/null ; then + echo "'which ncl' failed. NCL is required to produce PDFs. Exiting." + exit 1 +fi + + WDIR=${args[0]} opt=${args[1]} From 29bb14fdf7962effe2503731dbd3b1bf6144eba7 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 2 Jan 2022 18:28:40 -0600 Subject: [PATCH 255/301] update unit tests for interface changes --- .../homme/test_execs/thetal_kokkos_ut/caar_interface.F90 | 2 +- .../homme/test_execs/thetal_kokkos_ut/dirk_interface.F90 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/homme/test_execs/thetal_kokkos_ut/caar_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/caar_interface.F90 index 4b737903d570..346dc1a23d3c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/caar_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/caar_interface.F90 @@ -95,7 +95,7 @@ subroutine run_caar_f90 (nm1, n0, np1, dt, eta_ave_w, scale1, scale2, scale3, & theta_advect_form = 1 endif - call compute_andor_apply_rhs(np1,nm1,n0,1,dt, & + call compute_andor_apply_rhs(np1,nm1,n0,dt, & elem,hvcoord,hybrid,deriv, & 1,nelemd,.false.,eta_ave_w, & scale1,scale2,scale3) diff --git a/components/homme/test_execs/thetal_kokkos_ut/dirk_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/dirk_interface.F90 index 84e91c6994ae..33d9122d0ef6 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/dirk_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/dirk_interface.F90 @@ -182,13 +182,13 @@ subroutine compute_stage_value_dirk_f90(nm1,alphadt_nm1,n0,alphadt_n0,np1,dt2) b ! compute_stage_value_dirk doesn't actually use these. type (hybrid_t) :: hybrid type (derivative_t) :: deriv - integer :: qn0, itercount, nets, nete + integer :: itercount, nets, nete real (kind=real_kind) :: itererr nets = 1 nete = size(elem) call compute_stage_value_dirk(nm1, alphadt_nm1, n0, alphadt_n0, np1, dt2, & - qn0, elem, hvcoord, hybrid, deriv, nets, nete, itercount, itererr, & + elem, hvcoord, hybrid, deriv, nets, nete, itercount, itererr, & verbosity_in=0) end subroutine compute_stage_value_dirk_f90 From 1ee0965aad15379f9ba85f6571034e96b46b563e Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Mon, 3 Jan 2022 13:40:47 -0600 Subject: [PATCH 256/301] Remove oEC60to30v3 eco option, since it will no longer work --- components/mpas-ocean/cime_config/buildnml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 573bda96c61e..3d3f092ccac4 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -84,11 +84,6 @@ def buildnml(case, caseroot, compname): if ocn_ic_mode == 'spunup': ic_date = '200927' ic_prefix = 'oEC60to30v3.restartFrom_anvilG' - # modify initial condition file if ocean BGC is turned on - if ocn_bgc in ['eco_only', 'eco_and_dms', 'eco_and_macromolecules', 'eco_and_dms_and_macromolecules']: - ic_date = '180418' - ic_prefix = 'mpaso.rst.BCRC_CNPECACNT_1850_anvil02.0066-01-01.no-xtime' - eco_forcing_file = 'oEC60to30v3.monthlySurfaceForcingBGC.171120.nc' elif ocn_grid == 'oEC60to30v3wLI': decomp_date = '170328' From 60a5bdff4667d763e3e1bf41e5fffad0b09c1b18 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Mon, 3 Jan 2022 15:16:43 -0600 Subject: [PATCH 257/301] Update tests and testdefs to reflect new compsets and files --- cime_config/tests.py | 4 ++-- .../testdefs/testmods_dirs/elm/bgcexp/user_nl_mpassi | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 components/elm/cime_config/testdefs/testmods_dirs/elm/bgcexp/user_nl_mpassi diff --git a/cime_config/tests.py b/cime_config/tests.py index 54ba83135c42..4961637d3a35 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -166,8 +166,8 @@ "PEM_Ln9.ne30pg2_EC30to60E2r2.WCYCL1850", "ERP_Ld3.ne30pg2_EC30to60E2r2.WCYCL1850.allactive-pioroot1", "SMS_D_Ln5.conusx4v1_r05_oECv3.F2010", - "SMS_Ld2.ne30_oECv3.BGCEXP_CNTL_CNPECACNT_1850.elm-bgcexp", - "SMS_Ld2.ne30_oECv3.BGCEXP_CNTL_CNPRDCTC_1850.elm-bgcexp", + "SMS_Ld2.ne30pg2_r05_EC30to60E2r2.BGCEXP_CNTL_CNPECACNT_1850.elm-bgcexp", + "SMS_Ld2.ne30pg2_r05_EC30to60E2r2.BGCEXP_CNTL_CNPRDCTC_1850.elm-bgcexp", "SMS_D_Ld1.T62_oEC60to30v3.DTESTM", "SMS_D_Ld1.ne30pg2_r05_EC30to60E2r2.WCYCL1850", "ERP_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_fixed_subcycle", diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/bgcexp/user_nl_mpassi b/components/elm/cime_config/testdefs/testmods_dirs/elm/bgcexp/user_nl_mpassi deleted file mode 100644 index c3e0b678c0cc..000000000000 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/bgcexp/user_nl_mpassi +++ /dev/null @@ -1 +0,0 @@ -config_nsnowlayers = 1 From 22cc2f7249baeb563c5c3503241e84c6e9137641 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 4 Jan 2022 14:46:58 -0800 Subject: [PATCH 258/301] Updating to latest api 17 fates science tag In order to include the IBMfix branch tag commit, it is necessary to update to sci.1.49.2_api.17.0.0 at the least. Since the sci.1.15.0_api.17.0.0 tag is the last science update prior to a new api update, this one was selected. --- components/elm/src/external_models/fates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index f32d831ff174..38708294635e 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit f32d831ff17422653991ae578e1b74357e2ddd7f +Subproject commit 38708294635e62c98538e18249bbbfdbc066ada9 From ddebdcb09795a368b998e767a89e9cced9e4f64a Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Tue, 4 Jan 2022 17:44:32 -0600 Subject: [PATCH 259/301] improve 10/11 logic --- components/homme/src/share/vertremap_base.F90 | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/components/homme/src/share/vertremap_base.F90 b/components/homme/src/share/vertremap_base.F90 index d9e8e16ce344..ffc512a96cc7 100644 --- a/components/homme/src/share/vertremap_base.F90 +++ b/components/homme/src/share/vertremap_base.F90 @@ -539,6 +539,7 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,remap_alg) real(kind=real_kind) :: ppmdx(10,0:nlev+1) !grid spacings real(kind=real_kind) :: mymass, massn1, massn2, ext(2) integer :: i, j, k, q, kk, kid(nlev) + logical :: limit_linear_extrap call t_startf('remap_Q_ppm') do j = 1 , nx @@ -611,16 +612,17 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,remap_alg) ao(k) = ao(k) / dpo(k) !Divide out the old grid spacing because we want the tracer mixing ratio, not mass. enddo !Fill in ghost values. - if (remap_alg==10) then - ext(1) = minval(ao(1:nlev)) - ext(2) = maxval(ao(1:nlev)) - call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), 1,ext(1), ext(2)) - call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& - ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), 1, ext(1), ext(2)) - else if (remap_alg==11) then - call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), 0,ext(1), ext(2)) + + + if (remap_alg == 10 .or. remap_alg == 11) then + limit_linear_extrap = (remap_alg == 10) + if (limit_linear_extrap) then + ext(1) = minval(ao(1:nlev)) + ext(2) = maxval(ao(1:nlev)) + end if + call linextrap(dpo(2), dpo(1), dpo(0), dpo(-1), ao(2), ao(1), ao(0), ao(-1), limit_linear_extrap, ext(1), ext(2)) call linextrap(dpo(nlev-1), dpo(nlev), dpo(nlev+1), dpo(nlev+2),& - ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), 0, ext(1), ext(2)) + ao(nlev-1), ao(nlev), ao(nlev+1), ao(nlev+2), limit_linear_extrap, ext(1), ext(2)) else do k = 1 , gs ao(1 -k) = ao(1) @@ -776,7 +778,7 @@ subroutine linextrap(dx1,dx2,dx3,dx4,y1,y2,y3,y4,limit,lo,hi) real(kind=real_kind) :: den,num,a real(kind=real_kind) :: z3,z4 - integer :: limit + logical :: limit ! In exact arithmetic, the following is equivalent to ! x1 = half*dx1 @@ -799,7 +801,7 @@ subroutine linextrap(dx1,dx2,dx3,dx4,y1,y2,y3,y4,limit,lo,hi) a = num/den y4 = (1-a)*y1 + a*y2 - if (limit==1) then + if (limit) then y3 = max(lo, min(hi, y3)) y4 = max(lo, min(hi, y4)) endif From 13f6b81413374bd138ac85e04e052109187a8e68 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 4 Jul 2021 14:21:25 -0500 Subject: [PATCH 260/301] add simplified sponge layer including namelist option to specify sponge layer bottom/start --- .../namelist_files/namelist_defaults_eam.xml | 1 + .../namelist_files/namelist_definition.xml | 6 +++ components/homme/src/share/control_mod.F90 | 1 + components/homme/src/share/namelist_mod.F90 | 4 ++ .../src/theta-l/share/model_init_mod.F90 | 43 ++++++++----------- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/components/eam/bld/namelist_files/namelist_defaults_eam.xml b/components/eam/bld/namelist_files/namelist_defaults_eam.xml index 9e08cec5be19..e2f9144e6748 100755 --- a/components/eam/bld/namelist_files/namelist_defaults_eam.xml +++ b/components/eam/bld/namelist_files/namelist_defaults_eam.xml @@ -1245,6 +1245,7 @@ + 0d0 2.5e5 1e5 4.3e4 diff --git a/components/eam/bld/namelist_files/namelist_definition.xml b/components/eam/bld/namelist_files/namelist_definition.xml index da4b65204003..362374c4f716 100644 --- a/components/eam/bld/namelist_files/namelist_definition.xml +++ b/components/eam/bld/namelist_files/namelist_definition.xml @@ -5784,6 +5784,12 @@ Second-order viscosity applied only near the model top [m^2/s]. Default: Set by build-namelist. + +Bottom of sponge layer in hPa. +Default: 0 (use default value based on reference pressure at model top). + + baseline ne for scalar hypervis tuning diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 03bbd31f1fd3..8847072593a8 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -161,6 +161,7 @@ module control_mod real (kind=real_kind), public :: nu_q = -1 ! default = nu tracer viscosity real (kind=real_kind), public :: nu_p = -1 ! default = nu ps equ. viscosity real (kind=real_kind), public :: nu_top = 0.0D5 ! top-of-the-model viscosity + real (kind=real_kind), public :: tom_sponge_start=0 ! start of sponge layer, in hPa integer, public :: hypervis_subcycle=1 ! number of subcycles for hyper viscsosity timestep integer, public :: hypervis_subcycle_tom=0 ! number of subcycles for TOM diffusion diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 2623adc27469..7c1c5cbb0a11 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -64,6 +64,7 @@ module namelist_mod nu_div, & nu_p, & nu_top, & + tom_sponge_start, & dcmip16_mu, & dcmip16_mu_s, & dcmip16_mu_q, & @@ -284,6 +285,7 @@ subroutine readnl(par) nu_div, & nu_p, & nu_top, & + tom_sponge_start, & dcmip16_mu, & dcmip16_mu_s, & dcmip16_mu_q, & @@ -790,6 +792,7 @@ subroutine readnl(par) call MPI_bcast(nu_div, 1, MPIreal_t , par%root,par%comm,ierr) call MPI_bcast(nu_p, 1, MPIreal_t , par%root,par%comm,ierr) call MPI_bcast(nu_top, 1, MPIreal_t , par%root,par%comm,ierr) + call MPI_bcast(tom_sponge_start,1, MPIreal_t , par%root,par%comm,ierr) call MPI_bcast(dcmip16_mu, 1, MPIreal_t , par%root,par%comm,ierr) call MPI_bcast(dcmip16_mu_s, 1, MPIreal_t , par%root,par%comm,ierr) @@ -1211,6 +1214,7 @@ subroutine readnl(par) write(iulog,'(a,2e9.2)')"viscosity: nu_q = ",nu_q write(iulog,'(a,2e9.2)')"viscosity: nu_p = ",nu_p write(iulog,'(a,2e9.2)')"viscosity: nu_top = ",nu_top + write(iulog,'(a,2e9.2)')"viscosity: tom_sponge_start = ",tom_sponge_start if(dcmip16_mu/=0) write(iulog,'(a,2e9.2)')"1st order viscosity: dcmip16_mu = ",dcmip16_mu if(dcmip16_mu_s/=0)write(iulog,'(a,2e9.2)')"1st order viscosity: dcmip16_mu_s = ",dcmip16_mu_s diff --git a/components/homme/src/theta-l/share/model_init_mod.F90 b/components/homme/src/theta-l/share/model_init_mod.F90 index 9d3c1169b7a8..6da080b0d74e 100644 --- a/components/homme/src/theta-l/share/model_init_mod.F90 +++ b/components/homme/src/theta-l/share/model_init_mod.F90 @@ -23,7 +23,7 @@ module model_init_mod use viscosity_mod, only: make_c0_vector use kinds, only: real_kind,iulog use control_mod, only: qsplit,theta_hydrostatic_mode, hv_ref_profiles, & - hv_theta_correction + hv_theta_correction, tom_sponge_start use time_mod, only: timelevel_qdp, timelevel_t use physical_constants, only: g, TREF, Rgas, kappa use imex_mod, only: test_imex_jacobian @@ -121,35 +121,28 @@ subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) nlev_tom=0 if (hybrid%masterthread) write(iulog,*) "sponge layer nu_top viscosity scaling factor" do k=1,nlev - !press = (hvcoord%hyam(k)+hvcoord%hybm(k))*hvcoord%ps0 - !ptop = hvcoord%hyai(1)*hvcoord%ps0 - ! sponge layer starts at p=4*ptop - ! - ! some test cases have ptop=200mb - if (hvcoord%etai(1)==0) then - ! pure sigma coordinates could have etai(1)=0 - ptop_over_press = hvcoord%etam(1) / hvcoord%etam(k) + if (tom_sponge_start==0) then + ! some test cases have ptop=200mb + if (hvcoord%etai(1)==0) then + ! pure sigma coordinates could have etai(1)=0 + ptop_over_press = hvcoord%etam(1) / hvcoord%etam(k) + else + ptop_over_press = hvcoord%etai(1) / hvcoord%etam(k) + endif + ! active for p<10*ptop (following cd_core.F90 in CAM-FV) + ! CAM 26L and 30L: top 3 levels + ! E3SM 72L: top 6 levels + !original cam formula + !nu_scale_top(k) = 8*(1+tanh(log(ptop_over_press))) ! active for p<4*ptop + nu_scale_top(k) = 16*ptop_over_press**2 / (ptop_over_press**2 + 1) else - ptop_over_press = hvcoord%etai(1) / hvcoord%etam(k) + ptop_over_press = (tom_sponge_start/1d3) / hvcoord%etam(k) + nu_scale_top(k)=0.15d0 * ptop_over_press**2 endif - ! active for p<10*ptop (following cd_core.F90 in CAM-FV) - ! CAM 26L and 30L: top 3 levels - ! E3SM 72L: top 6 levels - !original cam formula - !nu_scale_top(k) = 8*(1+tanh(log(ptop_over_press))) ! active for p<4*ptop - nu_scale_top(k) = 16*ptop_over_press**2 / (ptop_over_press**2 + 1) - if (nu_scale_top(k)<0.15d0) nu_scale_top(k)=0 + if (nu_scale_top(k)>8d0) nu_scale_top(k)=8d0 - !nu_scale_top(k) = 8*(1+.911*tanh(log(ptop_over_press))) ! active for p<6.5*ptop - !if (nu_scale_top(k)<1d0) nu_scale_top(k)=0 - - ! original CAM3/preqx formula - !if (k==1) nu_scale_top(k)=4 - !if (k==2) nu_scale_top(k)=2 - !if (k==3) nu_scale_top(k)=1 - !if (k>3) nu_scale_top(k)=0 if (nu_scale_top(k)>0) nlev_tom=k From 72e8ca9b87965cc0fab0dd0bc3d0c6d23356ff96 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Wed, 5 Jan 2022 19:24:38 -0600 Subject: [PATCH 261/301] remove unused variables --- components/homme/src/share/hybvcoord_mod.F90 | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/components/homme/src/share/hybvcoord_mod.F90 b/components/homme/src/share/hybvcoord_mod.F90 index d7e756f568fc..4ff7559bd7bb 100644 --- a/components/homme/src/share/hybvcoord_mod.F90 +++ b/components/homme/src/share/hybvcoord_mod.F90 @@ -24,8 +24,6 @@ module hybvcoord_mod real(r8) hybm(plev) ! ps component of hybrid coordinate - midpoints real(r8) etam(plev) ! eta-levels at midpoints real(r8) etai(plevp) ! eta-levels at interfaces - real(r8) d_etam(plev) ! Delta eta at midpoints - real(r8) d_etai(plevp) ! Delta eta at interfaces real(r8) dp0(plev) ! average layer thickness end type @@ -177,20 +175,10 @@ subroutine set_layer_locations(hvcoord, lprint, masterproc) ! get eta level deltas: do k=1,plev - hvcoord%d_etam(k)=hvcoord%etai(k+1)-hvcoord%etai(k) ! compute this way so it is BFB with older code: hvcoord%dp0(k) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*hvcoord%ps0 - !hvcoord%dp0(k) = hvcoord%d_etam(k)*hvcoord%ps0 enddo - do k=2,plev - ! same as (d_etam(k)+d_etam(k-1)/2 - hvcoord%d_etai(k)=hvcoord%etam(k)-hvcoord%etam(k-1) - enddo - ! dont include the 1/2, to make for easier weigthed averaging of interface - ! quantities to midpoints - hvcoord%d_etai(1)=hvcoord%d_etam(1) ! /2 - hvcoord%d_etai(plev+1)=hvcoord%d_etam(plev) ! /2 ! ====================================================================== ! Test that midpoint A,B is mean of interface A,B From 90de3b52e8e8e29ce8786ad4152a614d5057b2a5 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 6 Jan 2022 18:37:47 -0600 Subject: [PATCH 262/301] Update bld files with outo-generated changes to match Registry --- .../mpas-albany-landice/bld/build-namelist | 3 +++ .../bld/build-namelist-section | 3 +++ .../namelist_files/namelist_defaults_mali.xml | 3 +++ .../namelist_definition_mali.xml | 24 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/components/mpas-albany-landice/bld/build-namelist b/components/mpas-albany-landice/bld/build-namelist index 44f325531593..07704ca78a75 100755 --- a/components/mpas-albany-landice/bld/build-namelist +++ b/components/mpas-albany-landice/bld/build-namelist @@ -456,6 +456,7 @@ if ($MALI_DYNAMIC eq 'TRUE') { } else { add_default($nl, 'config_tracer_advection', 'val'=>"none"); } +add_default($nl, 'config_restore_thickness_after_advection'); ############################## # Namelist group: solidearth # @@ -484,6 +485,7 @@ add_default($nl, 'config_calving_timescale'); add_default($nl, 'config_restore_calving_front'); add_default($nl, 'config_remove_icebergs'); add_default($nl, 'config_remove_small_islands'); +add_default($nl, 'config_calving_speed_limit'); add_default($nl, 'config_grounded_von_Mises_threshold_stress'); add_default($nl, 'config_floating_von_Mises_threshold_stress'); add_default($nl, 'config_finalize_damage_after_advection'); @@ -631,6 +633,7 @@ add_default($nl, 'config_SGH_pressure_calc'); add_default($nl, 'config_SGH_alpha'); add_default($nl, 'config_SGH_beta'); add_default($nl, 'config_SGH_conduc_coeff'); +add_default($nl, 'config_SGH_conduc_coeff_drowned'); add_default($nl, 'config_SGH_till_drainage'); add_default($nl, 'config_SGH_till_max'); add_default($nl, 'config_SGH_advection'); diff --git a/components/mpas-albany-landice/bld/build-namelist-section b/components/mpas-albany-landice/bld/build-namelist-section index 5a64956f8f3c..021e62dd8399 100644 --- a/components/mpas-albany-landice/bld/build-namelist-section +++ b/components/mpas-albany-landice/bld/build-namelist-section @@ -22,6 +22,7 @@ add_default($nl, 'config_effective_pressure_max'); add_default($nl, 'config_thickness_advection'); add_default($nl, 'config_tracer_advection'); +add_default($nl, 'config_restore_thickness_after_advection'); ############################## # Namelist group: solidearth # @@ -46,6 +47,7 @@ add_default($nl, 'config_calving_timescale'); add_default($nl, 'config_restore_calving_front'); add_default($nl, 'config_remove_icebergs'); add_default($nl, 'config_remove_small_islands'); +add_default($nl, 'config_calving_speed_limit'); add_default($nl, 'config_grounded_von_Mises_threshold_stress'); add_default($nl, 'config_floating_von_Mises_threshold_stress'); add_default($nl, 'config_finalize_damage_after_advection'); @@ -193,6 +195,7 @@ add_default($nl, 'config_SGH_pressure_calc'); add_default($nl, 'config_SGH_alpha'); add_default($nl, 'config_SGH_beta'); add_default($nl, 'config_SGH_conduc_coeff'); +add_default($nl, 'config_SGH_conduc_coeff_drowned'); add_default($nl, 'config_SGH_till_drainage'); add_default($nl, 'config_SGH_till_max'); add_default($nl, 'config_SGH_advection'); diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 1a759fa886d3..cb9ea5942caf 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -19,6 +19,7 @@ 'fo' 'none' +.false. 'none' @@ -37,6 +38,7 @@ .true. .false. .false. +100.0 1.0e6 1.0e6 .true. @@ -152,6 +154,7 @@ 1.25 1.5 0.001 +0.0 3.1709792e-11 2.0 'fo' diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml index f5b20ddafbbb..621b52495c83 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml @@ -151,6 +151,14 @@ Valid values: 'fo', 'none' Default: Defined in namelist_defaults.xml + +If true, reset thickness to values at previous timestep after advection occurs. This is used for spinning up tracer fields such as damage. When this is true, geometry changes from surface and basal mass balance (grounded or floating) and facemelting are not retained, but changes from calving are. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + @@ -269,6 +277,14 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +Limit calvingVelocity to this value. Currently only supported for von Mises calving. + +Valid values: Any positive real value +Default: Defined in namelist_defaults.xml + + Threshold von Mises stress value required for calving velocity to exceed ice velocity on grounded ice. sigma_max in Morlighem et al. (2016) eq. 4. 1 MPa default value is from Morlighem et al.'s calibration for Store Glacier. @@ -1064,6 +1080,14 @@ Valid values: positive real number Default: Defined in namelist_defaults.xml + +conductivity coefficient for subglacial water flux for fraction of water thickness that exceeds bump height. Use 0.0 or negative value to disable. + +Valid values: positive real number or 0.0 +Default: Defined in namelist_defaults.xml + + background subglacial till drainage rate From ccea41dd162c827bf2abf9d45c6a140412e95281 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 7 Jan 2022 10:41:32 -0600 Subject: [PATCH 263/301] remove unneeded reamp_alg argument from compute_ppm() --- components/homme/src/share/vertremap_base.F90 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/homme/src/share/vertremap_base.F90 b/components/homme/src/share/vertremap_base.F90 index ffc512a96cc7..bc7ed73b4468 100644 --- a/components/homme/src/share/vertremap_base.F90 +++ b/components/homme/src/share/vertremap_base.F90 @@ -630,7 +630,7 @@ subroutine remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,remap_alg) enddo endif !Compute monotonic and conservative PPM reconstruction over every cell - coefs(:,:) = compute_ppm( ao , ppmdx, remap_alg ) + coefs(:,:) = compute_ppm( ao , ppmdx) !Compute tracer values on the new grid by integrating from the old cell bottom to the new !cell interface to form a new grid mass accumulation. Taking the difference between !accumulation at successive interfaces gives the mass inside each cell. Since Qdp is @@ -681,11 +681,10 @@ end function compute_ppm_grids !This computes a limited parabolic interpolant using a net 5-cell stencil, but the stages of computation are broken up into 3 stages -function compute_ppm( a , dx, remap_alg ) result(coefs) +function compute_ppm( a , dx ) result(coefs) implicit none real(kind=real_kind), intent(in) :: a ( -1:nlev+2) !Cell-mean values real(kind=real_kind), intent(in) :: dx (10, 0:nlev+1) !grid spacings - integer :: remap_alg real(kind=real_kind) :: coefs(0:2, nlev ) !PPM coefficients (for parabola) real(kind=real_kind) :: ai (0:nlev ) !fourth-order accurate, then limited interface values real(kind=real_kind) :: dma(0:nlev+1) !An expression from Collela's '84 publication From b3e8364729dadce44bbd974c40fa5bd893567c8c Mon Sep 17 00:00:00 2001 From: Alice Barthel Date: Fri, 7 Jan 2022 11:36:11 -0600 Subject: [PATCH 264/301] modified comment on partial thickness to clarify --- .../src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F index 748a53b6bca9..f27b593833b9 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_mixed_layer_heat_budget.F @@ -271,7 +271,7 @@ subroutine ocn_compute_mixed_layer_heat_budget(domain, timeLevel, err) !{{{ k = k + 1 depth = depth + layerThickness(k, iCell) end do - !need the clearn up of how much is left to get to MLD mld - depth + !add partial thickness of last layer to get to MLD depth = depth - layerThickness(k, iCell) difference = dThreshMLD(iCell) - depth do iTracer = 1, num_tracers From e7beed5a65f4062d13ff7fcbbb280a0f416372c8 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 7 Jan 2022 13:55:27 -0600 Subject: [PATCH 265/301] update unit test after q_alg=1,2 removal --- .../cxx/cxx_f90_interface_preqx.cpp | 4 -- .../homme/src/share/cxx/HommexxEnums.hpp | 6 -- components/homme/src/share/cxx/PpmRemap.hpp | 72 ------------------- .../src/share/cxx/VerticalRemapManager.cpp | 22 +----- .../cxx/cxx_f90_interface_theta.cpp | 4 -- .../share_kokkos_ut/ppm_remap_interface.F90 | 29 ++------ .../share_kokkos_ut/ppm_remap_ut.cpp | 17 ----- .../remap_theta_interface.F90 | 3 +- .../thetal_kokkos_ut/remap_theta_ut.cpp | 7 +- 9 files changed, 10 insertions(+), 154 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index f300cf33c5ca..a34b3aa8443c 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -63,10 +63,6 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, if (remap_alg==1) { params.remap_alg = RemapAlg::PPM_MIRRORED; - } else if (remap_alg == 2) { - params.remap_alg = RemapAlg::PPM_FIXED_PARABOLA; - } else if (remap_alg == 3) { - params.remap_alg = RemapAlg::PPM_FIXED_MEANS; } if (time_step_type==5) { params.time_step_type = TimeStepType::ttype5; diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 7e0e11e921ef..6c8104904cca 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -58,8 +58,6 @@ enum class AdvectionForm { enum class RemapAlg { PPM_MIRRORED = 1, - PPM_FIXED_PARABOLA = 2, - PPM_FIXED_MEANS = 3, PPM_LIMITED_EXTRAP = 10 }; @@ -67,10 +65,6 @@ inline std::string remapAlg2str (const RemapAlg alg) { switch (alg) { case RemapAlg::PPM_MIRRORED: return "PPM Mirrored"; - case RemapAlg::PPM_FIXED_MEANS: - return "PPM Fixed Means"; - case RemapAlg::PPM_FIXED_PARABOLA: - return "PPM Fixed Parabola"; case RemapAlg::PPM_LIMITED_EXTRAP: return "PPM Limited Extrapolation"; } diff --git a/components/homme/src/share/cxx/PpmRemap.hpp b/components/homme/src/share/cxx/PpmRemap.hpp index b6e03f873279..4c837c908edd 100644 --- a/components/homme/src/share/cxx/PpmRemap.hpp +++ b/components/homme/src/share/cxx/PpmRemap.hpp @@ -104,78 +104,6 @@ struct PpmMirrored : public PpmBoundaryConditions { static constexpr const char *name() { return "Mirrored PPM"; } }; -// Corresponds to remap alg = 2 -struct PpmFixedParabola : public PpmBoundaryConditions { - static constexpr int fortran_remap_alg = 2; - - KOKKOS_INLINE_FUNCTION - static void apply_ppm_boundary( - ExecViewUnmanaged cell_means, - ExecViewUnmanaged parabola_coeffs) { - const auto INITIAL_PADDING = _ppm_consts::INITIAL_PADDING; - const auto gs = _ppm_consts::gs; - parabola_coeffs(0, 0) = cell_means(INITIAL_PADDING); - parabola_coeffs(0, 1) = cell_means(INITIAL_PADDING + 1); - - parabola_coeffs(0, NUM_PHYSICAL_LEV - 2) = - cell_means(INITIAL_PADDING + NUM_PHYSICAL_LEV - gs); - parabola_coeffs(0, NUM_PHYSICAL_LEV - 1) = - cell_means(INITIAL_PADDING + NUM_PHYSICAL_LEV - gs + 1); - - parabola_coeffs(1, 0) = 0.0; - parabola_coeffs(1, 1) = 0.0; - parabola_coeffs(2, 0) = 0.0; - parabola_coeffs(2, 1) = 0.0; - - parabola_coeffs(1, NUM_PHYSICAL_LEV - 2) = 0.0; - parabola_coeffs(1, NUM_PHYSICAL_LEV - 1) = 0.0; - parabola_coeffs(2, NUM_PHYSICAL_LEV - 2) = 0.0; - parabola_coeffs(2, NUM_PHYSICAL_LEV - 1) = 0.0; - } - - KOKKOS_INLINE_FUNCTION - static void fill_cell_means_gs( - KernelVariables &kv, - const ExecViewUnmanaged& dpo, - ExecViewUnmanaged cell_means) { - PpmMirrored::fill_cell_means_gs(kv, dpo, cell_means); - } - - static constexpr const char *name() { return "Fixed Parabola PPM"; } -}; - -// Corresponds to remap alg = 3 -struct PpmFixedMeans : public PpmBoundaryConditions { - static constexpr int fortran_remap_alg = 3; - - KOKKOS_INLINE_FUNCTION - static void apply_ppm_boundary( - ExecViewUnmanaged /* cell_means */, - ExecViewUnmanaged /* parabola_coeffs */) - { - // Nothing to do here - } - - KOKKOS_INLINE_FUNCTION - static void fill_cell_means_gs( - KernelVariables &kv, - const ExecViewUnmanaged&, - ExecViewUnmanaged cell_means) { - const int gs = _ppm_consts::gs; - constexpr int INITIAL_PADDING = _ppm_consts::INITIAL_PADDING; - Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team, gs), - [&](const int &k_0) { - cell_means(INITIAL_PADDING - 1 - k_0 - 1 + 1) = - cell_means(INITIAL_PADDING); - - cell_means(NUM_PHYSICAL_LEV + INITIAL_PADDING - gs + k_0 + 1 + 1) = - cell_means(NUM_PHYSICAL_LEV + INITIAL_PADDING - gs + 1 - 1 + 1); - }); // end ghost cell loop - } - - static constexpr const char *name() { return "Fixed Means PPM"; } -}; - // Corresponds to remap alg = 10 struct PpmLimitedExtrap : public PpmBoundaryConditions { static constexpr int fortran_remap_alg = 10; diff --git a/components/homme/src/share/cxx/VerticalRemapManager.cpp b/components/homme/src/share/cxx/VerticalRemapManager.cpp index 6c3112174b4c..1782abb95e7b 100644 --- a/components/homme/src/share/cxx/VerticalRemapManager.cpp +++ b/components/homme/src/share/cxx/VerticalRemapManager.cpp @@ -41,27 +41,7 @@ struct VerticalRemapManager::Impl { using namespace Remap::Ppm; const int qsize = m_remap_tracers ? m_params.qsize : 0; const int capacity = m_remap_tracers ? -1 : m_params.qsize; - if (m_params.remap_alg == RemapAlg::PPM_FIXED_PARABOLA) { - if (m_params.rsplit != 0) { - remapper = std::make_shared> >( - qsize, m_elements, m_tracers, m_hvcoord, capacity); - } else { - remapper = std::make_shared> >( - qsize, m_elements, m_tracers, m_hvcoord, capacity); - } - } else if (m_params.remap_alg == RemapAlg::PPM_FIXED_MEANS) { - if (m_params.rsplit != 0) { - remapper = std::make_shared> >( - qsize, m_elements, m_tracers, m_hvcoord, capacity); - } else { - remapper = std::make_shared> >( - qsize, m_elements, m_tracers, m_hvcoord, capacity); - } - } else if (m_params.remap_alg == RemapAlg::PPM_MIRRORED) { + if (m_params.remap_alg == RemapAlg::PPM_MIRRORED) { if (m_params.rsplit != 0) { remapper = std::make_shared> >( diff --git a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp index 9535e4545ea5..6f73c778b14e 100644 --- a/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/cxx_f90_interface_theta.cpp @@ -68,10 +68,6 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, if (remap_alg==1) { params.remap_alg = RemapAlg::PPM_MIRRORED; - } else if (remap_alg == 2) { - params.remap_alg = RemapAlg::PPM_FIXED_PARABOLA; - } else if (remap_alg == 3) { - params.remap_alg = RemapAlg::PPM_FIXED_MEANS; } else if (remap_alg == 10) { params.remap_alg = RemapAlg::PPM_LIMITED_EXTRAP; } diff --git a/components/homme/test_execs/share_kokkos_ut/ppm_remap_interface.F90 b/components/homme/test_execs/share_kokkos_ut/ppm_remap_interface.F90 index bfcf0ad45615..971b1a6dacf2 100644 --- a/components/homme/test_execs/share_kokkos_ut/ppm_remap_interface.F90 +++ b/components/homme/test_execs/share_kokkos_ut/ppm_remap_interface.F90 @@ -18,13 +18,10 @@ subroutine remap_Q_ppm_c_callable(Qdp,nx,qsize,dp1,dp2,alg) bind(c) real (kind=real_kind), intent(inout) :: Qdp(nx,nx,nlev,qsize) real (kind=real_kind), intent(in) :: dp1(nx,nx,nlev),dp2(nx,nx,nlev) - !aim for alg=1 or alg=2 only - if(alg == 1 .or. alg == 2 .or. alg == 3) then - ! Need to set alg in control_mod, b/c fortran reads it from there - vert_remap_q_alg = alg - call remap_Q_ppm(Qdp,nx,qsize,dp1,dp2) + if(alg == 1 .or. alg == 10 .or. alg == 11) then + call remap_Q_ppm(Qdp,nx,qsize,dp1,dp2,alg) else - call abortmp('compute_ppm_grids_c_callable: bad alg (not 1 or 2) .') + call abortmp('compute_ppm_grids_c_callable: bad alg (not 1,10 or 11) .') endif end subroutine remap_Q_ppm_c_callable @@ -38,15 +35,8 @@ subroutine compute_ppm_grids_c_callable(dx,rslt,alg) bind(c) real(kind=real_kind), intent(in) :: dx(-1:nlev+2) real(kind=real_kind), intent(out) :: rslt(10,0:nlev+1) !grid spacings - !aim for alg=1 or alg=2 only - if(alg == 1 .or. alg == 2 .or. alg == 3) then - ! Need to set alg in control_mod, b/c fortran reads it from there - vert_remap_q_alg = alg - rslt = compute_ppm_grids(dx) - rslt(4:10, nlev + 1) = 0.0 - else - call abortmp('compute_ppm_grids_c_callable: bad alg (not 1, 2, or 3) .') - endif + rslt = compute_ppm_grids(dx) + rslt(4:10, nlev + 1) = 0.0 end subroutine compute_ppm_grids_c_callable subroutine compute_ppm_c_callable(a,dx,coefs,alg) bind(c) @@ -59,14 +49,7 @@ subroutine compute_ppm_c_callable(a,dx,coefs,alg) bind(c) real(kind=real_kind), intent(in) :: dx (10, 0:nlev+1) !grid spacings real(kind=real_kind), intent(out):: coefs(0:2, nlev ) !PPM coefficients (for parabola) - !aim for alg=1 or alg=2 only - if(alg == 1 .or. alg == 2 .or. alg == 3) then - ! Need to set alg in control_mod, b/c fortran reads it from there - vert_remap_q_alg = alg - coefs = compute_ppm(a,dx) - else - call abortmp('compute_ppm_c_callable: bad alg (not 1, 2, or 3) .') - endif + coefs = compute_ppm(a,dx) end subroutine compute_ppm_c_callable end module remap_interface diff --git a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp index 15e53a79e3be..67f206080458 100644 --- a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp @@ -391,23 +391,6 @@ TEST_CASE("ppm_mirrored", "vertical remap") { SECTION("remap") { remap_test_mirrored.test_remap(); } } -TEST_CASE("ppm_fixed_parabola", "vertical remap") { - constexpr int num_elems = 2; - constexpr int num_remap = 3; - ppm_remap_functor_test remap_test_fixed(num_elems, num_remap); - SECTION("grid") { remap_test_fixed.test_grid(); } - SECTION("ppm") { remap_test_fixed.test_ppm(); } - SECTION("remap") { remap_test_fixed.test_remap(); } -} - -TEST_CASE("ppm_fixed_means", "vertical remap") { - constexpr int num_elems = 2; - constexpr int num_remap = 3; - ppm_remap_functor_test remap_test_fixed(num_elems, num_remap); - SECTION("grid") { remap_test_fixed.test_grid(); } - SECTION("ppm") { remap_test_fixed.test_ppm(); } - SECTION("remap") { remap_test_fixed.test_remap(); } -} TEST_CASE("binary_search","binary_search") { diff --git a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_interface.F90 b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_interface.F90 index ab61c09355e6..185a7f9e4664 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_interface.F90 +++ b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_interface.F90 @@ -60,7 +60,7 @@ subroutine run_remap_f90 (np1, np1_qdp, dt, rsplit_in, qsize_in, vr_alg, & dp_ptr, vtheta_dp_ptr, w_i_ptr, phi_i_ptr, v_ptr, & ps_ptr, eta_dot_dpdn_ptr, qdp_ptr) bind(c) use iso_c_binding, only: c_ptr, c_f_pointer, c_int, c_bool - use control_mod, only: vert_remap_q_alg, rsplit + use control_mod, only: vert_remap_q_alg, vert_remap_u_alg, rsplit use dimensions_mod, only: nelemd, nlev, nlevp, np, qsize, qsize_d use thetal_test_interface, only: hvcoord use geometry_interface_mod, only: hybrid @@ -111,6 +111,7 @@ subroutine run_remap_f90 (np1, np1_qdp, dt, rsplit_in, qsize_in, vr_alg, & ! set control variables rsplit = rsplit_in vert_remap_q_alg = vr_alg + vert_remap_u_alg = vr_alg qsize = qsize_in ! Call f90 vertical remap diff --git a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp index e78cfb0e0fae..b8e885372e13 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/remap_theta_ut.cpp @@ -181,15 +181,10 @@ TEST_CASE("remap", "remap_testing") { }; SECTION ("run_remap") { - auto remap_algs = {RemapAlg::PPM_FIXED_PARABOLA, RemapAlg::PPM_MIRRORED, RemapAlg::PPM_FIXED_MEANS, - RemapAlg::PPM_LIMITED_EXTRAP}; + auto remap_algs = {RemapAlg::PPM_MIRRORED, RemapAlg::PPM_LIMITED_EXTRAP}; auto remap_alg_f90 = [](const RemapAlg alg)->int { if (alg==RemapAlg::PPM_MIRRORED) { return 1; - } else if (alg==RemapAlg::PPM_FIXED_PARABOLA) { - return 2; - } else if (alg==RemapAlg::PPM_FIXED_MEANS) { - return 3; } else if (alg==RemapAlg::PPM_LIMITED_EXTRAP) { return 10; } From d97bfd278253c170ecd998e57bd9a50122131850 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 6 Jan 2022 14:45:47 -0700 Subject: [PATCH 266/301] Update CIME submodule ... to bc80870bfa0b08d154e89b1fd46493cad1b39a02 Changes: 1) Split CESM configs out of repo 2) Adds enforcing code formatting to pre-commit 3) Refactor all CIME internal testing, split into smaller files and move to a more-logical dir 4) Cleanup Scorpio I/O performance data in the run directory 5) Remove documentation sections for pieces no longer included in CIME 6) Improve Macro.make generation Fixes: *) Loads of small fixes related to recent code churn. 1) Cleanup Scorpio I/O performance data in the run directory [BFB] --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index a26c8e393d4e..bc80870bfa0b 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit a26c8e393d4e062459b6dfb9605700fe24665e50 +Subproject commit bc80870bfa0b08d154e89b1fd46493cad1b39a02 From bfdc5a23fbaeeea3bf00b315f40f51db27934883 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 10 Jan 2022 10:14:50 -0600 Subject: [PATCH 267/301] re-enable imex unit tests --- components/homme/src/theta-l/share/imex_mod.F90 | 5 +---- components/homme/src/theta-l/share/model_init_mod.F90 | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/components/homme/src/theta-l/share/imex_mod.F90 b/components/homme/src/theta-l/share/imex_mod.F90 index 65da7f1dca19..a43ca6fb6aea 100644 --- a/components/homme/src/theta-l/share/imex_mod.F90 +++ b/components/homme/src/theta-l/share/imex_mod.F90 @@ -498,10 +498,7 @@ subroutine test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) minjacerr=0 if (hybrid%masterthread) write(iulog,*)'Running IMEX Jacobian unit test...' do ie=nets,nete - do k=1,nlev - dp3d(:,:,k) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & - ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,tl%n0) - enddo + dp3d(:,:,:) = elem(ie)%state%dp3d(:,:,:,tl%n0) vtheta_dp(:,:,:) = elem(ie)%state%vtheta_dp(:,:,:,tl%n0) phi_i(:,:,:) = elem(ie)%state%phinh_i(:,:,:,tl%n0) phis(:,:) = elem(ie)%state%phis(:,:) diff --git a/components/homme/src/theta-l/share/model_init_mod.F90 b/components/homme/src/theta-l/share/model_init_mod.F90 index 9d3c1169b7a8..f32c9f05fc98 100644 --- a/components/homme/src/theta-l/share/model_init_mod.F90 +++ b/components/homme/src/theta-l/share/model_init_mod.F90 @@ -108,11 +108,9 @@ subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) enddo - ! unit test for analytic jacobian used by IMEX methods -#if 0 + ! unit test for analytic jacobian and tri-diag solve used by IMEX methods if (.not. theta_hydrostatic_mode) & call test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) -#endif !$omp master ! From 11e5eb82fb730325d978db21576e3af5c0769266 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Mon, 10 Jan 2022 15:39:26 -0700 Subject: [PATCH 268/301] Add defaults for ne4pg2 F20TR compsets --- components/elm/bld/namelist_files/namelist_defaults.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 749a00bf85e9..3111f519357d 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -373,6 +373,8 @@ lnd/clm2/surfdata_map/surfdata_ne120np4_simyr1850_c160211.nc lnd/clm2/surfdata_map/surfdata_ne11np4_simyr1850_c160614.nc lnd/clm2/surfdata_map/surfdata_ne4np4_simyr1850_c160614.nc + +lnd/clm2/surfdata_map/surfdata_ne4pg2_simyr1850_c210722.nc lnd/clm2/surfdata_map/surfdata_ne240np4_simyr1850_c170821.nc @@ -438,6 +440,8 @@ lnd/clm2/surfdata_map/surfdata_twpx4v1_simyr2000_c170706.nc use_crop=".false." >lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_simyr1850-2015_c180306.nc lnd/clm2/surfdata_map/landuse.timeseries_ne30np4.pg2_hist_simyr1850-2015_c210113.nc + lnd/clm2/surfdata_map/landuse.timeseries_ne4pg2_hist_simyr1850-2015_c210722.nc lnd/clm2/surfdata_map/landuse.timeseries_ne0np4_northamericax4v1.pg2_hist_simyr1850-2015_c211015.nc Date: Mon, 10 Jan 2022 16:23:50 -0700 Subject: [PATCH 269/301] Remove extra drydep list from use case --- .../eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml index 32dbab8537e0..0a9aaabc615c 100644 --- a/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml +++ b/components/eam/bld/namelist_files/use_cases/20TR_MMF-1mom_CMIP6.xml @@ -36,9 +36,6 @@ atm/cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc SERIAL - -'H2O2', 'H2SO4', 'SO2' - 1850-2000 From 822404a6a23cfec9e1f2695360eb1b922ab2fc04 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Mon, 10 Jan 2022 16:24:13 -0700 Subject: [PATCH 270/301] Remove CMIP6 from F20TR-MMFXX compset alias --- components/eam/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index acbcc57a6418..22bf2176cb7c 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -267,7 +267,7 @@ - F20TR-MMFXX-CMIP6 + F20TR-MMFXX 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV From 489daf5f84733ac1d1ac44b62cd19ffeb1429af4 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Mon, 10 Jan 2022 16:24:56 -0700 Subject: [PATCH 271/301] Change MMF VT test to use new F20TR compset --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 5bef6fcc766a..75f3ab540245 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -171,7 +171,7 @@ "SMS_D_Ld1.T62_oEC60to30v3.DTESTM", "SMS_D_Ld1.ne30pg2_r05_EC30to60E2r2.WCYCL1850", "ERP_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_fixed_subcycle", - "ERS_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_use_VT", + "ERS_Ln9.ne4pg2_ne4pg2.F20TR-MMFXX.eam-mmf_use_VT", "ERS_Ln9_P96x1.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_use_ESMT", "ERS_Ln9.ne4pg2_ne4pg2.F-MMFOMP.eam-single_thread", "ERS_Ln9.ne4pg2_ne4pg2.F-MMF1-RCEMIP", From 0bd54118141eb3245cb08a786d61e0607561758f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 11 Jan 2022 17:14:18 +0100 Subject: [PATCH 272/301] Fix interface locations for 60-layer PHC grid The previous values were the layer centers, not the layer interfaces for all but the top layer interface. --- .../mode_init/mpas_ocn_init_vertical_grids.F | 127 +++++++++--------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/components/mpas-ocean/src/mode_init/mpas_ocn_init_vertical_grids.F b/components/mpas-ocean/src/mode_init/mpas_ocn_init_vertical_grids.F index fdad27c24076..caf89e3b3cd4 100644 --- a/components/mpas-ocean/src/mode_init/mpas_ocn_init_vertical_grids.F +++ b/components/mpas-ocean/src/mode_init/mpas_ocn_init_vertical_grids.F @@ -140,9 +140,10 @@ end subroutine ocn_generate_uniform_vertical_grid!}}} ! ! routine ocn_generate_60layerPHC_vertical_grid ! - !> \brief 60 layer PHC vertical grid generator - !> \author Doug Jacobsen - !> \date 03/20/2015 + !> \brief 60 layer PHC vertical grid generator + !> \author Doug Jacobsen, Xylar Asay-Davis + !> \date 03/20/2015 + !> \modified 01/11/2022 !> \details !> This routine generates a 60 layer vertical grid based on the PHC data set. ! @@ -163,66 +164,66 @@ subroutine ocn_generate_60layerPHC_vertical_grid(interfaceLocations)!{{{ end if interfaceLocations(1) = 0.0_RKIND - interfaceLocations(2) = 500_RKIND - interfaceLocations(3) = 1500_RKIND - interfaceLocations(4) = 2500_RKIND - interfaceLocations(5) = 3500_RKIND - interfaceLocations(6) = 4500_RKIND - interfaceLocations(7) = 5500_RKIND - interfaceLocations(8) = 6500_RKIND - interfaceLocations(9) = 7500_RKIND - interfaceLocations(10) = 8500_RKIND - interfaceLocations(11) = 9500_RKIND - interfaceLocations(12) = 10500_RKIND - interfaceLocations(13) = 11500_RKIND - interfaceLocations(14) = 12500_RKIND - interfaceLocations(15) = 13500_RKIND - interfaceLocations(16) = 14500_RKIND - interfaceLocations(17) = 15500_RKIND - interfaceLocations(18) = 16509.83984375_RKIND - interfaceLocations(19) = 17547.904296875_RKIND - interfaceLocations(20) = 18629.125_RKIND - interfaceLocations(21) = 19766.025390625_RKIND - interfaceLocations(22) = 20971.134765625_RKIND - interfaceLocations(23) = 22257.826171875_RKIND - interfaceLocations(24) = 23640.880859375_RKIND - interfaceLocations(25) = 25137.013671875_RKIND - interfaceLocations(26) = 26765.416015625_RKIND - interfaceLocations(27) = 28548.361328125_RKIND - interfaceLocations(28) = 30511.91796875_RKIND - interfaceLocations(29) = 32686.794921875_RKIND - interfaceLocations(30) = 35109.34375_RKIND - interfaceLocations(31) = 37822.75390625_RKIND - interfaceLocations(32) = 40878.4609375_RKIND - interfaceLocations(33) = 44337.765625_RKIND - interfaceLocations(34) = 48273.66796875_RKIND - interfaceLocations(35) = 52772.796875_RKIND - interfaceLocations(36) = 57937.28515625_RKIND - interfaceLocations(37) = 63886.2578125_RKIND - interfaceLocations(38) = 70756.328125_RKIND - interfaceLocations(39) = 78700.25_RKIND - interfaceLocations(40) = 87882.5234375_RKIND - interfaceLocations(41) = 98470.5859375_RKIND - interfaceLocations(42) = 110620.421875_RKIND - interfaceLocations(43) = 124456.6953125_RKIND - interfaceLocations(44) = 140049.71875_RKIND - interfaceLocations(45) = 157394.640625_RKIND - interfaceLocations(46) = 176400.328125_RKIND - interfaceLocations(47) = 196894.421875_RKIND - interfaceLocations(48) = 218645.65625_RKIND - interfaceLocations(49) = 241397.15625_RKIND - interfaceLocations(50) = 264900.125_RKIND - interfaceLocations(51) = 288938.46875_RKIND - interfaceLocations(52) = 313340.46875_RKIND - interfaceLocations(53) = 337979.375_RKIND - interfaceLocations(54) = 362767.0625_RKIND - interfaceLocations(55) = 387645.21875_RKIND - interfaceLocations(56) = 412576.84375_RKIND - interfaceLocations(57) = 437539.28125_RKIND - interfaceLocations(58) = 462519.0625_RKIND - interfaceLocations(59) = 487508.375_RKIND - interfaceLocations(60) = 512502.84375_RKIND - interfaceLocations(61) = 537500_RKIND + interfaceLocations(2) = 10.0_RKIND + interfaceLocations(3) = 20.0_RKIND + interfaceLocations(4) = 30.0_RKIND + interfaceLocations(5) = 40.0_RKIND + interfaceLocations(6) = 50.0_RKIND + interfaceLocations(7) = 60.0_RKIND + interfaceLocations(8) = 70.0_RKIND + interfaceLocations(9) = 80.0_RKIND + interfaceLocations(10) = 90.0_RKIND + interfaceLocations(11) = 100.0_RKIND + interfaceLocations(12) = 110.0_RKIND + interfaceLocations(13) = 120.0_RKIND + interfaceLocations(14) = 130.0_RKIND + interfaceLocations(15) = 140.0_RKIND + interfaceLocations(16) = 150.0_RKIND + interfaceLocations(17) = 160.0_RKIND + interfaceLocations(18) = 170.19677734375_RKIND + interfaceLocations(19) = 180.76129150390625_RKIND + interfaceLocations(20) = 191.82119750976562_RKIND + interfaceLocations(21) = 203.49929809570312_RKIND + interfaceLocations(22) = 215.92340087890625_RKIND + interfaceLocations(23) = 229.23312377929688_RKIND + interfaceLocations(24) = 243.58447265625_RKIND + interfaceLocations(25) = 259.1557922363281_RKIND + interfaceLocations(26) = 276.1524963378906_RKIND + interfaceLocations(27) = 294.8147277832031_RKIND + interfaceLocations(28) = 315.4236145019531_RKIND + interfaceLocations(29) = 338.3122863769531_RKIND + interfaceLocations(30) = 363.8746032714844_RKIND + interfaceLocations(31) = 392.5804748535156_RKIND + interfaceLocations(32) = 424.9887390136719_RKIND + interfaceLocations(33) = 461.7665710449219_RKIND + interfaceLocations(34) = 503.7067565917969_RKIND + interfaceLocations(35) = 551.7491760253906_RKIND + interfaceLocations(36) = 606.9965515136719_RKIND + interfaceLocations(37) = 670.7285461425781_RKIND + interfaceLocations(38) = 744.3980407714844_RKIND + interfaceLocations(39) = 829.6069641113281_RKIND + interfaceLocations(40) = 928.0434265136719_RKIND + interfaceLocations(41) = 1041.3681945800781_RKIND + interfaceLocations(42) = 1171.0402526855469_RKIND + interfaceLocations(43) = 1318.0935363769531_RKIND + interfaceLocations(44) = 1482.9008483886719_RKIND + interfaceLocations(45) = 1664.9919738769531_RKIND + interfaceLocations(46) = 1863.0146179199219_RKIND + interfaceLocations(47) = 2074.873809814453_RKIND + interfaceLocations(48) = 2298.039276123047_RKIND + interfaceLocations(49) = 2529.903594970703_RKIND + interfaceLocations(50) = 2768.098846435547_RKIND + interfaceLocations(51) = 3010.670196533203_RKIND + interfaceLocations(52) = 3256.138885498047_RKIND + interfaceLocations(53) = 3503.448516845703_RKIND + interfaceLocations(54) = 3751.892791748047_RKIND + interfaceLocations(55) = 4001.011505126953_RKIND + interfaceLocations(56) = 4250.525604248047_RKIND + interfaceLocations(57) = 4500.259552001953_RKIND + interfaceLocations(58) = 4750.121307373047_RKIND + interfaceLocations(59) = 5000.045684814453_RKIND + interfaceLocations(60) = 5250.010955810547_RKIND + interfaceLocations(61) = 5499.989044189453_RKIND maxInterfaceLocation = maxval(interfaceLocations) From e677e181fa6b779b54b120cc50c13dc299013d16 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 11 Jan 2022 10:19:55 -0700 Subject: [PATCH 273/301] Update CIME submodule to b0ada2143def2bfc5f6d42941486d8264a52628a Changes: 1) Extend throughput and memory checking in baseline tests 2) Introduce a new compset naming convention which maintains backward compatibility and allows position independence. Fixes: 1) FUNIT needs CIME_NO_CMAKE_MACRO ON for now [BFB] --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index bc80870bfa0b..b0ada2143def 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit bc80870bfa0b08d154e89b1fd46493cad1b39a02 +Subproject commit b0ada2143def2bfc5f6d42941486d8264a52628a From 974743ba12860ec2ada9cc5952b751b4a67d65e4 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Mon, 10 Jan 2022 19:46:58 -0600 Subject: [PATCH 274/301] Extend e3sm_prod testing with throughput and memory checks --- cime_config/machines/config_machines.xml | 3 ++- cime_config/testmods_dirs/bench/noio/shell_commands | 2 ++ cime_config/testmods_dirs/bench/noio/user_nl_mpaso | 8 ++++++++ cime_config/testmods_dirs/bench/noio/user_nl_mpassi | 3 +++ cime_config/tests.py | 11 +++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 cime_config/testmods_dirs/bench/noio/shell_commands create mode 100644 cime_config/testmods_dirs/bench/noio/user_nl_mpaso create mode 100644 cime_config/testmods_dirs/bench/noio/user_nl_mpassi diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index bdd1b54bcef7..33b5c6f18f4c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1503,7 +1503,8 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 + 0.05 + 0.05 1000 /lcrc/group/e3sm/soft/perl/chrys/lib/perl5 diff --git a/cime_config/testmods_dirs/bench/noio/shell_commands b/cime_config/testmods_dirs/bench/noio/shell_commands new file mode 100644 index 000000000000..d67f637e66e0 --- /dev/null +++ b/cime_config/testmods_dirs/bench/noio/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange --append CAM_CONFIG_OPTS='-cosp' diff --git a/cime_config/testmods_dirs/bench/noio/user_nl_mpaso b/cime_config/testmods_dirs/bench/noio/user_nl_mpaso new file mode 100644 index 000000000000..fcaa3656188f --- /dev/null +++ b/cime_config/testmods_dirs/bench/noio/user_nl_mpaso @@ -0,0 +1,8 @@ +config_am_highfrequencyoutput_enable=false +config_am_globalstats_enable=false +config_am_meridionalheattransport_enable=false +config_am_timeseriesstatsmonthly_enable=false +config_am_timeseriesstatsmonthlymin_enable=false +config_am_timeseriesstatsmonthlymax_enable=false +config_am_eddyproductvariables_enable=false +config_am_oceanheatcontent_enable=false diff --git a/cime_config/testmods_dirs/bench/noio/user_nl_mpassi b/cime_config/testmods_dirs/bench/noio/user_nl_mpassi new file mode 100644 index 000000000000..f14524a6251d --- /dev/null +++ b/cime_config/testmods_dirs/bench/noio/user_nl_mpassi @@ -0,0 +1,3 @@ +config_am_timeseriesstatsmonthly_enable=false +config_am_timeseriesstatsdaily_enable=false +config_am_regionalstatistics_enable=false diff --git a/cime_config/tests.py b/cime_config/tests.py index 5bef6fcc766a..8e135a925719 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -245,6 +245,17 @@ ) }, + #e3sm performance-benching of production-like runs + "e3sm_prod_bench" : { + "tests" : ( + "PFS.ne30pg2_r05_oECv3.F2010.bench-noio", + "PFS.ne30pg2_r05_oECv3.F20TR.bench-noio", + "PFS.ne30pg2_r05_EC30to60E2r2.WCYCL1850.bench-noio", + "PFS.ne30pg2_EC30to60E2r2.WCYCL1850.bench-noio", + "PFS_PS.northamericax4v1pg2_WC14to60E2r3.WCYCL1850.bench-noio", + ) + }, + "fates" : { "tests" : ( "ERS_Ld9.1x1_brazil.IELMFATES", From c1e8a784cc0e40a3531dc3cb02100a89fb75d1c2 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 12 Jan 2022 14:43:37 -0600 Subject: [PATCH 275/301] Make surfaceTemperatureFluxWithoutRunoff private in omp pragma --- .../mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index ea08a50220a9..da5995f12a1d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -540,7 +540,7 @@ subroutine ocn_surface_bulk_forcing_active_tracers(meshPool, forcingPool, tracer ! indices on tracerGroup are (iTracer, iLevel, iCell) if (.not. bulkThicknessFluxOff) then !$omp parallel - !$omp do schedule(runtime) + !$omp do schedule(runtime) private(surfaceTemperatureFluxWithoutRunoff) do iCell = 1, nCells ! Accumulate fluxes that use the surface temperature From a793e4f1e5099133df9d6f310927ba7ec4cd9148 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Wed, 12 Jan 2022 15:13:38 -0600 Subject: [PATCH 276/301] Add SMS_D_Ld3.T62_oQU120.CMPASO-IAF to e3sm_integration --- cime_config/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/tests.py b/cime_config/tests.py index 5bef6fcc766a..05881db4b627 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -169,6 +169,7 @@ "SMS_Ld2.ne30_oECv3.BGCEXP_CNTL_CNPECACNT_1850.elm-bgcexp", "SMS_Ld2.ne30_oECv3.BGCEXP_CNTL_CNPRDCTC_1850.elm-bgcexp", "SMS_D_Ld1.T62_oEC60to30v3.DTESTM", + "SMS_D_Ld3.T62_oQU120.CMPASO-IAF", "SMS_D_Ld1.ne30pg2_r05_EC30to60E2r2.WCYCL1850", "ERP_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_fixed_subcycle", "ERS_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_use_VT", From 6868308cbc2194123f534236623461d0ad9f0f4d Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Wed, 12 Jan 2022 16:50:44 -0700 Subject: [PATCH 277/301] Fix ELM config_component wildcards to get correct use case for MMF --- components/elm/cime_config/config_component.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index 616a14efdf78..9f8aceea29b9 100644 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -71,7 +71,7 @@ 1850_SCMIP6_control 1850_CMIP6bgc_control 1850_SCMIP6_control - 20thC_CMIP6_transient + 20thC_CMIP6_transient 20thC_CMIP6bgc_transient 20thC_bgc_transient 1950_CMIP6_control From 71c1ae52641375fce186259df3c0b3ce431e73da Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Thu, 13 Jan 2022 11:03:26 -0700 Subject: [PATCH 278/301] Use MPASSI for F20TR-MMFXX compset --- components/eam/cime_config/config_compsets.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 22bf2176cb7c..639dc6844f2a 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -268,7 +268,7 @@ F20TR-MMFXX - 20TR_EAM%MMFXX-CMIP6_ELM%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 20TR_EAM%MMFXX-CMIP6_ELM%SP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV From 38e3e36f2162e2840d25d86d66b2685fcbffb936 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Hillman" Date: Thu, 13 Jan 2022 12:12:45 -0700 Subject: [PATCH 279/301] Fix resolution in F20TR-MMFXX test --- cime_config/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 75f3ab540245..3a7acce0bda5 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -171,7 +171,7 @@ "SMS_D_Ld1.T62_oEC60to30v3.DTESTM", "SMS_D_Ld1.ne30pg2_r05_EC30to60E2r2.WCYCL1850", "ERP_Ln9.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_fixed_subcycle", - "ERS_Ln9.ne4pg2_ne4pg2.F20TR-MMFXX.eam-mmf_use_VT", + "ERS_Ln9.ne4pg2_oQU480.F20TR-MMFXX.eam-mmf_use_VT", "ERS_Ln9_P96x1.ne4pg2_ne4pg2.F-MMFXX.eam-mmf_use_ESMT", "ERS_Ln9.ne4pg2_ne4pg2.F-MMFOMP.eam-single_thread", "ERS_Ln9.ne4pg2_ne4pg2.F-MMF1-RCEMIP", From 75ec9dbbc09e918c55634f8881561b770ce7bcdf Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 13 Jan 2022 12:53:26 -0700 Subject: [PATCH 280/301] Fix missing dependency in one case of gen_f90_targets This bug was causing .F changes to not be reflected in rebuilds. [BFB] --- components/mpas-framework/src/cmake_utils.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/mpas-framework/src/cmake_utils.cmake b/components/mpas-framework/src/cmake_utils.cmake index c3a25f238db6..ce2af30cf286 100644 --- a/components/mpas-framework/src/cmake_utils.cmake +++ b/components/mpas-framework/src/cmake_utils.cmake @@ -61,7 +61,8 @@ function(genf90_targets RAW_SOURCES_ARG INCLUDES_ARG CPPDEFS_ARG NO_PREPROCESS_A add_custom_command ( OUTPUT ${CMAKE_BINARY_DIR}/${SOURCE_F90} COMMAND cpp -P -traditional ${CPPDEFS_ARG} ${INCLUDES_I} -Uvector - ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE} > ${CMAKE_BINARY_DIR}/${SOURCE_F90}) + ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE} > ${CMAKE_BINARY_DIR}/${SOURCE_F90} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE}) endif() list(APPEND LOCAL_SOURCES ${CMAKE_BINARY_DIR}/${SOURCE_F90}) else() From ce6b9739516f4e4b61a987dfed44de3b3afe8adc Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Thu, 13 Jan 2022 14:02:59 -0800 Subject: [PATCH 281/301] Updating assignment to fates pft_areafrac If not using a surface dataset with topounits, only access the first index for the topounit dimension as max_topounit is defined in these cases to be one. Otherwise we uses the column type indexing. --- .../elm/src/main/elmfates_interfaceMod.F90 | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/components/elm/src/main/elmfates_interfaceMod.F90 b/components/elm/src/main/elmfates_interfaceMod.F90 index 99d337f26399..a4ed27310e6c 100644 --- a/components/elm/src/main/elmfates_interfaceMod.F90 +++ b/components/elm/src/main/elmfates_interfaceMod.F90 @@ -521,7 +521,6 @@ subroutine init(this, bounds_proc ) type(bounds_type) :: bounds_clump integer :: nmaxcol integer :: ndecomp - real(r8) :: wt_nat_patch_toposum ! Initialize the FATES communicators with the HLM ! This involves to stages @@ -638,25 +637,30 @@ subroutine init(this, bounds_proc ) ! --------------------------------------------------------------------------- g = col_pp%gridcell(c) - t = col_pp%topounit(c) this%fates(nc)%sites(s)%lat = grc_pp%latdeg(g) this%fates(nc)%sites(s)%lon = grc_pp%londeg(g) + ! Check whether or not the surface dataset has topounits. If it doesn't set the + ! index t to max_topounits, which should be 1. Otherwise, determine the index + ! from the columntype + if (has_topounit) then + t = col_pp%topounit(c) + else + if (max_topounits .ne. 1) then + write(iulog,*) 'max_topounits should only be one when has_topounit is false' + write(iulog,*) 'max_topounits, has_topounit: ', max_topounits, has_topounit + call endrun(msg=errMsg(sourcefile, __LINE__)) + else + t = max_topounits + endif + endif + ! initialize static layers for reduced complexity FATES versions from HLM this%fates(nc)%bc_in(s)%pft_areafrac(:)=0._r8 do m = natpft_lb,natpft_ub ft = m-natpft_lb - - ! For now, sum the weights along all topounits for a given gridcell - wt_nat_patch_toposum = sum(wt_nat_patch(g,:,m)) - this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch_toposum - - !this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch(g,t,m) - - !write(iulog,*) 'elmfates: has_topounit: ',has_topounit - !write(iulog,*) 'elmfates: wt_nat_patch(g,1,m): ', wt_nat_patch(g,1,m) - !write(iulog,*) 'elmfates: sum wt_nat_patch: ', wt_nat_patch_toposum + this%fates(nc)%bc_in(s)%pft_areafrac(ft)=wt_nat_patch(g,t,m) end do if(abs(sum(this%fates(nc)%bc_in(s)%pft_areafrac(natpft_lb:natpft_ub))-1.0_r8).gt.1.0e-9)then From 797657921b1f18a151a917c95e673df1d890b0b3 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 16 Jan 2022 19:31:01 -0600 Subject: [PATCH 282/301] put a stop for xx dycore + omp --- components/eam/src/dynamics/se/dyn_comp.F90 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index 947fac55b15b..b2a22c91284d 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -113,6 +113,12 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out) integer :: npes_se integer :: npes_se_stride +#ifdef KOKKOS_TARGET +#ifdef _OPENMP + call endrun( 'kokkos dycore does not run with threads') +#endif +#endif + !---------------------------------------------------------------------- ! Initialize dynamics grid variables From 25f515fe4198dcb9651e901bc874c4ccb2863eb2 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 16 Jan 2022 19:47:20 -0600 Subject: [PATCH 283/301] cleanup --- components/eam/bld/configure | 1 - components/eam/src/dynamics/se/dyn_comp.F90 | 2 - components/homme/src/share/control_mod.F90 | 2 - components/homme/src/share/cxx/prim_step.cpp | 4 -- .../src/theta-l_kokkos/cxx/CamForcing.cpp | 3 -- .../src/theta-l_kokkos/cxx/ForcingFunctor.hpp | 15 ------- .../src/theta-l_kokkos/prim_driver_mod.F90 | 45 ------------------- 7 files changed, 72 deletions(-) diff --git a/components/eam/bld/configure b/components/eam/bld/configure index 4b5cd9e48c06..312a50a0afce 100755 --- a/components/eam/bld/configure +++ b/components/eam/bld/configure @@ -786,7 +786,6 @@ if (defined $opts{'dyn'}) { } if (substr($opts{'dyn_target'},0,5) eq 'theta') { # theta-l (f code) and theta-l_kokkos (cxx code) use 'theta-l' namelist defaults - # ??????????????? or theta ? $cfg_ref->set('dyn_target', 'theta-l'); } print "SE dycore target options:\n"; diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index b2a22c91284d..20d4a484ddde 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -12,7 +12,6 @@ Module dyn_comp use time_manager, only: is_first_step use spmd_utils, only : iam, npes_cam => npes use pio, only: file_desc_t - use control_mod, only: nsplit_iteration implicit none private @@ -373,7 +372,6 @@ subroutine dyn_run( dyn_state, rc ) if (.not. use_3dfrc) then do n=1,se_nsplit - nsplit_iteration = n ! forward-in-time RK, with subcycling call t_startf('prim_run_subcycle') call prim_run_subcycle(dyn_state%elem,hybrid,nets,nete,& diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 6385716585c2..03bbd31f1fd3 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -68,8 +68,6 @@ module control_mod ! Default values make qsplit and rsplit control the time steps. integer, public :: dt_remap_factor = -1, dt_tracer_factor = -1 - integer, public :: nsplit_iteration = 0 - integer, public :: LFTfreq=0 ! leapfrog-trapazoidal frequency (shallow water only) ! interspace a lf-trapazoidal step every LFTfreq leapfrogs ! 0 = disabled diff --git a/components/homme/src/share/cxx/prim_step.cpp b/components/homme/src/share/cxx/prim_step.cpp index dfb31187904b..8ab37cda79b6 100644 --- a/components/homme/src/share/cxx/prim_step.cpp +++ b/components/homme/src/share/cxx/prim_step.cpp @@ -145,10 +145,6 @@ void prim_step_flexible (const Real dt, const bool compute_diagnostics) { if (apply_forcing) { -//printf("OG param nsplit %d, param nsplit iter %d \n",params.nsplit,params.nsplit_iteration); -//std::cout<< "OG in appl forcing for tracers,dt_q " << std::to_string(dt_q)<<"\n"; -//std::cout<< "OG in appl forcing for tracers,dt_q_nsplit " << std::to_string(dt_q_nsplit)<<"\n"; - if (params.ftype == ForcingAlg::FORCING_0) apply_cam_forcing_tracers(dt_q); if (params.ftype == ForcingAlg::FORCING_2) apply_cam_forcing_tracers(dt_q_nsplit); } diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 1b59d4805f88..02b999db16e9 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -20,9 +20,6 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, const SimulationParams& p) { GPTLstart("ApplyCAMForcing_tracers"); -//original -// ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, false, p.moisture); - bool adjustment = false; if ( p.ftype == ForcingAlg::FORCING_0) adjustment = false; diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index c5abc40bfe07..d92c58566d7a 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -199,10 +199,6 @@ class ForcingFunctor constexpr int LAST_INT_PACK = ColInfo::LastPack; constexpr int LAST_INT_PACK_END = ColInfo::LastPackEnd; -//printf("OG in TagStates!!!!!!!"); - - - KernelVariables kv(team, m_tu_states); Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team,NP*NP), [&](const int idx) { @@ -224,16 +220,6 @@ class ForcingFunctor Kokkos::parallel_for(Kokkos::ThreadVectorRange(kv.team,NUM_LEV), [&](const int ilev) { -#if 0 -//printf("i %d,j %d",igp,jgp); -if(kv.ie == 0 && ilev == (NUM_LEV-1) && igp ==0 && jgp == 0){ - //for(int ii=0; ii< 4; ++ii) - // for (int jj=0; jj<4; ++jj){ - // auto u = Homme::subview(m_state.m_v,kv.ie,m_data.np1,0,ii,jj); -//printf("i %d,j %d,u after subcycle is %1.29e \n",ii,jj, u(0)[0]); -printf("OG fvtheta %1.15e , vtheta %1.15e \n",fvtheta(ilev)[7], vtheta(ilev)[7]); -} -#endif vtheta(ilev) += m_dt*fvtheta(ilev); phi(ilev) += m_dt*fphi(ilev); @@ -259,7 +245,6 @@ printf("OG fvtheta %1.15e , vtheta %1.15e \n",fvtheta(ilev)[7], vtheta(ilev)[7]) m_np1_qdp = np1_qdp; m_adjustment = adjustment; -//this should sit in SP m_moist = (moisture==MoistDry::MOIST); Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 7fe29bd0b04b..6ed839947236 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -425,9 +425,6 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv ! Test forcing is only for standalone Homme (and only for some tests/configurations) if (compute_forcing_and_push_to_c) then - -!print *, 'OG IN PUSH FORCING', elem(1)%derived%fvtheta(1,1,72),elem_derived_fvtheta(1,1,72,1) - call compute_test_forcing_dummy(elem,hybrid,hvcoord,tl%n0,n0_qdp,max(dt_q,dt_remap),nets,nete,tl) call t_startf('push_to_cxx') call push_forcing_to_c(elem_derived_FM, elem_derived_FVTheta, elem_derived_FT, & @@ -473,48 +470,6 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv end subroutine prim_run_subcycle -#if 0 - subroutine setup_element_pointers (elem) - use element_state, only : allocate_element_arrays, elem_state_v, elem_state_w_i, elem_state_vtheta_dp, & - elem_state_phinh_i, elem_state_dp3d, elem_state_ps_v, elem_state_phis, & - elem_state_Qdp, elem_state_Q, elem_derived_omega_p, & - elem_accum_pener, elem_accum_kener, elem_accum_iener, & - elem_accum_qvar, elem_accum_qmass, elem_accum_q1mass - ! - ! Inputs - ! - type (element_t), intent(inout) :: elem(:) - ! - ! Locals - ! - integer :: ie - - call allocate_element_arrays(nelemd) - - do ie=1,nelemd - elem(ie)%state%v => elem_state_v(:,:,:,:,:,ie) - elem(ie)%state%w_i => elem_state_w_i(:,:,:,:,ie) - elem(ie)%state%vtheta_dp => elem_state_vtheta_dp(:,:,:,:,ie) - elem(ie)%state%phinh_i => elem_state_phinh_i(:,:,:,:,ie) - elem(ie)%state%dp3d => elem_state_dp3d(:,:,:,:,ie) - elem(ie)%state%ps_v => elem_state_ps_v(:,:,:,ie) - elem(ie)%state%Q => elem_state_Q(:,:,:,:,ie) - elem(ie)%state%Qdp => elem_state_Qdp(:,:,:,:,:,ie) - elem(ie)%state%phis => elem_state_phis(:,:,ie) - elem(ie)%derived%omega_p => elem_derived_omega_p(:,:,:,ie) - - elem(ie)%accum%KEner => elem_accum_KEner (:,:,:,ie) - elem(ie)%accum%PEner => elem_accum_PEner (:,:,:,ie) - elem(ie)%accum%IEner => elem_accum_IEner (:,:,:,ie) - elem(ie)%accum%Qvar => elem_accum_Qvar (:,:,:,:,ie) - elem(ie)%accum%Qmass => elem_accum_Qmass (:,:,:,:,ie) - elem(ie)%accum%Q1mass => elem_accum_Q1mass (:,:,:,ie) - - enddo - - end subroutine setup_element_pointers -#endif - !the next 2 routines have logic for push to/from F and for forcing routine ! From 95c9859cc11b5eb8af3a2b8ac5c1af28ddf583eb Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 16 Jan 2022 19:57:05 -0600 Subject: [PATCH 284/301] removing 0/0 for padding --- .../src/share/cxx/vector/KokkosKernels_Vector_SIMD.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/homme/src/share/cxx/vector/KokkosKernels_Vector_SIMD.hpp b/components/homme/src/share/cxx/vector/KokkosKernels_Vector_SIMD.hpp index 8f8fcf469c16..61a9256404b2 100644 --- a/components/homme/src/share/cxx/vector/KokkosKernels_Vector_SIMD.hpp +++ b/components/homme/src/share/cxx/vector/KokkosKernels_Vector_SIMD.hpp @@ -140,9 +140,10 @@ VECTOR_SIMD_LOOP #else KOKKOS_FORCEINLINE_FUNCTION void debug_set_invalid(int left, int right) { - for(int i = left; i <= right; i++) { - _data[i] = 0.0 / 0.0; - } +// removing this for now as this causes crash in DEBUG mode +// for(int i = left; i <= right; i++) { +// _data[i] = 0.0 / 0.0; +// } } #endif From 771bd912b0503af2554bcacf4acb6a91a92dc266 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 16 Jan 2022 20:31:12 -0600 Subject: [PATCH 285/301] adding ers test, no threads, passes --- cime_config/tests.py | 3 ++- .../testdefs/testmods_dirs/eam/hommexx/shell_commands | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 54ba83135c42..35beca389e41 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -55,7 +55,8 @@ "SMS.ne4_oQU240.F2010.eam-cosplite", "SMS_R_Ld5.ne4_ne4.FSCM5A97.eam-scm", "SMS_D_Ln5.ne4_oQU240.F2010", - "SMS_Ln5.ne4pg2_oQU480.F2010" + "SMS_Ln5.ne4pg2_oQU480.F2010", + "ERS.ne4_oQU240.F2010.eam-hommexx" ) }, diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands index cb43c8c52f90..373b43be5a91 100644 --- a/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/hommexx/shell_commands @@ -1 +1,6 @@ ./xmlchange CAM_TARGET=theta-l_kokkos + +./xmlchange NTHRDS=1 + + + From 00831894ab2d8ce3548b74f865c9630e67b337af Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sun, 16 Jan 2022 20:42:09 -0600 Subject: [PATCH 286/301] remove build xx test --- cime_config/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/tests.py b/cime_config/tests.py index 35beca389e41..26876c76d2d7 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -140,7 +140,6 @@ "SMS.f09_g16_a.IGELM_MLI", "SMS_P12x2.ne4_oQU240.WCYCL1850NS.allactive-mach_mods", "ERS_Ln9.ne4pg2_ne4pg2.F-MMF1.eam-mmf_crmout", - "SMS_B.ne4_ne4.F-EAM-AQP1.eam-hommexx", ) }, From e0cd9105b0185ecbeebe3db0277d4c098e22e4b0 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 17 Jan 2022 14:12:58 -0600 Subject: [PATCH 287/301] following AMB suggestions --- components/eam/src/dynamics/se/dyn_comp.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index 20d4a484ddde..15214ece1ce2 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -113,8 +113,8 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out) integer :: npes_se_stride #ifdef KOKKOS_TARGET -#ifdef _OPENMP - call endrun( 'kokkos dycore does not run with threads') +#if defined(HORIZ_OPENMP) || defined(COLUMN_OPENMP) + call endrun( 'in this EAM configuration, kokkos dycore does not run with threads yet') #endif #endif From bfd9e0d9e248b35de15dc46fffea9679955db0f1 Mon Sep 17 00:00:00 2001 From: noel Date: Tue, 18 Jan 2022 12:56:52 -0800 Subject: [PATCH 288/301] For PM, remove a line to load a module that is no longer needed. Only affects gpu builds. --- cime_config/machines/config_machines.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 33b5c6f18f4c..a7a62b301c8f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -130,13 +130,11 @@ - cpe-cuda cudatoolkit craype-accel-nvidia80 - cpe-cuda cudatoolkit craype-accel-nvidia80 From 4d7f1e5bc853684bc8c63607648b2530e68ea44c Mon Sep 17 00:00:00 2001 From: Walter Hannah Date: Wed, 19 Jan 2022 09:24:45 -0700 Subject: [PATCH 289/301] Update ROF_NCPL for testing AMIP compsets --- driver-mct/cime_config/config_component_e3sm.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver-mct/cime_config/config_component_e3sm.xml b/driver-mct/cime_config/config_component_e3sm.xml index 26be9d252e86..a2baffe42b3d 100644 --- a/driver-mct/cime_config/config_component_e3sm.xml +++ b/driver-mct/cime_config/config_component_e3sm.xml @@ -13,6 +13,7 @@ logical TRUE,FALSE + TRUE run_flags env_run.xml @@ -548,6 +549,7 @@ 6 4 8 + $ATM_NCPL run_coupling env_run.xml From 43d555987d9dbae1689f54356018b1c8ea3b3037 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 19 Jan 2022 15:27:04 -0700 Subject: [PATCH 290/301] Restore inadvertent deletion of FFLAGS from intel_cori-haswell.cmake This was accidentally deleted in 66432b7db334187c8b49c07312e5d58682806e53. Note that I am restoring the version of this line currently on master that was updated Nov. 11, 2021, in ee9eabe95ba8d69e4f7667d006bf1fa417325a24. --- cime_config/machines/cmake_macros/intel_cori-haswell.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake index d526d628ca02..bedb91e9b820 100644 --- a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake @@ -1,3 +1,4 @@ +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml")i if (NOT DEBUG) string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") endif() From 48ed1bb865bf1d8c9e5cfd321be3bff5c47172f8 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 20 Jan 2022 12:35:04 -0700 Subject: [PATCH 291/301] Remove inadvertent stray character that was in previous PR --- cime_config/machines/cmake_macros/intel_cori-haswell.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake index bedb91e9b820..5f40a9786db8 100644 --- a/cime_config/machines/cmake_macros/intel_cori-haswell.cmake +++ b/cime_config/machines/cmake_macros/intel_cori-haswell.cmake @@ -1,4 +1,4 @@ -string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml")i +string(APPEND FFLAGS " -fp-model consistent -fimf-use-svml") if (NOT DEBUG) string(APPEND FFLAGS " -O2 -debug minimal -qno-opt-dynamic-align") endif() From 95a00767d7603d501d9eafa4121b722a930f57a4 Mon Sep 17 00:00:00 2001 From: Balwinder Singh Date: Thu, 20 Jan 2022 17:01:47 -0800 Subject: [PATCH 292/301] Fixes a typo in compy's intel cmake file This typo breaks model compilation in the debug mode on Compy Fixes #4746 [BFB] - Bit-For-Bit --- cime_config/machines/cmake_macros/intel_compy.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/intel_compy.cmake b/cime_config/machines/cmake_macros/intel_compy.cmake index 9e4a6870fd51..c21003ae449c 100644 --- a/cime_config/machines/cmake_macros/intel_compy.cmake +++ b/cime_config/machines/cmake_macros/intel_compy.cmake @@ -3,7 +3,7 @@ if (COMP_NAME STREQUAL gptl) endif() string(APPEND CPPDEFS " -DLINUX") if (DEBUG) - string(APPEND FFLAGS " -checkd all -ftrapuv -init=snan") + string(APPEND FFLAGS " -check all -ftrapuv -init=snan") endif() set(NETCDF_PATH "$ENV{NETCDF_HOME}") set(PNETCDF_PATH "$ENV{PNETCDF_HOME}") From 805d20f07ae0ef8316ee68eee75461440070f10c Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 21 Jan 2022 12:58:56 -0600 Subject: [PATCH 293/301] Update PGI compiler runs of E3SM on GPUs --- cime_config/machines/Depends.pgigpu.cmake | 14 +++++++++++++- .../machines/cmake_macros/pgigpu_swing.cmake | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/Depends.pgigpu.cmake b/cime_config/machines/Depends.pgigpu.cmake index 3a91b92d629e..3517a4d31e31 100644 --- a/cime_config/machines/Depends.pgigpu.cmake +++ b/cime_config/machines/Depends.pgigpu.cmake @@ -94,11 +94,19 @@ set(FILES_NEED_OPENACC_FLAGS # add accelerator/gpu flags for MPAS files set(CPPDEFS "${CPPDEFS} -DMPAS_OPENACC") list(APPEND MPAS_ADD_ACC_FLAGS + ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_si.f90 + ${CMAKE_BINARY_DIR}/core_ocean/mode_forward/mpas_ocn_time_integration_split.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_diagnostics.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_diagnostics_variables.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_equation_of_state_jm.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_equation_of_state_linear.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_equation_of_state_wright.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_mesh.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_surface_bulk_forcing.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_surface_land_ice_fluxes.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_tendency.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_tracer_advection_mono.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_tracer_advection_std.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_forcing_explicit_bottom_drag.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_forcing_surface_stress.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_hadv_coriolis.f90 @@ -106,7 +114,11 @@ list(APPEND MPAS_ADD_ACC_FLAGS ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_hmix_del4.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_hmix_leith.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_pressure_grad.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_tidal_potential.f90 ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vel_vadv.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_vmix.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_tracer_advection_shared.f90 + ${CMAKE_BINARY_DIR}/core_ocean/shared/mpas_ocn_tracer_advection_vert.f90 # seaice ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_mesh_pool.f90 ${CMAKE_BINARY_DIR}/core_seaice/shared/mpas_seaice_velocity_solver_variational.f90 @@ -118,6 +130,6 @@ foreach(ITEM IN LISTS FILES_NEED_OPENACC_FLAGS) endforeach() foreach(ITEM IN LISTS MPAS_ADD_ACC_FLAGS) - e3sm_add_flags("${ITEM}" "-Minline -acc -ta=tesla:ccall,fastmath,loadcache:L1,unroll,fma,deepcopy,nonvvm -Minfo=accel") + e3sm_add_flags("${ITEM}" "-acc -gpu=cc70,cc60,deepcopy -Minfo=accel") endforeach() diff --git a/cime_config/machines/cmake_macros/pgigpu_swing.cmake b/cime_config/machines/cmake_macros/pgigpu_swing.cmake index 955639298c63..6b010b2197be 100644 --- a/cime_config/machines/cmake_macros/pgigpu_swing.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_swing.cmake @@ -6,7 +6,7 @@ if (NOT DEBUG) string(APPEND CFLAGS " -O2 -Mvect=nosimd") string(APPEND FFLAGS " -O2 -Mvect=nosimd -DSUMMITDEV_PGI") endif() -string(APPEND LDFLAGS " -Minline -ta=tesla:ccall,fastmath,loadcache:L1,unroll,fma,managed,deepcopy,nonvvm -Mcuda -Minfo=accel") +string(APPEND LDFLAGS " -gpu=cc70,cc60,deepcopy -Minfo=accel") execute_process(COMMAND $ENV{NETCDF_FORTRAN_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -llapack -lblas") execute_process(COMMAND $ENV{NETCDF_C_PATH}/bin/nc-config --libs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE) From aaef3353c67e944722bad518152939f19a777825 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 21 Jan 2022 14:34:50 -0500 Subject: [PATCH 294/301] Run ne30pg2 tests on 4 nodes on Summit+Ascent to avoid OOM errors --- cime_config/allactive/config_pesall.xml | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 5bd8e31f6b6c..b448e68f2fe4 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -8789,6 +8789,33 @@ + + + + summit|ascent: any compset on ne30np4.pg2 grid + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + + From 53cf0fd850be242534de4a59601c41eb54f04429 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 21 Jan 2022 14:45:29 -0500 Subject: [PATCH 295/301] Update and remove deprecated PGI/NVHPC flags on Summit --- cime_config/machines/cmake_macros/pgigpu_summit.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/cmake_macros/pgigpu_summit.cmake b/cime_config/machines/cmake_macros/pgigpu_summit.cmake index f3ba8dc978ed..ed292a1bdbdf 100644 --- a/cime_config/machines/cmake_macros/pgigpu_summit.cmake +++ b/cime_config/machines/cmake_macros/pgigpu_summit.cmake @@ -7,7 +7,7 @@ endif() if (NOT DEBUG) string(APPEND FFLAGS " -O2 -Mvect=nosimd -DSUMMITDEV_PGI") endif() -string(APPEND LDFLAGS " -Minline -ta=nvidia,cc70,fastmath,loadcache:L1,unroll,fma,managed,ptxinfo -Mcuda -Minfo=accel") +string(APPEND LDFLAGS " -gpu=cc70,cc60,deepcopy -Minfo=accel") string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{HDF5_PATH}/lib -lhdf5_hl -lhdf5 -L$ENV{NETCDF_C_PATH}/lib -lnetcdf -L$ENV{NETCDF_FORTRAN_PATH}/lib -lnetcdff -L$ENV{ESSL_PATH}/lib64 -lessl -L$ENV{OLCF_NETLIB_LAPACK_ROOT}/lib -llapack") set(CXX_LINKER "FORTRAN") string(APPEND CXX_LIBS " -lstdc++") From e3e4fb9a46b2e9a6db58350163e7bac3d031927d Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 21 Jan 2022 14:49:28 -0500 Subject: [PATCH 296/301] Fix missing CPP macro in threaded PGI runs --- components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index 87f3c01608f2..f90d3aea170e 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -1782,8 +1782,10 @@ subroutine ocn_diagnostic_solve_z_coordinates(layerThickness, zMid, zTop, ssh)!{ ssh(iCell) = zTop(minLevelCell(iCell),iCell) end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif end subroutine ocn_diagnostic_solve_z_coordinates!}}} From e5e8122c58cf5e35641dbdb15eafa9c519953f68 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Fri, 21 Jan 2022 15:16:16 -0600 Subject: [PATCH 297/301] fix preqxx build --- .../homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index a34b3aa8443c..a885bb91e527 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -109,7 +109,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, if (ftype == -1) { params.ftype = ForcingAlg::FORCING_OFF; } else if (ftype == 0) { - params.ftype = ForcingAlg::FORCING_DEBUG; + params.ftype = ForcingAlg::FORCING_0; } else if (ftype == 2) { params.ftype = ForcingAlg::FORCING_2; } @@ -173,7 +173,7 @@ void cxx_push_forcing_to_f90(F90Ptr elem_derived_FM, F90Ptr elem_derived_FT, sync_to_host(elements.m_forcing.m_ft, ft_f90); const SimulationParams ¶ms = Context::singleton().get(); - if (params.ftype == ForcingAlg::FORCING_DEBUG) { + if (params.ftype == ForcingAlg::FORCING_0) { if (tracers.fq.data() == nullptr) { tracers.fq = decltype(tracers.fq)("fq", elements.num_elems()); } @@ -198,7 +198,7 @@ void f90_push_forcing_to_cxx(F90Ptr elem_derived_FM, F90Ptr elem_derived_FT, const SimulationParams ¶ms = Context::singleton().get(); Tracers &tracers = Context::singleton().get(); - if (params.ftype == ForcingAlg::FORCING_DEBUG) { + if (params.ftype == ForcingAlg::FORCING_0) { if (tracers.fq.data() == nullptr) { tracers.fq = decltype(tracers.fq)("fq", elements.num_elems()); } From b010d803b5be44beba636fefb076fa8d6bf8bf11 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Sat, 22 Jan 2022 12:04:36 -0500 Subject: [PATCH 298/301] Update pe-layout --- cime_config/allactive/config_pesall.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index b448e68f2fe4..a59deff3dac2 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -8794,14 +8794,14 @@ summit|ascent: any compset on ne30np4.pg2 grid - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + -2 + -2 + -2 + -2 + -2 + -2 + -2 + -2 1 From 2845fa7e176bc8354680dde696f3987d4da4f58d Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sat, 22 Jan 2022 17:36:10 -0600 Subject: [PATCH 299/301] fix preqx build --- components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 | 4 ++-- components/homme/src/preqx_kokkos/prim_driver_mod.F90 | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 index 0b8ad51bd21c..8d0c271526b9 100644 --- a/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 +++ b/components/homme/src/preqx_kokkos/preqx_f2c_mod.F90 @@ -137,12 +137,12 @@ subroutine push_forcing_to_c(FM, FT, FQ) bind(c) end subroutine push_forcing_to_c ! Run dycore for a full atm timesteps - subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step) bind(c) + subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step,nsplit_iteration) bind(c) use iso_c_binding, only: c_int, c_double ! ! Inputs ! - integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step + integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step, nsplit_iteration real (kind=c_double), intent(in) :: tstep end subroutine prim_run_subcycle_c diff --git a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 index ad31e0ec1523..549a198cbc64 100644 --- a/components/homme/src/preqx_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/preqx_kokkos/prim_driver_mod.F90 @@ -202,7 +202,7 @@ subroutine prim_init_kokkos_functors () end subroutine prim_init_kokkos_functors - subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hvcoord,nsubstep) + subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hvcoord, nsplit_iteration) use iso_c_binding, only : c_int, c_ptr, c_loc use control_mod, only : qsplit, rsplit, statefreq use dimensions_mod, only : nelemd @@ -218,12 +218,12 @@ subroutine prim_run_subcycle(elem, hybrid, nets, nete, dt, single_column, tl, hv use perf_mod, only: t_startf, t_stopf use prim_state_mod, only: prim_printstate interface - subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step) bind(c) + subroutine prim_run_subcycle_c(tstep,nstep,nm1,n0,np1,next_output_step,nsplit_iteration) bind(c) use iso_c_binding, only: c_int, c_double ! ! Inputs ! - integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step + integer(kind=c_int), intent(in) :: nstep, nm1, n0, np1, next_output_step, nsplit_iteration real (kind=c_double), intent(in) :: tstep end subroutine prim_run_subcycle_c @@ -251,7 +251,7 @@ end subroutine cxx_push_results_to_f90 real(kind=real_kind), intent(in) :: dt ! "timestep dependent" timestep logical, intent(in) :: single_column type (TimeLevel_t), intent(inout) :: tl - integer, intent(in) :: nsubstep ! nsubstep = 1 .. nsplit + integer, intent(in) :: nsplit_iteration ! nsplit_iteration = 1 .. nsplit ! ! Locals ! @@ -274,7 +274,7 @@ end subroutine cxx_push_results_to_f90 compute_diagnostics = .true. endif - call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep) + call prim_run_subcycle_c(dt,nstep_c,nm1_c,n0_c,np1_c,nextOutputStep,nsplit_iteration) ! Set final timelevels from C into Fortran structure tl%nstep = nstep_c From 601e5125026c066277a8c9281ca691f48fd41a74 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Sat, 22 Jan 2022 17:36:47 -0600 Subject: [PATCH 300/301] fix runs with debug flags, init was missing --- components/homme/src/share/physical_constants.F90 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/homme/src/share/physical_constants.F90 b/components/homme/src/share/physical_constants.F90 index d2909526f902..b4c9aa10c47a 100644 --- a/components/homme/src/share/physical_constants.F90 +++ b/components/homme/src/share/physical_constants.F90 @@ -76,7 +76,11 @@ module physical_constants #endif real (kind=real_kind), public :: scale_factor, scale_factor_inv, domain_size, laplacian_rigid_factor -real (kind=real_kind), public :: Lx, Ly, Sx, Sy, dx, dy, dx_ref, dy_ref +real (kind=real_kind), public :: Lx = 0.0 +real (kind=real_kind), public :: Ly = 0.0 +real (kind=real_kind), public :: Sx = 0.0 +real (kind=real_kind), public :: Sy = 0.0 +real (kind=real_kind), public :: dx, dy, dx_ref, dy_ref real (kind=real_kind), public, parameter :: TREF = 288 From c8866c524366e9e1378139b579ab1be252f30472 Mon Sep 17 00:00:00 2001 From: Oksana Guba Date: Mon, 24 Jan 2022 16:33:15 -0600 Subject: [PATCH 301/301] fix to finalize --- components/eam/src/dynamics/se/stepon.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/eam/src/dynamics/se/stepon.F90 b/components/eam/src/dynamics/se/stepon.F90 index 44c3b7071566..ffc9a445cdc5 100644 --- a/components/eam/src/dynamics/se/stepon.F90 +++ b/components/eam/src/dynamics/se/stepon.F90 @@ -604,7 +604,7 @@ end subroutine stepon_run3 subroutine stepon_final(dyn_in, dyn_out) use dyn_grid, only: fv_physgrid_final, fv_nphys use cam_logfile, only: iulog - use prim_driver_base,only: prim_finalize + use prim_driver_mod,only: prim_finalize ! !PARAMETERS: ! WARNING: intent(out) here means that pointers in dyn_in and dyn_out ! are nullified. Unless this memory is released in some other routine,