From 45bc7e4eabc5c8c148d15e8a044e58635255db1e Mon Sep 17 00:00:00 2001 From: Graham Harper Date: Tue, 29 Oct 2024 18:06:34 -0600 Subject: [PATCH] Add build infrastructure, compsets, and basis stat computations using cldera-tools For more information, see Luca's PR #7 in CLDERA-E3SM --- components/cmake/build_model.cmake | 11 ++ components/cmake/common_setup.cmake | 4 + components/eam/bld/configure | 5 + .../eam/cime_config/config_component.xml | 4 + .../eam/cime_config/config_compsets.xml | 5 + .../usermods_dirs/volcaero/shell_commands | 1 + .../usermods_dirs/volcaero/user_nl_eam | 18 +++ components/eam/src/control/cam_comp.F90 | 103 ++++++++++++++++++ components/eam/src/cpl/atm_comp_mct.F90 | 19 ++++ .../eam/src/physics/cam/physics_buffer.F90.in | 74 +++++++++++++ .../elm/cime_config/config_component.xml | 1 + 11 files changed, 245 insertions(+) create mode 100644 components/eam/cime_config/usermods_dirs/volcaero/shell_commands create mode 100644 components/eam/cime_config/usermods_dirs/volcaero/user_nl_eam diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index f0e6298d6850..bf081d6bd46d 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -305,6 +305,17 @@ macro(build_model COMP_CLASS COMP_NAME) if (USE_RRTMGPXX) target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface) endif() + if (USE_CLDERA_PROFILING) + if (DEBUG) + set (CLDERA_PATH_POSTFIX debug) + else() + set (CLDERA_PATH_POSTFIX release) + endif() + message ("CLDERA root DIR: ${CLDERA_PATH}/${CLDERA_PATH_POSTFIX}") + find_package(Cldera REQUIRED PATHS ${CLDERA_PATH}/${CLDERA_PATH_POSTFIX}) + target_link_libraries(${TARGET_NAME} PRIVATE cldera-profiling) + target_compile_definitions(${TARGET_NAME} PRIVATE CLDERA_PROFILING) + endif() endif() if (COMP_NAME STREQUAL "elm") if (USE_PETSC) diff --git a/components/cmake/common_setup.cmake b/components/cmake/common_setup.cmake index 5522687bd396..b0e2f30f5ab5 100644 --- a/components/cmake/common_setup.cmake +++ b/components/cmake/common_setup.cmake @@ -48,6 +48,10 @@ else() set(USE_YAKL FALSE) endif() +# Detect if CLDERA profiling tool lib has to be used +string(FIND "${CAM_CONFIG_OPTS}" "-cldera_profiling" USE_CLDERA_PROFILING) + + #=============================================================================== # set CPP options (must use this before any flags or cflags settings) #=============================================================================== diff --git a/components/eam/bld/configure b/components/eam/bld/configure index c11f89599a60..7d9113e6b102 100755 --- a/components/eam/bld/configure +++ b/components/eam/bld/configure @@ -291,6 +291,10 @@ OPTIONS Options for radiation: -rrtmgpxx Use RRTMGP++ code + + Options for CLDERA-E3SM + + -cldera_profiling Build EAM with support for CLDERA profiling, linking to cldera-profiling EOF } @@ -367,6 +371,7 @@ GetOptions( "crm=s" => \$opts{'crm'}, "pam_dycor=s" => \$opts{'pam_dycor'}, "rrtmgpxx" => \$opts{'rrtmgpxx'}, + "cldera_profiling" => \$opts{'cldera_profiling'}, "debug" => \$opts{'debug'}, "rain_evap_to_coarse_aero" => \$opts{'rain_evap_to_coarse_aero'}, "bc_dep_to_snow_updates" => \$opts{'bc_dep_to_snow_updates'}, diff --git a/components/eam/cime_config/config_component.xml b/components/eam/cime_config/config_component.xml index 785a6469650d..dd8a567964c4 100755 --- a/components/eam/cime_config/config_component.xml +++ b/components/eam/cime_config/config_component.xml @@ -55,6 +55,9 @@ -bc_dep_to_snow_updates -co2_cycle + + -cldera_profiling + -nlev 72 @@ -237,6 +240,7 @@ $SRCROOT/components/eam/cime_config/usermods_dirs/aquap $SRCROOT/components/eam/cime_config/usermods_dirs/rcemip $SRCROOT/components/eam/cime_config/usermods_dirs/scm + $SRCROOT/components/eam/cime_config/usermods_dirs/volcaero run_component_cam env_case.xml diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 6b3c41321eea..567d382351fd 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -41,6 +41,11 @@ 20TR_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + F20TR-CLDERA + 20TR_EAM%CMIP6-CLDERA_ELM%SPBC_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV + + F2010-CICE 2010_EAM%CMIP6_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV diff --git a/components/eam/cime_config/usermods_dirs/volcaero/shell_commands b/components/eam/cime_config/usermods_dirs/volcaero/shell_commands new file mode 100644 index 000000000000..e82c3fb89661 --- /dev/null +++ b/components/eam/cime_config/usermods_dirs/volcaero/shell_commands @@ -0,0 +1 @@ +./xmlchange RUN_STARTDATE="1991-01-01" diff --git a/components/eam/cime_config/usermods_dirs/volcaero/user_nl_eam b/components/eam/cime_config/usermods_dirs/volcaero/user_nl_eam new file mode 100644 index 000000000000..dda59f9089fd --- /dev/null +++ b/components/eam/cime_config/usermods_dirs/volcaero/user_nl_eam @@ -0,0 +1,18 @@ +ext_frc_specifier = + 'SO2 -> /sems-data-store/ACME/cldera/data/E3SM/model_input/emissions/volc/merge_cmip6_and_volc_so2/merged_cmip6_and_volc_so2.nc', + 'SOAG -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1850-2014_c180205.nc', + 'bc_a4 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc', + 'num_a1 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc', + 'num_a2 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc', + 'num_a4 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc', + 'pom_a4 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc', + 'so4_a1 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc', + 'so4_a2 -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc' + +prescribed_volcaero_filetype = 'VOLC_MIXING_RATIO' +prescribed_volcaero_datapath = '/sems-data-store/ACME/cldera/data/E3SM/model_input/emissions/volc/zero-presc-volc-mmr/' +prescribed_volcaero_file = 'BMW_volcanic_1850-3009_all_zero.nc' + +use_hetfrz_classnuc = .true. +hist_hetfrz_classnuc = .true. + diff --git a/components/eam/src/control/cam_comp.F90 b/components/eam/src/control/cam_comp.F90 index ddb453e36073..b6006fe2c340 100644 --- a/components/eam/src/control/cam_comp.F90 +++ b/components/eam/src/control/cam_comp.F90 @@ -100,6 +100,19 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, & use cam_pio_utils, only: init_pio_subsystem use cam_instance, only: inst_suffix use conditional_diag, only: cnd_diag_info, cnd_diag_alloc +#if defined(CLDERA_PROFILING) +use iso_c_binding, only: c_loc + use cldera_interface_mod, only: cldera_add_partitioned_field, max_str_len, & + cldera_get_field_names, & + cldera_set_field_partition, & + cldera_commit_all_fields + use physics_buffer, only: physics_buffer_desc, col_type_grid, pbuf_get_index, & + pbuf_get_field_rank, pbuf_get_field_dims, pbuf_get_field, & + pbuf_get_field_name, pbuf_has_field + use ppgrid, only: begchunk, endchunk, pcols, pver + use phys_grid, only: get_ncols_p + use constituents, only: cnst_get_ind +#endif #if ( defined SPMD ) real(r8) :: mpi_wtime ! External @@ -126,6 +139,16 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, & integer :: dtime_cam ! Time-step logical :: log_print ! Flag to print out log information or not character(len=cs) :: filein ! Input namelist filename +#if defined(CLDERA_PROFILING) + character(len=max_str_len), allocatable :: cldera_field_names(:) + character(len=max_str_len) :: fname + integer :: c, nfields, idx, ifield, rank, icmp, nparts, part_dim, ipart, fsize, ncols + integer :: nlcols + integer, allocatable :: dims(:), dims2(:) + logical :: in_pbuf, in_q + real(r8), pointer :: field1d(:), field2d(:,:), field3d(:,:,:) + type(physics_buffer_desc), pointer :: field_desc +#endif !----------------------------------------------------------------------- etamid = nan ! @@ -205,6 +228,86 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, & if (single_column) call iop_intht() call intht() +#if defined(CLDERA_PROFILING) + cldera_field_names = cldera_get_field_names() + nfields = size(cldera_field_names) + nparts = endchunk - begchunk + 1 + ! All fields are partitioned over cols index, which is the first + part_dim = 1 + in_pbuf = .false. + in_q = .false. + ! do ifield=1,40 + ! print *, pbuf_get_field_name(ifield) + ! enddo + allocate(dims2(2)) + ! print *, "adding cldera fields" + ! print *, "pcols,pver:", pcols, pver + ! print *, "begchunk, endchunk:", begchunk, endchunk + nlcols = 0 + do c = begchunk,endchunk + nlcols = nlcols + get_ncols_p(c) + enddo + ! print *, "num my cols:",nlcols + + do ifield=1,nfields + fname = cldera_field_names(ifield) + + ! print *, "- adding "//fname + + if (pbuf_has_field(fname)) then + ! Use pbuf + idx = pbuf_get_index(fname) + in_pbuf = .true. + field_desc => pbuf2d(idx,begchunk) + + ! Figure out rank + rank = pbuf_get_field_rank(idx) + dims = pbuf_get_field_dims(idx) + else + ! Try check constituents list. + ! NOTE: for now, if not in pbuf or Q, let's abort. + ! Later on, you *may* want to explore other options + call cnst_get_ind(fname,ind=idx,abrtf=.true.) + if (idx .ge. 1) then + in_q = .true. + + ! Rank and dims are given by Q + rank = 2 + dims2(2) = pver + dims = dims2 + endif + endif + dims(1) = nlcols + ! print *, " rank:", rank + ! print *, " dims:", dims + + call cldera_add_partitioned_field(fname,dims,nparts,part_dim) + do ipart = 1,nparts + c = begchunk+ipart-1 ! Chunk + if (in_pbuf) then + ncols = get_ncols_p(c) + if (rank .eq. 1) then + call pbuf_get_field(pbuf2d, c, idx, field1d) + call cldera_set_field_partition(fname,ipart,ncols,field1d) + elseif (rank .eq. 2) then + call pbuf_get_field(pbuf2d, c, idx, field2d) + call cldera_set_field_partition(fname,ipart,ncols,field2d) + else + call pbuf_get_field(pbuf2d, c, idx, field3d) + call cldera_set_field_partition(fname,ipart,ncols,field3d) + endif + else + ncols = phys_state(c)%ncol + field2d=>phys_state(c)%q(:,:,idx) + call cldera_set_field_partition(fname,ipart,ncols,field2d) + endif + enddo + enddo + + call cldera_commit_all_fields() +#endif + + end subroutine cam_init ! diff --git a/components/eam/src/cpl/atm_comp_mct.F90 b/components/eam/src/cpl/atm_comp_mct.F90 index ca16ec46bee3..a71287e6a6fb 100644 --- a/components/eam/src/cpl/atm_comp_mct.F90 +++ b/components/eam/src/cpl/atm_comp_mct.F90 @@ -140,6 +140,9 @@ module atm_comp_mct !================================================================================ subroutine atm_init_mct( EClock, cdata_a, x2a_a, a2x_a, NLFilename ) +#if defined(CLDERA_PROFILING) + use cldera_interface_mod, only: cldera_init +#endif !----------------------------------------------------------------------- ! @@ -224,6 +227,10 @@ subroutine atm_init_mct( EClock, cdata_a, x2a_a, a2x_a, NLFilename ) #endif if (first_time) then + +#if defined(CLDERA_PROFILING) + call cldera_init(mpicom_atm) +#endif call cam_instance_init(ATMID) @@ -570,6 +577,9 @@ subroutine atm_run_mct( EClock, cdata_a, x2a_a, a2x_a) use pmgrid, only: plev, plevp use constituents, only: pcnst use shr_sys_mod, only: shr_sys_flush +#if defined(CLDERA_PROFILING) + use cldera_interface_mod, only: cldera_compute_stats +#endif ! ! Arguments @@ -800,11 +810,16 @@ subroutine atm_run_mct( EClock, cdata_a, x2a_a, a2x_a) endif #endif + call cldera_compute_stats(ymd,tod) + end subroutine atm_run_mct !================================================================================ subroutine atm_final_mct( EClock, cdata_a, x2a_a, a2x_a) +#if defined(CLDERA_PROFILING) + use cldera_interface_mod, only: cldera_clean_up +#endif type(ESMF_Clock) ,intent(inout) :: EClock type(seq_cdata) ,intent(inout) :: cdata_a @@ -815,6 +830,10 @@ subroutine atm_final_mct( EClock, cdata_a, x2a_a, a2x_a) call cam_final( cam_out, cam_in ) call t_stopf('cam_final') +#if defined(CLDERA_PROFILING) + call cldera_clean_up () +#endif + end subroutine atm_final_mct !================================================================================ diff --git a/components/eam/src/physics/cam/physics_buffer.F90.in b/components/eam/src/physics/cam/physics_buffer.F90.in index 9a95ab3fe0bb..bf996e9df7a0 100644 --- a/components/eam/src/physics/cam/physics_buffer.F90.in +++ b/components/eam/src/physics/cam/physics_buffer.F90.in @@ -124,7 +124,10 @@ module physics_buffer pbuf_old_tim_idx, &! return the index for the oldest time pbuf_update_tim_idx, &! update the index for the oldest time pbuf_col_type_index, & + pbuf_has_field, & pbuf_get_field_name, & + pbuf_get_field_rank, & + pbuf_get_field_dims, & pbuf_get_field, & pbuf_add_field, & pbuf_register_subcol, & @@ -235,6 +238,25 @@ CONTAINS ! Return a pointer to the current chunks physics_buffer. ! + function pbuf_has_field(fname) result(found) + character(len=16) :: fname + logical :: found + + integer :: i + type(physics_buffer_hdr), pointer :: hdr + + found = .false. + hdr => hdrbuffertop + + do while (associated(hdr)) + if (trim(hdr%name) == trim(fname)) then + found = .true. + exit + endif + hdr=>hdr%nexthdr + end do + end function pbuf_has_field + function pbuf_get_field_name(index) integer, intent(in) :: index character(len=16) :: pbuf_get_field_name @@ -763,6 +785,58 @@ CONTAINS end function pbuf_get_index + function pbuf_get_field_rank (index,col_type) result(rank) + + integer, intent(in) :: index + integer, intent(in), optional :: col_type + + type(physics_buffer_hdr), pointer :: hdr + integer :: col_type_use, i, rank + + if (present(col_type)) then + col_type_use = col_type + else + col_type_use = col_type_grid + endif + + hdr => hdrbuffertop + do i=2,index + hdr=>hdr%nexthdr + enddo + + rank = 6; + do i=6,1,-1 + if (hdr%dimsizes(i,col_type_use) .ne. 1) then + exit + endif + rank = rank-1 + enddo + end function pbuf_get_field_rank + + function pbuf_get_field_dims (index,col_type) result(dims) + + integer, intent(in) :: index + integer, intent(in), optional :: col_type + + type(physics_buffer_hdr), pointer :: hdr + integer :: col_type_use, i + integer, allocatable :: dims(:) + + if (present(col_type)) then + col_type_use = col_type + else + col_type_use = col_type_grid + endif + + hdr => hdrbuffertop + do i=2,index + hdr=>hdr%nexthdr + end do + + allocate(dims(pbuf_get_field_rank(index,col_type))) + dims = hdr%dimsizes(:,col_type_use) + end function pbuf_get_field_dims + !========================================================================================= ! diff --git a/components/elm/cime_config/config_component.xml b/components/elm/cime_config/config_component.xml index 4e637ef55802..269a4aee32da 100755 --- a/components/elm/cime_config/config_component.xml +++ b/components/elm/cime_config/config_component.xml @@ -83,6 +83,7 @@ 20thC_CMIP6_transient 20thC_CMIP6_transient 20thC_CMIP6bgc_transient + 20thC_CMIP6_transient 20thC_CMIP6bgc_transient 20thC_bgc_transient 1950_CMIP6_control