Merge pull request E3SM-Project#7 from sandialabs/bartgol/cldera-tool…

…s-hooks

Add hooks to be able to link against cldera-profiling tool

cime_config/machines/cmake_macros/gnu_mappy.cmake

@@ -1,4 +1,5 @@
 set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
+set(CLDERA_PATH "/sems-data-store/ACME/cldera/cldera-tools/install")
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
 endif()

components/cmake/build_model.cmake

@@ -274,7 +274,18 @@ function(build_model COMP_CLASS COMP_NAME)
         target_link_libraries(${TARGET_NAME} PRIVATE samxx)
       endif()
       if (USE_RRTMGPXX)
-          target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
+        target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
+      endif()
+      if (USE_CLDERA_PROFILING)
+        if (DEBUG)
+          set (CLDERA_PATH_POSTFIX debug)
+        else()
+          set (CLDERA_PATH_POSTFIX release)
+        endif()
+        message ("CLDERA root DIR: ${CLDERA_PATH}/${CLDERA_PATH_POSTFIX}")
+        find_package(Cldera REQUIRED PATHS ${CLDERA_PATH}/${CLDERA_PATH_POSTFIX})
+        target_link_libraries(${TARGET_NAME} PRIVATE cldera-profiling)
+        target_compile_definitions(${TARGET_NAME} PRIVATE CLDERA_PROFILING)
       endif()
     endif()
     if (USE_KOKKOS)

components/cmake/common_setup.cmake

@@ -53,6 +53,10 @@ if (${USE_YAKL})
   set(USE_CXX TRUE)
 endif()
 
+# Detect if CLDERA profiling tool lib has to be used
+string(FIND "${CAM_CONFIG_OPTS}" "-cldera_profiling" USE_CLDERA_PROFILING)
+
+
 #===============================================================================
 # set CPP options (must use this before any flags or cflags settings)
 #===============================================================================

components/eam/bld/configure

@@ -280,6 +280,10 @@ OPTIONS
      -crm_adv           CRM advection scheme [MPDATA | UM5]
      -crm <model>       CRM model [sam | samomp | samxx]
      -rrtmgpxx          Use RRTMGP++ code
+
+  Options for CLDERA-E3SM
+
+     -cldera_profiling  Build EAM with support for CLDERA profiling, linking to cldera-profiling
 EOF
 }
 
@@ -353,6 +357,7 @@ GetOptions(
     "crm_adv=s"                 => \$opts{'crm_adv'},
     "crm=s"                     => \$opts{'crm'},
     "rrtmgpxx"                  => \$opts{'rrtmgpxx'},
+    "cldera_profiling"          => \$opts{'cldera_profiling'},
     "debug"                     => \$opts{'debug'},
     "rain_evap_to_coarse_aero"  => \$opts{'rain_evap_to_coarse_aero'},
     "bc_dep_to_snow_updates"    => \$opts{'bc_dep_to_snow_updates'},

components/eam/cime_config/config_component.xml

@@ -68,6 +68,9 @@
       <value compset="_EAM%MMF.*-RCE"  >-aquaplanet -rce </value>
       <value compset="_EAM%MMF.*-SCM"  >-scam </value>
 
+      <!-- CLDERA support -->
+      <value compset="CLDERA"        >-cldera_profiling</value>
+
       <!--  -->
       <value compset="GEOS_EAM"       >-offline_dyn</value>
       <value compset="GEOS_EAM"      >-nlev 56</value>
@@ -240,6 +243,7 @@
     <value compset="EAM%MMF.*-RCE">$SRCROOT/components/eam/cime_config/usermods_dirs/rcemip</value>
     <value compset="EAM%MMF.*-SCM">$SRCROOT/components/eam/cime_config/usermods_dirs/scm</value>
     <value compset="EAM%SCAM">$SRCROOT/components/eam/cime_config/usermods_dirs/scm</value>
+    <value compset="EAM%CMIP6-CLDERA">$SRCROOT/components/eam/cime_config/usermods_dirs/volcaero</value>
     </values>
     <group>run_component_cam</group>
     <file>env_case.xml</file>

components/eam/cime_config/config_compsets.xml

@@ -51,6 +51,11 @@
     <lname>20TR_EAM%CMIP6_ELM%SPBC_MPASSI%PRES_DOCN%DOM_MOSART_SGLC_SWAV</lname>
   </compset>
 
+  <compset>
+    <alias>F20TR-CLDERA</alias>
+    <lname>20TR_EAM%CMIP6-CLDERA_ELM%SPBC_MPASSI%PRES_DOCN%DOM_SROF_SGLC_SWAV</lname>
+  </compset>
+
   <compset>
     <alias>F2010-CICE</alias>
     <lname>2010_EAM%CMIP6_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV</lname>

components/eam/cime_config/usermods_dirs/volcaero/shell_commands

@@ -0,0 +1 @@
+./xmlchange RUN_STARTDATE="1991-01-01"

components/eam/cime_config/usermods_dirs/volcaero/user_nl_eam

@@ -0,0 +1,18 @@
+ext_frc_specifier    = 
+        'SO2         -> /sems-data-store/ACME/cldera/data/E3SM/model_input/emissions/volc/merge_cmip6_and_volc_so2/merged_cmip6_and_volc_so2.nc',
+        'SOAG        -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_soag_elev_1850-2014_c180205.nc',
+        'bc_a4       -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_bc_a4_elev_1850-2014_c180205.nc',
+        'num_a1      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a1_elev_1850-2014_c180205.nc',
+        'num_a2      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a2_elev_1850-2014_c180205.nc',
+        'num_a4      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_num_a4_elev_1850-2014_c180205.nc',
+        'pom_a4      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_pom_a4_elev_1850-2014_c180205.nc',
+        'so4_a1      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a1_elev_1850-2014_c180205.nc',
+        'so4_a2      -> $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/emis/DECK_ne30/cmip6_mam4_so4_a2_elev_1850-2014_c180205.nc'
+
+prescribed_volcaero_filetype = 'VOLC_MIXING_RATIO'
+prescribed_volcaero_datapath = '/sems-data-store/ACME/cldera/data/E3SM/model_input/emissions/volc/zero-presc-volc-mmr/'
+prescribed_volcaero_file = 'BMW_volcanic_1850-3009_all_zero.nc'
+
+use_hetfrz_classnuc  = .true.
+hist_hetfrz_classnuc = .true.
+

components/eam/src/control/cam_comp.F90

@@ -97,6 +97,19 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, &
    use scamMod,          only: single_column
    use cam_pio_utils,    only: init_pio_subsystem
    use cam_instance,     only: inst_suffix
+#if defined(CLDERA_PROFILING)
+use iso_c_binding, only: c_loc
+   use cldera_interface_mod, only: cldera_add_partitioned_field, max_str_len, &
+                                   cldera_get_field_names, &
+                                   cldera_set_field_partition, &
+                                   cldera_commit_all_fields
+   use physics_buffer,   only: physics_buffer_desc, col_type_grid, pbuf_get_index, &
+                               pbuf_get_field_rank, pbuf_get_field_dims, pbuf_get_field, &
+                               pbuf_get_field_name, pbuf_has_field
+   use ppgrid,           only: begchunk, endchunk, pcols, pver
+   use phys_grid,        only: get_ncols_p
+   use constituents,     only: cnst_get_ind
+#endif
 
 #if ( defined SPMD )   
    real(r8) :: mpi_wtime  ! External
@@ -123,6 +136,16 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, &
    integer :: dtime_cam        ! Time-step
    logical :: log_print        ! Flag to print out log information or not
    character(len=cs) :: filein ! Input namelist filename
+#if defined(CLDERA_PROFILING)
+   character(len=max_str_len), allocatable :: cldera_field_names(:)
+   character(len=max_str_len) :: fname
+   integer :: c, nfields, idx, ifield, rank, icmp, nparts, part_dim, ipart, fsize, ncols
+   integer :: nlcols
+   integer, allocatable :: dims(:), dims2(:)
+   logical :: in_pbuf, in_q
+   real(r8), pointer :: field1d(:), field2d(:,:), field3d(:,:,:)
+   type(physics_buffer_desc), pointer :: field_desc
+#endif
    !-----------------------------------------------------------------------
    etamid = nan
    !
@@ -198,6 +221,86 @@ subroutine cam_init( cam_out, cam_in, mpicom_atm, &
    if (single_column) call scm_intht()
    call intht()
 
+#if defined(CLDERA_PROFILING)
+   cldera_field_names = cldera_get_field_names()
+   nfields = size(cldera_field_names)
+   nparts = endchunk - begchunk + 1
+   ! All fields are partitioned over cols index, which is the first
+   part_dim = 1
+   in_pbuf = .false.
+   in_q = .false.
+   ! do ifield=1,40
+   !    print *, pbuf_get_field_name(ifield)
+   ! enddo
+   allocate(dims2(2))
+   ! print *, "adding cldera fields"
+   ! print *, "pcols,pver:", pcols, pver
+   ! print *, "begchunk, endchunk:", begchunk, endchunk
+   nlcols = 0
+   do c = begchunk,endchunk
+    nlcols = nlcols +  get_ncols_p(c)
+   enddo
+   ! print *, "num my cols:",nlcols
+
+   do ifield=1,nfields
+     fname = cldera_field_names(ifield)
+
+     ! print *, "- adding "//fname
+
+     if (pbuf_has_field(fname)) then
+       ! Use pbuf
+       idx = pbuf_get_index(fname)
+       in_pbuf = .true.
+       field_desc => pbuf2d(idx,begchunk)
+
+       ! Figure out rank
+       rank = pbuf_get_field_rank(idx)
+       dims = pbuf_get_field_dims(idx)
+     else
+       ! Try check constituents list.
+       ! NOTE: for now, if not in pbuf or Q, let's abort.
+       !       Later on, you *may* want to explore other options
+       call cnst_get_ind(fname,ind=idx,abrtf=.true.)
+       if (idx .ge. 1) then
+         in_q = .true.
+
+         ! Rank and dims are given by Q
+         rank = 2
+         dims2(2) = pver
+         dims = dims2
+       endif
+     endif
+     dims(1) = nlcols
+     ! print *, "  rank:", rank
+     ! print *, "  dims:", dims
+
+     call cldera_add_partitioned_field(fname,dims,nparts,part_dim)
+     do ipart = 1,nparts
+       c = begchunk+ipart-1 ! Chunk
+       if (in_pbuf) then
+         ncols = get_ncols_p(c)
+         if (rank .eq. 1) then
+           call pbuf_get_field(pbuf2d, c, idx, field1d)
+           call cldera_set_field_partition(fname,ipart,ncols,field1d)
+         elseif (rank .eq. 2) then
+           call pbuf_get_field(pbuf2d, c, idx, field2d)
+           call cldera_set_field_partition(fname,ipart,ncols,field2d)
+         else 
+           call pbuf_get_field(pbuf2d, c, idx, field3d)
+           call cldera_set_field_partition(fname,ipart,ncols,field3d)
+         endif
+       else
+         ncols = phys_state(c)%ncol
+         field2d=>phys_state(c)%q(:,:,idx)
+         call cldera_set_field_partition(fname,ipart,ncols,field2d)
+       endif
+     enddo
+   enddo
+
+   call cldera_commit_all_fields()
+#endif
+
+
 end subroutine cam_init
 
 !

components/eam/src/cpl/atm_comp_mct.F90

@@ -104,6 +104,9 @@ module atm_comp_mct
 !================================================================================
 
   subroutine atm_init_mct( EClock, cdata_a, x2a_a, a2x_a, NLFilename )
+#if defined(CLDERA_PROFILING)
+    use cldera_interface_mod, only: cldera_init
+#endif
 
     !-----------------------------------------------------------------------
     !
@@ -174,6 +177,10 @@ subroutine atm_init_mct( EClock, cdata_a, x2a_a, a2x_a, NLFilename )
          gsMap=gsMap_atm, dom=dom_a, infodata=infodata)
 
     if (first_time) then
+
+#if defined(CLDERA_PROFILING)
+       call cldera_init(mpicom_atm)
+#endif
 
        call cam_instance_init(ATMID)
 
@@ -469,6 +476,9 @@ subroutine atm_run_mct( EClock, cdata_a, x2a_a, a2x_a)
     use pmgrid,          only: plev, plevp
     use constituents,    only: pcnst
     use shr_sys_mod,     only: shr_sys_flush
+#if defined(CLDERA_PROFILING)
+    use cldera_interface_mod, only: cldera_compute_stats
+#endif
 
     ! 
     ! Arguments
@@ -657,11 +667,16 @@ subroutine atm_run_mct( EClock, cdata_a, x2a_a, a2x_a)
     endif
 #endif
 
+    call cldera_compute_stats(ymd,tod)
+
   end subroutine atm_run_mct
 
   !================================================================================
 
   subroutine atm_final_mct( EClock, cdata_a, x2a_a, a2x_a)
+#if defined(CLDERA_PROFILING)
+    use cldera_interface_mod, only: cldera_clean_up
+#endif
 
     type(ESMF_Clock)            ,intent(inout) :: EClock
     type(seq_cdata)             ,intent(inout) :: cdata_a
@@ -672,6 +687,10 @@ subroutine atm_final_mct( EClock, cdata_a, x2a_a, a2x_a)
     call cam_final( cam_out, cam_in )
     call t_stopf('cam_final')
 
+#if defined(CLDERA_PROFILING)
+   call cldera_clean_up ()
+#endif
+
   end subroutine atm_final_mct
 
   !================================================================================

components/eam/src/physics/cam/physics_buffer.F90.in

@@ -124,7 +124,10 @@ module physics_buffer
             pbuf_old_tim_idx,     &! return the index for the oldest time
             pbuf_update_tim_idx,  &! update the index for the oldest time
             pbuf_col_type_index,  &
+            pbuf_has_field,       &
             pbuf_get_field_name,  &
+            pbuf_get_field_rank,  &
+            pbuf_get_field_dims,  &
             pbuf_get_field,       &
             pbuf_add_field,       &
             pbuf_register_subcol, &
@@ -235,6 +238,25 @@ CONTAINS
   !  Return a pointer to the current chunks physics_buffer.
   !
 
+  function pbuf_has_field(fname) result(found)
+    character(len=16) :: fname
+    logical :: found
+
+    integer :: i
+    type(physics_buffer_hdr), pointer :: hdr
+
+    found = .false.
+    hdr => hdrbuffertop
+
+    do while (associated(hdr))
+       if (trim(hdr%name) == trim(fname)) then
+         found = .true.
+         exit
+       endif
+       hdr=>hdr%nexthdr
+    end do
+  end function pbuf_has_field
+
   function pbuf_get_field_name(index)
     integer, intent(in) :: index
     character(len=16) :: pbuf_get_field_name
@@ -763,6 +785,58 @@ CONTAINS
 
   end function pbuf_get_index
 
+  function pbuf_get_field_rank (index,col_type) result(rank)
+
+    integer, intent(in) :: index
+    integer, intent(in), optional :: col_type
+
+    type(physics_buffer_hdr), pointer :: hdr
+    integer :: col_type_use, i, rank
+
+    if (present(col_type)) then
+      col_type_use = col_type
+    else
+      col_type_use = col_type_grid
+    endif
+
+    hdr => hdrbuffertop
+    do i=2,index
+       hdr=>hdr%nexthdr
+    enddo
+
+    rank = 6;
+    do i=6,1,-1
+      if (hdr%dimsizes(i,col_type_use) .ne. 1) then
+        exit
+      endif
+      rank = rank-1
+    enddo
+  end function pbuf_get_field_rank
+
+  function pbuf_get_field_dims (index,col_type) result(dims)
+
+    integer, intent(in) :: index
+    integer, intent(in), optional :: col_type
+
+    type(physics_buffer_hdr), pointer :: hdr
+    integer :: col_type_use, i
+    integer, allocatable :: dims(:)
+
+    if (present(col_type)) then
+      col_type_use = col_type
+    else
+      col_type_use = col_type_grid
+    endif
+
+    hdr => hdrbuffertop
+    do i=2,index
+       hdr=>hdr%nexthdr
+    end do
+
+    allocate(dims(pbuf_get_field_rank(index,col_type)))
+    dims = hdr%dimsizes(:,col_type_use)
+  end function pbuf_get_field_dims
+
   !=========================================================================================
 
   ! 

components/elm/cime_config/config_component.xml

@@ -72,6 +72,7 @@
       <value compset="1850(?:SOI)?_EAM%CMIP6_ELM*_BGC%BCRC">1850_CMIP6bgc_control</value>
       <value compset="1850S_EAM%CMIP6_ELM">1850_SCMIP6_control</value>
       <value compset="20TR.*_EAM%.*CMIP6_ELM">20thC_CMIP6_transient</value>
+      <value compset="20TR.*_EAM%.*CMIP6-CLDERA_ELM">20thC_CMIP6_transient</value>
       <value compset="20TR(?:SOI)?_EAM%CMIP6_ELM*_BGC%B">20thC_CMIP6bgc_transient</value> 
       <value compset="20TR.*_ELM%[^_]*BGC">20thC_bgc_transient</value>
       <value compset="1950(?:SOI)?_EAM%CMIP6_ELM">1950_CMIP6_control</value>