Compile-time ng in ecCKD and TripleClouds + GPTL timing library

Makefile

@@ -44,6 +44,31 @@ ifdef SINGLE_PRECISION
 CPPFLAGS += -DPARKIND1_SINGLE
 endif
 
+# Option to declare number of g-points (inner dimension) at compile time, beneficial when NG is small   
+# make PROFILE=... NG_SW=32 NG_LW=32 (when using 32-term ECCKD models)
+ifdef NG_SW
+CPPFLAGS += -DNG_SW=$(NG_SW)
+endif
+ifdef NG_LW
+CPPFLAGS += -DNG_LW=$(NG_LW)
+endif
+
+# Use the General Purpose Timing Library (GPTL)?
+ifeq ($(GPTL_TIMING),1)
+CPPFLAGS += -DUSE_TIMING
+TIMING_INCLUDE = -I$(TIME_DIR)/include
+LDFLAGS += -L$(TIME_DIR)/lib -Wl,-rpath=$(TIME_DIR)/lib 
+LIBS_TIMING = -lgptlf -lgptl -rdynamic -lunwind # -rdynamic  
+else ifeq ($(GPTL_TIMING),2) 
+# Use GPTL with PAPI to estimate FLOPS and other performance metrics with hardware counters
+# only tested to work with gcc, other compilers probably need different flags
+CPPFLAGS += -DUSE_TIMING  -DUSE_PAPI
+TIMING_INCLUDE = -I$(TIME_DIR)/include 
+LDFLAGS  += -L$(TIME_DIR)/lib -Wl,-rpath=$(TIME_DIR)/lib 
+LIBS_TIMING = -lgptlf -lgptl  -rdynamic -lpapi -lunwind
+# LIBS_TIMING      += -lgptl -lpapi
+endif
+
 # If PRINT_ENTRAPMENT_DATA=1 was given on the "make" command line
 # then the SPARTACUS shortwave solver will write data to fort.101 and
 # fort.102
@@ -64,9 +89,9 @@ endif
 # Consolidate flags
 export FC
 export FCFLAGS = $(WARNFLAGS) $(BASICFLAGS) $(CPPFLAGS) -I../include \
-	$(OPTFLAGS) $(DEBUGFLAGS) $(NETCDF_INCLUDE) $(OMPFLAG)
+	$(OPTFLAGS) $(DEBUGFLAGS) $(NETCDF_INCLUDE) $(TIMING_INCLUDE) $(OMPFLAG)
 export LIBS    = $(LDFLAGS) -L../lib -lradiation -lutilities \
-	-lifsrrtm -lifsaux $(FCLIBS) $(NETCDF_LIB) $(OMPFLAG)
+	-lifsrrtm -lifsaux $(FCLIBS) $(NETCDF_LIB) $(LIBS_TIMING) $(OMPFLAG)
 
 # Do we include Dr Hook from ECMWF's fiat library?
 ifdef FIATDIR

Makefile_include.intel_atos

@@ -38,3 +38,6 @@ BASICFLAGS = -module ../mod -fpe0 -assume byterecl -align array64byte \
 	   -finline-functions -finline-limit=1500 -Winline -assume realloc_lhs
 DEBUGFLAGS = -g
 OMPFLAG = -qopenmp -qopenmp-threadprivate=compat
+
+# location of General Purpose Timing Library (compile with GPTL_TIMING=1 or GPTL_TIMING=2)
+TIME_DIR=/home/papu/gptl-ifort

driver/ecrad_ifs_driver_blocked.F90

@@ -65,7 +65,16 @@ program ecrad_ifs_driver
   use ecrad_driver_read_input,  only : read_input
   use easy_netcdf
   use ifs_blocking
-
+#ifdef USE_TIMING
+#ifndef USE_PAPI
+  ! Timing library
+ use gptl,                  only: gptlstart, gptlstop, gptlinitialize, gptlpr, gptlfinalize, gptlsetoption, &
+                                  gptlpercent, gptloverhead, gptlpr_file
+#else
+#include "f90papi.h"
+#include "gptl.inc"
+#endif    
+#endif  
   implicit none
 
 #include "radiation_scheme.intfb.h"
@@ -137,7 +146,29 @@ program ecrad_ifs_driver
 
 !  integer    :: iband(20), nweights
 !  real(jprb) :: weight(20)
-
+#ifdef USE_TIMING
+  integer :: ret
+  integer values(8)
+  character(len=100) :: timing_file_name, name_gas_model, name_solver
+  character(5) :: prefix, suffix
+  !
+  ! Initialize timers
+  !
+  ret = gptlsetoption (gptlpercent, 1)        ! Turn on "% of" print
+  ret = gptlsetoption (gptloverhead, 0)       ! Turn off overhead estimate
+
+#ifdef USE_PAPI  
+#ifdef PARKIND1_SINGLE
+  ret = GPTLsetoption (PAPI_SP_OPS, 1);
+#else
+  ret = GPTLsetoption (PAPI_DP_OPS, 1);
+#endif
+! ret = GPTLsetoption (GPTL_IPC, 1);
+ret = GPTLsetoption (PAPI_L1_DCM, 1);
+ret = GPTLsetoption (GPTL_L3MRT, 1);
+#endif  
+  ret = gptlinitialize()
+#endif
 
   ! --------------------------------------------------------
   ! Section 2: Configure
@@ -365,6 +396,10 @@ program ecrad_ifs_driver
     write(nulout,'(a)')  'Performing radiative transfer calculations'
   end if
 
+#ifdef USE_TIMING
+    ret =  gptlstart('radiation')
+#endif   
+
   ! Option of repeating calculation multiple time for more accurate
   ! profiling
 #ifndef NO_OPENMP
@@ -444,7 +479,9 @@ program ecrad_ifs_driver
 !    end if
 
   end do
-
+#ifdef USE_TIMING
+  ret =  gptlstop('radiation')
+#endif
 #ifndef NO_OPENMP
   tstop = omp_get_wtime()
   write(nulout, '(a,g12.5,a)') 'Time elapsed in radiative transfer: ', tstop-tstart, ' seconds'
@@ -497,5 +534,15 @@ program ecrad_ifs_driver
   if (driver_config%iverbose >= 2) then
     write(nulout,'(a)') '------------------------------------------------------------------------------------'
   end if
-
+#ifdef USE_TIMING
+  ! End timers
+  call yradiation%rad_config%get_gas_optics_name_sw(name_gas_model)
+  call yradiation%rad_config%get_solver_name_sw(name_solver)
+  call date_and_time(values=values)
+  write(timing_file_name,'(a,a,a,a,a,i0,a,i0,a,i4.4)') 'timing.', trim(name_gas_model), '_', trim(name_solver), &
+    & '_block', driver_config%nblocksize, '_nrep', driver_config%nrepeat, '_', values(8)
+  write(nulout,'(a,a)')  'Writing GPTL timing output to ', trim(timing_file_name)
+  ret = gptlpr_file(trim(timing_file_name))
+  ret = gptlfinalize()
+#endif
 end program ecrad_ifs_driver

radiation/ecrad_config.h

@@ -37,3 +37,15 @@
 ! tasks. The MPI version is not used for writing files.
 
 !#define EASY_NETCDF_READ_MPI 1
+
+! Allow size of inner dimension (number of g-points) to be known at compile time if NG_LW/NG_SW is defined
+#ifdef NG_LW
+  integer, parameter :: ng_lw = NG_LW
+#else
+#define ng_lw ng_lw_in
+#endif
+#ifdef NG_SW
+  integer, parameter :: ng_sw = NG_SW
+#else
+#define ng_sw ng_sw_in
+#endif

radiation/ecrad_definitions.F90

@@ -0,0 +1,22 @@
+module ecrad_definitions
+
+  use parkind1, only : jpim
+
+  implicit none
+
+  ! By making the number of regions a parameter NREGION, the compiler
+  ! can optimize better.  We also provide a preprocessor parameter as
+  ! in one or two cases the operations on 2x2 and 3x3 matrices
+  ! (e.g. matrix exponentials) are completely different algorithms.
+#define NUM_REGIONS 2
+  integer(jpim), parameter :: NREGION = NUM_REGIONS
+
+! Allow size of inner dimension (number of g-points) to be known at compile time if NG_LW is defined
+#ifdef NG_LW
+  integer(jpim), parameter :: ng_lw = NG_LW
+#else
+#define ng_lw ng_lw_in
+#endif
+
+
+end module ecrad_definitions

radiation/radiation_aerosol_optics.F90

@@ -18,12 +18,11 @@
 !   2022-03-27  R. Hogan  Add setup_general_aerosol_optics_legacy to use RRTM aerosol files with ecCKD
 !   2022-11-22  P. Ukkonen / R. Hogan  Optimizations to enhance vectorization
 
-#include "ecrad_config.h"
-
 module radiation_aerosol_optics
 
   implicit none
   public
+#include "ecrad_config.h"
 
 contains
 

radiation/radiation_aerosol_optics_data.F90

@@ -16,8 +16,6 @@
 !   2017-10-23  R. Hogan  Renamed single-character variables
 !   2018-04-20  A. Bozzo  Read optical properties at selected wavelengths
 
-#include "ecrad_config.h"
-
 module radiation_aerosol_optics_data
 
   use parkind1,      only : jprb
@@ -28,6 +26,8 @@ module radiation_aerosol_optics_data
 
   private :: get_line
 
+#include "ecrad_config.h"
+
   ! The following provide possible values for
   ! aerosol_optics_type%iclass, which is used to map the user's type
   ! index to the hydrophobic or hydrophilic types loaded from the

radiation/radiation_aerosol_optics_description.F90

@@ -13,15 +13,15 @@
 ! Email:   r.j.hogan@ecmwf.int
 !
 
-#include "ecrad_config.h"
-
 module radiation_aerosol_optics_description
 
   use parkind1,      only : jprb
 
   implicit none
   public
 
+#include "ecrad_config.h"
+
   !---------------------------------------------------------------------
   ! This type holds the metadata from an aerosol optical property
   ! file, enabling the user to request the index to the aerosol type

radiation/radiation_cloud_cover.F90

@@ -18,14 +18,14 @@
 ! Modifications
 !   2020-10-07  R. Hogan  Ensure iobj1 initialized in case of alpha_obj==0
 
-#include "ecrad_config.h"
-
 module radiation_cloud_cover
 
   use parkind1, only           : jprb
 
   public
 
+#include "ecrad_config.h"
+
   ! Three overlap schemes.  Note that "Exponential" means that
   ! clear-sky regions have no special significance for computing the
   ! cumulative cloud cover: non-contiguous clouds are exponentially

radiation/radiation_cloud_optics_data.F90

@@ -13,7 +13,7 @@
 ! Email:   r.j.hogan@ecmwf.int
 !
 
-#include "ecrad_config.h"
+! #include "ecrad_config.h"
 
 module radiation_cloud_optics_data
 
@@ -22,6 +22,7 @@ module radiation_cloud_optics_data
   implicit none
   public
 
+#include "ecrad_config.h"
   !---------------------------------------------------------------------
   ! This type holds the configuration information to compute
   ! cloud optical properties

radiation/radiation_config.F90

@@ -33,8 +33,6 @@
 ! files in this directory, please inform Robin Hogan.
 !
 
-#include "ecrad_config.h"
-
 module radiation_config
 
   use parkind1,                      only : jprb
@@ -49,6 +47,7 @@ module radiation_config
 
   implicit none
   public
+#include "ecrad_config.h"
 
   ! Configuration codes: use C-style enumerators to avoid having to
   ! remember the numbers
@@ -641,6 +640,8 @@ module radiation_config
      procedure :: set_aerosol_wavelength_mono
      procedure :: consolidate_sw_albedo_intervals
      procedure :: consolidate_lw_emiss_intervals
+     procedure :: get_gas_optics_name_sw
+     procedure :: get_solver_name_sw
 
   end type config_type
 
@@ -2135,4 +2136,16 @@ subroutine print_enum(message_str, enum_str, name, val)
     write(nulout,'(a,a,a,a,i0,a)') str, ' (', name, '=', val,')'
   end subroutine print_enum
 
+  subroutine get_gas_optics_name_sw(this, out_str)
+    class(config_type),   intent(inout) :: this
+    character(len=*),   intent(out) :: out_str
+    write(out_str,'(a)') trim(GasModelName(this%i_gas_model_sw))
+  end subroutine get_gas_optics_name_sw
+
+  subroutine get_solver_name_sw(this, out_str)
+    class(config_type),   intent(inout) :: this
+    character(len=*),   intent(out) :: out_str
+    write(out_str,'(a)') trim(SolverName(this%i_solver_sw))
+  end subroutine get_solver_name_sw
+
 end module radiation_config

radiation/radiation_ecckd.F90

@@ -14,8 +14,6 @@
 ! License: see the COPYING file for details
 !
 
-#include "ecrad_config.h"
-
 module radiation_ecckd
 
   use parkind1, only : jprb
@@ -478,9 +476,15 @@ subroutine calc_optical_depth_ckd_model(this, ncol, nlev, istartcol, iendcol, nm
     ! Output variables
 
     ! Layer absorption optical depth for each g point
+#if defined NG_SW && defined NG_LW && NG_SW==NG_LW 
+    real(jprb),            intent(out) :: optical_depth_fl(NG_SW,nlev,istartcol:iendcol)
+    ! In the shortwave only, the Rayleigh scattering optical depth
+    real(jprb),  optional, intent(out) :: rayleigh_od_fl(NG_SW,nlev,istartcol:iendcol)
+#else 
     real(jprb),            intent(out) :: optical_depth_fl(this%ng,nlev,istartcol:iendcol)
     ! In the shortwave only, the Rayleigh scattering optical depth
     real(jprb),  optional, intent(out) :: rayleigh_od_fl(this%ng,nlev,istartcol:iendcol)
+#endif
 
     ! Local variables
 

radiation/radiation_ecckd_gas.F90

@@ -14,8 +14,6 @@
 ! License: see the COPYING file for details
 !
 
-#include "ecrad_config.h"
-
 module radiation_ecckd_gas
 
   use parkind1, only : jprb
@@ -25,6 +23,8 @@ module radiation_ecckd_gas
 
   public
 
+#include "ecrad_config.h"
+
   ! Concentration dependence of individual gases
   enum, bind(c)
     enumerator :: IConcDependenceNone = 0, &

radiation/radiation_flux.F90

@@ -20,14 +20,13 @@
 !   2021-01-20  R. Hogan  Added heating_rate_out_of_physical_bounds function
 !   2022-12-07  R. Hogan  Added top-of-atmosphere spectral output
 
-#include "ecrad_config.h"
-
 module radiation_flux
 
   use parkind1, only : jprb
 
   implicit none
   public
+#include "ecrad_config.h"
 
   !---------------------------------------------------------------------
   ! This derived type contains the output from the radiation

radiation/radiation_general_cloud_optics_data.F90

@@ -14,8 +14,6 @@
 ! License: see the COPYING file for details
 !
 
-#include "ecrad_config.h"
-
 module radiation_general_cloud_optics_data
 
   use parkind1, only : jprb
@@ -24,6 +22,8 @@ module radiation_general_cloud_optics_data
 
   public
 
+#include "ecrad_config.h"
+
   !---------------------------------------------------------------------
   ! This type holds the configuration information to compute optical
   ! properties for a particular type of cloud or hydrometeor in one of