From cddef6e3bd047b9a27cf8ffbf6b038526f019f83 Mon Sep 17 00:00:00 2001 From: Niklas Wagner Date: Fri, 24 Feb 2023 12:56:19 +0100 Subject: [PATCH 1/6] Enable maxpft>1 - set `maxpft=4` as new default in `bldsva/intf_oas3/common_build_interface.ksh` - rewrite workaround in `bldsva/intf_oas3/common_build_interface.ksh` correcting `forc_hgt` according to pft types present in CLM35 grid cell. This is now working for `maxpft!=1` as well. --- .../clm3_5/tsmp/Biogeophysics1Mod.F90 | 68 +++++++++++++++---- bldsva/intf_oas3/common_build_interface.ksh | 5 +- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 index 90900712..731ffa04 100755 --- a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 +++ b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 @@ -183,12 +183,22 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & real(r8) :: psit !negative potential of soil real(r8) :: hr !relative humidity real(r8) :: wx !partial volume of ice and water of surface layer +!NWa +!NOTE: Arrays can start from arbitrary index... +! So in CLM the distributed arraies (over diff. CPUs) does take the index +! from the original array. +! original_array(0:10) --> array1(0:5) and array(6:10) +! The Uper and lower index bound is stored in `lbg`, `ubg`, etc. + real(r8) :: tmp_displacement_max(lbg:ubg) !helper var + real(r8) :: tmp_displacement !helper var +!NWa !------------------------------------------------------------------------------ ! Assign local pointers to derived type members (gridcell-level) forc_hgt_t => clm_a2l%forc_hgt_t forc_pbot => clm_a2l%forc_pbot + forc_q => clm_a2l%forc_q forc_t => clm_a2l%forc_t forc_th => clm_a2l%forc_th @@ -398,31 +408,65 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & end do -!CPS change forc_hgt = forc_hgt + z0m + displa at PFT level, works for 1 pft max -! if (max_pft_per_col .eq. 1) then - if (maxpatch_pft .eq. 1) then +! NWa change forc_hgt = forc_hgt + z0m + displa at PFT level +! forc_hgt is send from COSMO to CLM via fix value. In fully coupled mode, this +! fix value can conflict with the actual canopy height, showing up as CLM error +! with the message: +! > [...] forc height is below canopy height [...] +! To workaround this, forc_hgt is adjusted (increased) according to pft types +! present within each gridcell. +! As multiple pfts could be present in one gridcell, first the displacement is +! calculated for each pft present, and second the maximum displacement is taken +! to correct the forc_hgt. This way we do not run into above error. +! However, keep in mind that this correction is still a workaround. +! NWa Do not forget to allocate below variables +tmp_displacement = 0._r8 ! scalar +tmp_displacement_max = 0._r8 ! same shape as forc_hgt +! +! First calculate max displacement for all columns and pfts on grid lvl + ! max_pft_per_col = im code festgelegte max moeglicher pft (17) do pi = 1,max_pft_per_col + ! num_nolakec: number of column non-lake points in column filter do fc = 1,num_nolakec c = filter_nolakec(fc) l = clandunit(c) g = cgridcell(c) if (pi <= npfts(c)) then p = pfti(c) + pi - 1 - + if (frac_sno(c) > 0._r8 ) then - forc_hgt_u(p) = forc_hgt_u(g) + z0mg(c) + displa(p) - forc_hgt_t(g) = forc_hgt_t(g) + z0mg(c) + displa(p) - forc_hgt_q(g) = forc_hgt_q(g) + z0mg(c) + displa(p) + ! Calculate displacement for related pft on related column + ! If this displacement is higher than displacement_max overwrite + ! displacement_max + tmp_displacement = z0mg(c) + displa(p) + if (tmp_displacement > tmp_displacement_max(g)) then + tmp_displacement_max(g) = tmp_displacement + end if else - forc_hgt_u(g) = forc_hgt_u(g) + z0m(p) + displa(p) - forc_hgt_t(g) = forc_hgt_t(g) + z0m(p) + displa(p) - forc_hgt_q(g) = forc_hgt_q(g) + z0m(p) + displa(p) + ! Calculate displacement for related pft on related column + ! If this displacement is higher than displacement_max overwrite + ! displacement_max + tmp_displacement = z0m(p) + displa(p) + if (tmp_displacement > tmp_displacement_max(g)) then + tmp_displacement_max(g) = tmp_displacement + end if end if end if end do end do - end if -!CPS +! Second increase forc_hgt on grid lvl with max displacement + ! num_nolakec: number of column non-lake points in column filter + do fc = 1,num_nolakec + c = filter_nolakec(fc) + g = cgridcell(c) + + ! Update forc_hgt on grid lvl + forc_hgt_u(g) = forc_hgt_u(g) + tmp_displacement_max(g) + !write(6,*)'DEBUG: forc_hgt_u(g)', forc_hgt_u(g) + forc_hgt_t(g) = forc_hgt_t(g) + tmp_displacement_max(g) + forc_hgt_q(g) = forc_hgt_q(g) + tmp_displacement_max(g) + end do +!! NWR end subroutine Biogeophysics1 diff --git a/bldsva/intf_oas3/common_build_interface.ksh b/bldsva/intf_oas3/common_build_interface.ksh index d849f6bf..00021603 100755 --- a/bldsva/intf_oas3/common_build_interface.ksh +++ b/bldsva/intf_oas3/common_build_interface.ksh @@ -633,7 +633,10 @@ route "${cyellow}>>> c_configure_clm${cnormal}" cp $rootdir/bldsva/intf_oas3/${mList[1]}/oas3/oas_clm_init.F90 $clmdir/src/oas3 spmd="on" # settings are [on | off ] (default is off) - maxpft="1" # settings are 4->17 (default is 4) + # NWa 20230214 + # Test maxpft>1 + maxpft="4" # settings are 4->17 (default is 4) + #maxpft="1" # settings are 4->17 (default is 4) rtm="off" # settings are [on | off ] (default is off) cps_catch="off" # settings are [on | off ] (default is off) usr_src="$clmdir/bld/usr.src " From f79cf5683ac88c218ae5b2a58b29d4a83e300a04 Mon Sep 17 00:00:00 2001 From: Niklas Wagner Date: Mon, 27 Mar 2023 10:51:19 +0200 Subject: [PATCH 2/6] Add a command line option to modify maxpft - Add a command line option to modify maxpft long: --maxpft=X short -f=X - Default behaviour is still maxpft=1 --- bldsva/build_tsmp.ksh | 13 ++++++++----- bldsva/intf_oas3/common_build_interface.ksh | 4 ---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bldsva/build_tsmp.ksh b/bldsva/build_tsmp.ksh index 33384258..cdee26f3 100755 --- a/bldsva/build_tsmp.ksh +++ b/bldsva/build_tsmp.ksh @@ -32,6 +32,7 @@ getDefaults(){ def_mode="0" #0: let flags decide, 1:batch, 2:interactive def_cplscheme="true" def_readCLM="false" + def_maxpft="1" def_freeDrain="false" #compiler optimization @@ -94,6 +95,7 @@ setDefaults(){ freeDrain=$def_freeDrain readCLM=$def_readCLM + maxpft=${def_maxpft} cplscheme=$def_cplscheme mode=$def_mode @@ -563,9 +565,10 @@ printState(){ print "${cred}(25)${cnormal} profiling (default=$def_profiling): ${cgreen}$profiling ${cnormal}" print "${cred}(26)${cnormal} Couple-Scheme (default=$def_cplscheme): ${cgreen}$cplscheme ${cnormal}" print "${cred}(27)${cnormal} readCLM: Consistently read CLM-mask (default=$def_readCLM): ${cgreen}$readCLM ${cnormal}" - print "${cred}(28)${cnormal} Compiles ParFlow with free drainage feature (default=$def_freeDrain): ${cgreen}$freeDrain ${cnormal}" - print "${cred}(29)${cnormal} compiler (default=$defaultcompiler): ${cgreen}$compiler ${cnormal}" - print "${cred}(30)${cnormal} processor (default=$defaultprocessor): ${cgreen}$processor ${cnormal}" + print "${cred}(28)${cnormal} maxpft: Set maxpft per grid cell for CLM (default=$def_maxpft): ${cgreen}$maxpft ${cnormal}" + print "${cred}(29)${cnormal} Compiles ParFlow with free drainage feature (default=$def_freeDrain): ${cgreen}$freeDrain ${cnormal}" + print "${cred}(30)${cnormal} compiler (default=$defaultcompiler): ${cgreen}$compiler ${cnormal}" + print "${cred}(31)${cnormal} processor (default=$defaultprocessor): ${cgreen}$processor ${cnormal}" } check(){ @@ -863,6 +866,7 @@ getGitInfo(){ USAGE+="[c:combination? Combination of component models.]:[combination:='$def_combination']" USAGE+="[C:cplscheme? Couple-Scheme for CLM/COS coupling.]:[cplscheme:='$def_cplscheme']" USAGE+="[r:readclm? Flag to consistently read in CLM mask.]:[readclm:='$def_readCLM']" + USAGE+="[f:maxpft? Flag to control maxpft per grid cell in CLM.]:[maxpft:='$def_maxpft']" USAGE+="[d:freedrain? Compiles ParFlow with free drainage feature.]:[freedrain:='$def_freeDrain']" USAGE+="[W:optoas?Build option for Oasis.]:[optoas:='${def_options["oas"]}']{" @@ -916,6 +920,7 @@ getGitInfo(){ c) combination="$OPTARG" ; args=1 ;; C) cplscheme="$OPTARG" ; args=1 ;; r) readCLM="$OPTARG" ; args=1 ;; + f) maxpft="$OPTARG" ; args=1 ;; d) freeDrain="$OPTARG" ; args=1 ;; #DA T) options+=(["icon"]="$OPTARG") ; args=1 ;; @@ -944,8 +949,6 @@ getGitInfo(){ done - - comment " source list with supported machines and configurations" . $rootdir/bldsva/supported_versions.ksh check diff --git a/bldsva/intf_oas3/common_build_interface.ksh b/bldsva/intf_oas3/common_build_interface.ksh index 00021603..8ae1597d 100755 --- a/bldsva/intf_oas3/common_build_interface.ksh +++ b/bldsva/intf_oas3/common_build_interface.ksh @@ -633,10 +633,6 @@ route "${cyellow}>>> c_configure_clm${cnormal}" cp $rootdir/bldsva/intf_oas3/${mList[1]}/oas3/oas_clm_init.F90 $clmdir/src/oas3 spmd="on" # settings are [on | off ] (default is off) - # NWa 20230214 - # Test maxpft>1 - maxpft="4" # settings are 4->17 (default is 4) - #maxpft="1" # settings are 4->17 (default is 4) rtm="off" # settings are [on | off ] (default is off) cps_catch="off" # settings are [on | off ] (default is off) usr_src="$clmdir/bld/usr.src " From c6afa9ce83ffc50216a69f059ad1b4cbf2177ea4 Mon Sep 17 00:00:00 2001 From: Niklas Wagner Date: Mon, 27 Mar 2023 10:57:16 +0200 Subject: [PATCH 3/6] Remove personal comments - I have marked my changes in the src-code with !NWa to indicate which change was introduced by me. However, this is what git is for, so it is not necessary. As a good practice, I have removed these comments. --- bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 index 731ffa04..734d4501 100755 --- a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 +++ b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 @@ -183,7 +183,6 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & real(r8) :: psit !negative potential of soil real(r8) :: hr !relative humidity real(r8) :: wx !partial volume of ice and water of surface layer -!NWa !NOTE: Arrays can start from arbitrary index... ! So in CLM the distributed arraies (over diff. CPUs) does take the index ! from the original array. @@ -191,7 +190,6 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & ! The Uper and lower index bound is stored in `lbg`, `ubg`, etc. real(r8) :: tmp_displacement_max(lbg:ubg) !helper var real(r8) :: tmp_displacement !helper var -!NWa !------------------------------------------------------------------------------ ! Assign local pointers to derived type members (gridcell-level) @@ -408,7 +406,7 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & end do -! NWa change forc_hgt = forc_hgt + z0m + displa at PFT level +! Change forc_hgt = forc_hgt + z0m + displa at PFT level ! forc_hgt is send from COSMO to CLM via fix value. In fully coupled mode, this ! fix value can conflict with the actual canopy height, showing up as CLM error ! with the message: @@ -419,7 +417,6 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & ! calculated for each pft present, and second the maximum displacement is taken ! to correct the forc_hgt. This way we do not run into above error. ! However, keep in mind that this correction is still a workaround. -! NWa Do not forget to allocate below variables tmp_displacement = 0._r8 ! scalar tmp_displacement_max = 0._r8 ! same shape as forc_hgt ! @@ -462,11 +459,9 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & ! Update forc_hgt on grid lvl forc_hgt_u(g) = forc_hgt_u(g) + tmp_displacement_max(g) - !write(6,*)'DEBUG: forc_hgt_u(g)', forc_hgt_u(g) forc_hgt_t(g) = forc_hgt_t(g) + tmp_displacement_max(g) forc_hgt_q(g) = forc_hgt_q(g) + tmp_displacement_max(g) end do -!! NWR end subroutine Biogeophysics1 From a9e63245e3ac9238db017aec0aef5797bca49076 Mon Sep 17 00:00:00 2001 From: Niklas Wagner <78956063+niklaswr@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:24:12 +0200 Subject: [PATCH 4/6] Update bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 - Adding best practice for FORTRAN array notation. Scalar: A vs. Vector: A(:) Co-authored-by: s-poll --- bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 index 734d4501..ef393364 100755 --- a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 +++ b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 @@ -418,7 +418,7 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & ! to correct the forc_hgt. This way we do not run into above error. ! However, keep in mind that this correction is still a workaround. tmp_displacement = 0._r8 ! scalar -tmp_displacement_max = 0._r8 ! same shape as forc_hgt +tmp_displacement_max(:) = 0._r8 ! same shape as forc_hgt ! ! First calculate max displacement for all columns and pfts on grid lvl ! max_pft_per_col = im code festgelegte max moeglicher pft (17) From 670c6c09878af8541f2faf539a974f50cbc2dd05 Mon Sep 17 00:00:00 2001 From: Niklas Wagner <78956063+niklaswr@users.noreply.github.com> Date: Tue, 28 Mar 2023 07:25:27 +0200 Subject: [PATCH 5/6] Update bldsva/build_tsmp.ksh - add comment indicating maxpft=4 is CLM3.5 default. Co-authored-by: s-poll --- bldsva/build_tsmp.ksh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bldsva/build_tsmp.ksh b/bldsva/build_tsmp.ksh index cdee26f3..fedcf0b7 100755 --- a/bldsva/build_tsmp.ksh +++ b/bldsva/build_tsmp.ksh @@ -32,7 +32,7 @@ getDefaults(){ def_mode="0" #0: let flags decide, 1:batch, 2:interactive def_cplscheme="true" def_readCLM="false" - def_maxpft="1" + def_maxpft="1" # (CLM default is 4) def_freeDrain="false" #compiler optimization From 85359a25ec8312e8d066375b4a3443c5952fb06f Mon Sep 17 00:00:00 2001 From: Niklas Wagner <78956063+niklaswr@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:11:30 +0200 Subject: [PATCH 6/6] Update bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 - Remove unused src-code Co-authored-by: s-poll --- bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 index ef393364..ff60dd0c 100755 --- a/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 +++ b/bldsva/intf_oas3/clm3_5/tsmp/Biogeophysics1Mod.F90 @@ -426,7 +426,6 @@ subroutine Biogeophysics1(lbg, ubg, lbc, ubc, lbp, ubp, & ! num_nolakec: number of column non-lake points in column filter do fc = 1,num_nolakec c = filter_nolakec(fc) - l = clandunit(c) g = cgridcell(c) if (pi <= npfts(c)) then p = pfti(c) + pi - 1