Merge branch 'wanggangsheng/lnd/alm-pflotran-bgc-rebase3' into next (PR

#1649)

Migrate NGEE-Arctic CLM-PFLOTRAN to ACME Land Model through a
generic clm_interface

1. clm_interface_dataType consists of biogeochemistry (bgcType)
and thermal-hydrology (thType).
2. Rename namelist use_bgc_interface to use_clm_interface to
include bgc & th interfaces.
3. Data passing between ALM & clm_interface is implemented in
clm_interface_funcsMod.F90.
4. Data passing between clm_interface & PFLOTRAN is implemented in
clm_interface_pflotranMod.F90.
5. Global-scale ALM-PFLOTRAN-bgc coupled run is tested.
6. The clm_interface is extended to incorporate PLFOTRAN
thermal-hydrology in next step.
7. Add configuration for mach "cades" for alm-pflotran coupled run.
8. makefile: update "PETSC_LIB"-"variables" dir for
petsc-git-version 1a9d3c3.

[BFB]

Conflicts:
	components/clm/src/biogeochem/CNNStateUpdate1Mod.F90
	components/clm/src/biogeochem/CNNitrogenFluxType.F90
	components/clm/src/biogeochem/PStateUpdate1Mod.F90
	components/clm/src/biogeochem/PStateUpdate2Mod.F90
	components/clm/src/biogeochem/PStateUpdate3Mod.F90
	components/clm/src/biogeophys/HydrologyNoDrainageMod.F90
	components/clm/src/main/clm_initializeMod.F90
	components/clm/src/main/controlMod.F90

cime/config/acme/machines/Makefile

@@ -251,8 +251,9 @@ ifeq ($(strip $(USE_PETSC)), TRUE)
     $(error PETSC_PATH must be defined when USE_PETSC is TRUE)
   endif
 
-  # Get the "PETSC_LIB" list an env var
-  include $(PETSC_PATH)/conf/variables
+  # Get the "PETSC_LIB" list an env var 
+  include $(PETSC_PATH)/lib/petsc/conf/variables
+
 endif
 
 # Set Trilinos info if it is being used
@@ -542,7 +543,7 @@ endif
 
 # Add PETSc libraries
 ifeq ($(strip $(USE_PETSC)), TRUE)
-  SLIBS += -L${LIB_PETSC} ${PETSC_LIB}
+  SLIBS += ${PETSC_LIB}
 endif
 
 # Add trilinos libraries; too be safe, we include all libraries included in the trilinos build,
@@ -877,7 +878,7 @@ cleanrof:
 	$(RM) -fr $(EXEROOT)/rof/obj
 
 cleanlnd:
-	$(RM) -f $(LIBROOT)/liblnd.a
+	$(RM) -f $(LIBROOT)/$(LNDLIB)
 	$(RM) -fr $(EXEROOT)/lnd/obj
 
 cleancsmshare:

cime/config/acme/machines/config_compilers.xml

@@ -82,6 +82,18 @@ for mct, etc.
   <ADD_CPPDEFS MODEL="pop"> -D_USE_FLOW_CONTROL </ADD_CPPDEFS>
 </compiler>
 
+
+<!-- hacking of mach/compiler generated 'Macros.make' for coupling with pflotran -->
+<!-- ideally it should go with CLM configuration -->
+<compiler>
+  <ADD_FFLAGS  MODEL="clm"    CLM_PFLOTRAN_COUPLED="TRUE"> -I$(CLM_PFLOTRAN_SOURCE_DIR) </ADD_FFLAGS>
+  <ADD_CPPDEFS MODEL="clm"    CLM_PFLOTRAN_COUPLED="TRUE"> -DCLM_PFLOTRAN </ADD_CPPDEFS>
+  <ADD_CPPDEFS MODEL="clm"    CLM_PFLOTRAN_COUPLED="TRUE" CLM_PFLOTRAN_COLMODE="TRUE"> -DCOLUMN_MODE </ADD_CPPDEFS>
+  <ADD_LDFLAGS MODEL="driver" CLM_PFLOTRAN_COUPLED="TRUE"> -L$(CLM_PFLOTRAN_SOURCE_DIR) -lpflotran $(PETSC_LIB) </ADD_LDFLAGS>
+</compiler>
+<!-- end of hacking 'Macros.make' for coupling with pflotran -->
+
+
 <compiler COMPILER="ibm">
   <!-- http://publib.boulder.ibm.com/infocenter/comphelp/v7v91/index.jsp
  Notes:  (see xlf user's guide for the details)

cime/config/acme/machines/config_machines.xml

@@ -1894,6 +1894,16 @@
           <command name="load">netcdf-hdf5parallel/4.3.3.1</command>
         </modules>
       </module_system>
+
+      <!-- for CLM-PFLOTRAN coupling, the PETSC_PATH must be defined specifically upon machines -->
+      <environment_variables>
+        <env name="PETSC_PATH" compiler="gnu" mpilib="openmpi">/software/user_tools/current/cades-ccsi/petsc4pf/openmpi-1.10-gcc-5.3</env>      
+       <!-- hack for PFLOTRAN coupling (this is a temporary solution, and user must manually edit it after case.setup)-->
+       <env name="CLM_PFLOTRAN_COUPLED">FALSE</env>
+       <env name="CLM_PFLOTRAN_COLMODE">FALSE</env>
+       <!-- dir for pflotran source code -->
+       <env name="CLM_PFLOTRAN_SOURCE_DIR">/lustre/or-hydra/cades-ccsi/$USER/models/pflotran-interface/src/clm-pflotran</env>
+      </environment_variables>       
 </machine>
 
 <machine MACH="titan">

cime/config/acme/testmods_dirs/allactive/clm/bgcinterface/user_nl_clm

@@ -0,0 +1,2 @@
+use_clm_interface = .true.
+use_clm_bgc = .true.

cime/scripts/lib/update_acme_tests.py

@@ -73,6 +73,7 @@
                              ("SMS_Ld4.f45_f45.ICLM45ED","clm-fates"),
                              ("ERS.f19_g16.I1850CLM45","clm-betr"),
                              ("ERS.f19_g16.I1850CLM45","clm-vst"),
+                             ("ERS.f09_g16.I1850CLM45CN","clm-bgcinterface"),
                               "ERS.ne11_oQU240.I20TRCLM45",
                               "ERS.f09_g16.IMCLM45BC")
                              ),

components/clm/bld/namelist_files/namelist_definition_clm4_5.xml

@@ -1599,7 +1599,7 @@ Specifies the method for decomposing CLM grids across processors.
 <!-- Namelist options related to the bgc & pflotran interface
 <!-- ========================================================================================  -->
 
-<entry id="use_bgc_interface" type="logical" category="default_settings"
+<entry id="use_clm_interface" type="logical" category="default_settings"
     group="clm_inparm" valid_values="">
     Runtime flag to turn on/off bgc interface.
 </entry>

components/clm/src/biogeochem/CNAllocationBetrMod.F90

@@ -30,7 +30,6 @@ module CNAllocationBeTRMod
   use ColumnType          , only : col_pp                
   use VegetationType      , only : veg_pp                
   ! bgc interface & pflotran module switches
-  use clm_varctl          , only: use_bgc_interface,use_clm_bgc, use_pflotran, pf_cmode
   use clm_varctl          , only : nu_com
   use SoilStatetype       , only : soilstate_type
   use WaterStateType      , only : waterstate_type

components/clm/src/biogeochem/CNBalanceCheckMod.F90

@@ -132,14 +132,14 @@ subroutine BeginPBalance(bounds, num_soilc, filter_soilc, &
     integer :: fc   ! lake filter indices
     !-----------------------------------------------------------------------
 
-    associate(                                         & 
+    associate(                                           &
          totcolp   => phosphorusstate_vars%totcolp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
          !X.YANG - checking P balance problem, starting from VEGP 
          totpftp   => phosphorusstate_vars%totpftp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
          totsomp   => phosphorusstate_vars%totsomp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         cwdp   => phosphorusstate_vars%cwdp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
+         cwdp   => phosphorusstate_vars%cwdp_col       , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg
          totlitp   => phosphorusstate_vars%totlitp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         sminp   => phosphorusstate_vars%sminp_col , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
+         sminp   => phosphorusstate_vars%sminp_col     , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg
 
          col_begpb => phosphorusstate_vars%begpb_col     & ! Output: [real(r8) (:)]  phosphorus mass, beginning of time step (gP/m**2)
          )
@@ -186,7 +186,7 @@ subroutine CBalanceCheck(bounds, &
 
          gpp                     =>    carbonflux_vars%gpp_col                 , & ! Input:  [real(r8) (:) ]  (gC/m2/s) gross primary production      
          er                      =>    carbonflux_vars%er_col                  , & ! Input:  [real(r8) (:) ]  (gC/m2/s) total ecosystem respiration, autotrophic + heterotrophic
-         col_fire_closs          =>    carbonflux_vars%fire_closs_col      , & ! Input:  [real(r8) (:) ]  (gC/m2/s) total column-level fire C loss
+         col_fire_closs          =>    carbonflux_vars%fire_closs_col          , & ! Input:  [real(r8) (:) ]  (gC/m2/s) total column-level fire C loss
          col_hrv_xsmrpool_to_atm =>    carbonflux_vars%hrv_xsmrpool_to_atm_col , & ! Input:  [real(r8) (:) ]  (gC/m2/s) excess MR pool harvest mortality 
          dwt_closs               =>    carbonflux_vars%dwt_closs_col           , & ! Input:  [real(r8) (:) ]  (gC/m2/s) total carbon loss from product pools and conversion
          product_closs           =>    carbonflux_vars%product_closs_col       , & ! Input:  [real(r8) (:) ]  (gC/m2/s) total wood product carbon loss
@@ -254,6 +254,11 @@ subroutine CBalanceCheck(bounds, &
             write(iulog,*)'begcb       = ',col_begcb(c)
             write(iulog,*)'endcb       = ',col_endcb(c),carbonstate_vars%totsomc_col(c)
             write(iulog,*)'delta store = ',col_endcb(c)-col_begcb(c)
+
+            if (use_pflotran .and. pf_cmode) then
+               write(iulog,*)'pf_delta_decompc      = ',col_decompc_delta(c)*dt
+            end if
+
             call endrun(msg=errMsg(__FILE__, __LINE__))
          end if
       end if !use_ed
@@ -286,7 +291,7 @@ subroutine NBalanceCheck(bounds, &
     real(r8):: dt             ! radiation time step (seconds)
     !-----------------------------------------------------------------------
 
-    associate(                                                             & 
+    associate(                                                                 &
          totcoln             =>    nitrogenstate_vars%totcoln_col            , & ! Input:  [real(r8) (:)]  (gN/m2) total column nitrogen, incl veg 
          ndep_to_sminn       =>    nitrogenflux_vars%ndep_to_sminn_col       , & ! Input:  [real(r8) (:)]  atmospheric N deposition to soil mineral N (gN/m2/s)
          nfix_to_sminn       =>    nitrogenflux_vars%nfix_to_sminn_col       , & ! Input:  [real(r8) (:)]  symbiotic/asymbiotic N fixation to soil mineral N (gN/m2/s)
@@ -298,16 +303,15 @@ subroutine NBalanceCheck(bounds, &
          smin_no3_leached    =>    nitrogenflux_vars%smin_no3_leached_col    , & ! Input:  [real(r8) (:)]  soil mineral NO3 pool loss to leaching (gN/m2/s)
          smin_no3_runoff     =>    nitrogenflux_vars%smin_no3_runoff_col     , & ! Input:  [real(r8) (:)]  soil mineral NO3 pool loss to runoff (gN/m2/s)
          f_n2o_nit           =>    nitrogenflux_vars%f_n2o_nit_col           , & ! Input:  [real(r8) (:)]  flux of N2o from nitrification [gN/m^2/s]
-         col_fire_nloss      =>    nitrogenflux_vars%fire_nloss_col      , & ! Input:  [real(r8) (:)]  total column-level fire N loss (gN/m2/s)
+         col_fire_nloss      =>    nitrogenflux_vars%fire_nloss_col          , & ! Input:  [real(r8) (:)]  total column-level fire N loss (gN/m2/s)
          dwt_nloss           =>    nitrogenflux_vars%dwt_nloss_col           , & ! Input:  [real(r8) (:)]  (gN/m2/s) total nitrogen loss from product pools and conversion
          product_nloss       =>    nitrogenflux_vars%product_nloss_col       , & ! Input:  [real(r8) (:)]  (gN/m2/s) total wood product nitrogen loss
          som_n_leached       =>    nitrogenflux_vars%som_n_leached_col       , & ! Input:  [real(r8) (:)]  total SOM N loss from vertical transport
          ! pflotran:
-         col_decompn_delta   =>    nitrogenflux_vars%externaln_to_decomp_delta_col, & ! Input: [real(r8) (:) ] (gN/m2/s) summarized net change of whole column N i/o to decomposing pool bwtn time-step
-         col_no3_delta       =>    nitrogenflux_vars%no3_net_transport_delta_col  , & ! Input: [real(r8) (:) ] (gN/m2/s) summarized net change of whole column NO3 leaching bwtn time-step
+         col_decompn_delta   =>    nitrogenflux_vars%externaln_to_decomp_delta_col  , & ! Input: [real(r8) (:) ] (gN/m2/s) summarized net change of whole column N i/o to decomposing pool bwtn time-step
 
-         col_ninputs         =>    nitrogenflux_vars%ninputs_col         , & ! Output: [real(r8) (:)]  column-level N inputs (gN/m2/s)         
-         col_noutputs        =>    nitrogenflux_vars%noutputs_col        , & ! Output: [real(r8) (:)]  column-level N outputs (gN/m2/s)        
+         col_ninputs         =>    nitrogenflux_vars%ninputs_col             , & ! Output: [real(r8) (:)]  column-level N inputs (gN/m2/s)
+         col_noutputs        =>    nitrogenflux_vars%noutputs_col            , & ! Output: [real(r8) (:)]  column-level N outputs (gN/m2/s)
          col_begnb           =>    nitrogenstate_vars%begnb_col              , & ! Output: [real(r8) (:)]  nitrogen mass, beginning of time step (gN/m**2)
          col_endnb           =>    nitrogenstate_vars%endnb_col              , & ! Output: [real(r8) (:)]  nitrogen mass, end of time step (gN/m**2)
          col_errnb           =>    nitrogenstate_vars%errnb_col                & ! Output: [real(r8) (:)]  nitrogen balance error for the timestep (gN/m**2)
@@ -342,7 +346,15 @@ subroutine NBalanceCheck(bounds, &
            else
             col_noutputs(c) = col_noutputs(c) + f_n2o_nit(c)
 
-            col_noutputs(c) = col_noutputs(c) + smin_no3_leached(c) + smin_no3_runoff(c)
+            if(use_pflotran .and. pf_cmode) then
+               ! inclusion of aq. NH4 transport by PFLOTRAN-bgc
+               col_noutputs(c) = col_noutputs(c) + sminn_leached(c)
+            else
+
+               col_noutputs(c) = col_noutputs(c) + smin_no3_leached(c) + smin_no3_runoff(c)
+
+            endif
+
            end if
          endif
 
@@ -358,10 +370,6 @@ subroutine NBalanceCheck(bounds, &
          if (use_pflotran .and. pf_cmode) then
             col_errnb(c) = col_errnb(c) - col_decompn_delta(c)*dt
             ! here is '-' adjustment. It says that the adding to PF decomp n pools was less.
-
-            ! if not hydrology-coupled, NO3 leaching/runoff at previous time-step used as 'source' (out, -) to PFLOTRAN bgc
-            if (.not.pf_hmode) col_errnb(c) = col_errnb(c) + col_no3_delta(c)*dt
-            ! here is '+' adjustment. It says that the taking to PF no3 pools was more.
          end if
 
          if (abs(col_errnb(c)) > 1e-8_r8) then
@@ -394,7 +402,14 @@ subroutine NBalanceCheck(bounds, &
          write(iulog,*)'fire                  = ',col_fire_nloss(c)*dt
          write(iulog,*)'dwt                   = ',dwt_nloss(c)*dt 
          write(iulog,*)'prod                  = ',product_nloss(c)*dt
+
+         if (use_pflotran .and. pf_cmode) then
+            write(iulog,*)'pf_delta_decompn      = ',col_decompn_delta(c)*dt
+         end if
+
          call endrun(msg=errMsg(__FILE__, __LINE__))
+
+
       end if
 
     end associate
@@ -426,37 +441,37 @@ subroutine PBalanceCheck(bounds, &
 
     real(r8) :: leafp_to_litter_col(bounds%begc:bounds%endc) 
     real(r8) :: frootp_to_litter_col(bounds%begc:bounds%endc) 
-   real(r8):: flux_mineralization_col(bounds%begc:bounds%endc)   !!  local temperary variable
+    real(r8):: flux_mineralization_col(bounds%begc:bounds%endc)   !  local temperary variable
     !-----------------------------------------------------------------------
 
-    associate(                                                             & 
+    associate(                                                                   &
          totcolp             =>    phosphorusstate_vars%totcolp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
          supplement_to_sminp =>    phosphorusflux_vars%supplement_to_sminp_col , & ! Input:  [real(r8) (:)]  supplemental P supply (gP/m2/s)         
          sminp_leached       =>    phosphorusflux_vars%sminp_leached_col       , & ! Input:  [real(r8) (:)]  soil mineral P pool loss to leaching (gP/m2/s)
-         col_fire_ploss      =>    phosphorusflux_vars%fire_ploss_col      , & ! Input:  [real(r8) (:)]  total column-level fire P loss (gP/m2/s)
+         col_fire_ploss      =>    phosphorusflux_vars%fire_ploss_col          , & ! Input:  [real(r8) (:)]  total column-level fire P loss (gP/m2/s)
          dwt_ploss           =>    phosphorusflux_vars%dwt_ploss_col           , & ! Input:  [real(r8) (:)]  (gP/m2/s) total phosphorus loss from product pools and conversion
          product_ploss       =>    phosphorusflux_vars%product_ploss_col       , & ! Input:  [real(r8) (:)]  (gP/m2/s) total wood product phosphorus loss
-         primp_to_labilep=>    phosphorusflux_vars%primp_to_labilep_col    , &
-         secondp_to_occlp=>    phosphorusflux_vars%secondp_to_occlp_col    , &
-         fert_p_to_sminp =>    phosphorusflux_vars%fert_p_to_sminp_col     , &
+         primp_to_labilep    =>    phosphorusflux_vars%primp_to_labilep_col    , &
+         secondp_to_occlp    =>    phosphorusflux_vars%secondp_to_occlp_col    , &
+         fert_p_to_sminp     =>    phosphorusflux_vars%fert_p_to_sminp_col     , &
 
-         col_pinputs         =>    phosphorusflux_vars%pinputs_col         , & ! Output: [real(r8) (:)]  column-level P inputs (gP/m2/s)         
-         col_poutputs        =>    phosphorusflux_vars%poutputs_col        , & ! Output: [real(r8) (:)]  column-level P outputs (gP/m2/s)        
+         col_pinputs         =>    phosphorusflux_vars%pinputs_col             , & ! Output: [real(r8) (:)]  column-level P inputs (gP/m2/s)
+         col_poutputs        =>    phosphorusflux_vars%poutputs_col            , & ! Output: [real(r8) (:)]  column-level P outputs (gP/m2/s)
          col_begpb           =>    phosphorusstate_vars%begpb_col              , & ! Output: [real(r8) (:)]  phosphorus mass, beginning of time step (gP/m**2)
          col_endpb           =>    phosphorusstate_vars%endpb_col              , & ! Output: [real(r8) (:)]  phosphorus mass, end of time step (gP/m**2)
-         col_errpb           =>    phosphorusstate_vars%errpb_col              ,  & ! Output: [real(r8) (:)]  phosphorus balance error for the timestep (gP/m**2)
+         col_errpb           =>    phosphorusstate_vars%errpb_col              , & ! Output: [real(r8) (:)]  phosphorus balance error for the timestep (gP/m**2)
 
          !X.YANG testing P Balance, from VEGP
          totpftp             =>    phosphorusstate_vars%totpftp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
          totsomp             =>    phosphorusstate_vars%totsomp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         cwdp             =>    phosphorusstate_vars%cwdp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         totlitp             =>    phosphorusstate_vars%totlitp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         sminp             =>    phosphorusstate_vars%sminp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg 
-         leafp_to_litter       =>    phosphorusflux_vars%leafp_to_litter_patch       , & ! Input:  [real(r8) (:)]  soil mineral P pool loss to leaching (gP/m2/s)
-         frootp_to_litter       =>   phosphorusflux_vars%frootp_to_litter_patch      , & ! Input:  [real(r8) (:)]  soil mineral P pool loss to leaching (gP/m2/s)
-         sminp_to_plant         =>   phosphorusflux_vars%sminp_to_plant_col          ,&
-         cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool ,     &
-         pf => phosphorusflux_vars                                               &
+         cwdp                =>    phosphorusstate_vars%cwdp_col               , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg
+         totlitp             =>    phosphorusstate_vars%totlitp_col            , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg
+         sminp                 =>  phosphorusstate_vars%sminp_col              , & ! Input:  [real(r8) (:)]  (gP/m2) total column phosphorus, incl veg
+         leafp_to_litter       =>  phosphorusflux_vars%leafp_to_litter_patch   , & ! Input:  [real(r8) (:)]  soil mineral P pool loss to leaching (gP/m2/s)
+         frootp_to_litter      =>  phosphorusflux_vars%frootp_to_litter_patch  , & ! Input:  [real(r8) (:)]  soil mineral P pool loss to leaching (gP/m2/s)
+         sminp_to_plant        =>  phosphorusflux_vars%sminp_to_plant_col      , &
+         cascade_receiver_pool =>  decomp_cascade_con%cascade_receiver_pool    , &
+         pf                    =>  phosphorusflux_vars                           &
          )
 
       ! set time steps
@@ -473,18 +488,18 @@ subroutine PBalanceCheck(bounds, &
 
       !! immobilization/mineralization in litter-to-SOM and SOM-to-SOM fluxes
       !! - X.YANG
-         ! column loop
-         do fc = 1,num_soilc
-            c = filter_soilc(fc)
-            flux_mineralization_col(c) = 0._r8
-         enddo
+        ! column loop
+      do fc = 1,num_soilc
+         c = filter_soilc(fc)
+         flux_mineralization_col(c) = 0._r8
+      enddo
 
       do k = 1, ndecomp_cascade_transitions
          if ( cascade_receiver_pool(k) /= 0 ) then  ! skip terminal transitions
                ! column loop
                do fc = 1,num_soilc
                   c = filter_soilc(fc)
-                    flux_mineralization_col(c) = flux_mineralization_col(c) - &
+                  flux_mineralization_col(c) = flux_mineralization_col(c) - &
                                                pf%decomp_cascade_sminp_flux_col(c,k)*dt
                end do
          else
@@ -498,6 +513,12 @@ subroutine PBalanceCheck(bounds, &
          endif
       end do
 
+      ! column loop
+      do fc = 1,num_soilc
+         c = filter_soilc(fc)
+         flux_mineralization_col(c) = flux_mineralization_col(c) + &
+                                       pf%biochem_pmin_col(c)
+      end do
 
 
       ! column loop