refactor fates dispersal method

This changes the dispersal method from using an mpi sum reduction into
a gather collective to avoid the non-associativity of mpi floating point
reductions.  This will enable exact restarts with different pelayouts.

components/elm/src/main/elmfates_interfaceMod.F90

@@ -598,12 +598,14 @@ subroutine init(this, bounds_proc )
       ! is not turned on
       ! ---------------------------------------------------------------------------------
 
+      use spmdMod,                  only : npes
+      use decompMod,                only : procinfo
       use FatesInterfaceMod,        only : FatesReportParameters
       use FatesParameterDerivedMod, only : param_derived
       use FatesInterfaceTypesMod,   only : numpft_fates => numpft
       use elm_varsur,               only : wt_nat_patch
-      use topounit_varcon           , only: max_topounits, has_topounit
-      use FATESFireFactoryMod       , only: create_fates_fire_data_method
+      use topounit_varcon,          only : max_topounits, has_topounit
+      use FATESFireFactoryMod,      only : create_fates_fire_data_method
 
       implicit none
 
@@ -638,11 +640,11 @@ subroutine init(this, bounds_proc )
       if (fates_dispersal_kernel_mode .ne. fates_dispersal_kernel_none) then
          ! Initialize fates global seed dispersal array for all nodes
          call get_proc_global(ng=numg)
-         call this%fates_seed%init(numg,numpft_fates)
+         call this%fates_seed%init(npes,numg,procinfo%ncells,numpft_fates)
 
          ! Initialize the array of nearest neighbors for fates-driven grid cell communications
          ! This must be called after surfrd_get_data and decompInit_lnd
-         call DetermineGridCellNeighbors(lneighbors,numg)
+         call DetermineGridCellNeighbors(lneighbors,this%fates_seed,numg)
       end if
 
       nclumps = get_proc_clumps()
@@ -1309,7 +1311,7 @@ subroutine wrap_update_hlmfates_dyn(this, nc, bounds_clump,      &
 
           ! Accumulate seeds from sites to the gridcell local outgoing buffer
           if ((fates_dispersal_kernel_mode .ne. fates_dispersal_kernel_none) .and. IsItDispersalTime()) then
-             this%fates_seed%outgoing_local(g,:) = this%fates_seed%outgoing_local(g,:) + this%fates(nc)%sites(s)%seed_out(:)
+             this%fates_seed%outgoing_local(g,:) = this%fates(nc)%sites(s)%seed_out(:)
           end if
 
           veg_pp%is_veg(col_pp%pfti(c):col_pp%pftf(c))        = .false.
@@ -2535,6 +2537,7 @@ subroutine WrapSeedGlobal(this,is_restart_flag)
    ! Call mpi procedure to provide the global seed output distribution array to every gridcell.
    ! This could be conducted with a more sophisticated halo-type structure or distributed graph.
 
+   use decompMod,              only : procinfo
    use spmdMod,                only : MPI_REAL8, MPI_SUM, mpicom
    use FatesDispersalMod,      only : lneighbors, neighbor_type
    use FatesInterfaceTypesMod, only : numpft_fates => numpft
@@ -2570,16 +2573,12 @@ subroutine WrapSeedGlobal(this,is_restart_flag)
       ! Re-initialize incoming seed buffer for this time step
       this%fates_seed%incoming_global(:,:) = 0._r8
 
-      ! Re-initialize the outgoing global seed array buffer
-      this%fates_seed%outgoing_global(:,:) = 1.e6_r8  ! Is this acting as seed rain?
-
       ! Distribute and sum outgoing seed data from all nodes to all nodes
-      ! mpi_allgather should work here as well since gridcells values are not split across nodes
-      ! This would allow for reduction in the outgoing local array size
+      call MPI_Allgatherv(this%fates_seed%outgoing_local, procinfo%ncells*numpft_fates, MPI_REAL8, &
+                          this%fates_seed%outgoing_global, this%fates_seed%ncells_array*numpft_fates, this%fates_seed%begg_array, &
+                          MPI_REAL8, mpicom, ier)
+
       call get_proc_global(ng=numg)
-      call mpi_allreduce(this%fates_seed%outgoing_local, this%fates_seed%outgoing_global, &
-                         numg*numpft_fates, MPI_REAL8, MPI_SUM, mpicom, ier)
-
       do g = 1, numg
 
          ! Calculate the current gridcell incoming seed for each gridcell index