From f78aa42b1b9e6f6c019fb7549e5e49771957bc52 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 3 Dec 2019 22:25:28 +0000 Subject: [PATCH 01/20] Reset to zero coupling arrays for accumulated snow, large scale rain, and convective rain at the end of each coupling step if coupling with chemistry model. --- atmos_model.F90 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/atmos_model.F90 b/atmos_model.F90 index 23e30e76c..bb94bc163 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -1457,6 +1457,24 @@ subroutine update_atmos_chemistry(state, rc) enddo enddo + ! -- zero out accumulated fields +!$OMP parallel do default (none) & +!$OMP shared (nj, ni, Atm_block, IPD_Control, IPD_Data) & +!$OMP private (j, jb, i, ib, nb, ix) + do j = 1, nj + jb = j + Atm_block%jsc - 1 + do i = 1, ni + ib = i + Atm_block%isc - 1 + nb = Atm_block%blkno(ib,jb) + ix = Atm_block%ixp(ib,jb) + IPD_Data(nb)%coupling%rainc_cpl(ix) = zero + if (.not.IPD_Control%cplflx) then + IPD_Data(nb)%coupling%rain_cpl(ix) = zero + IPD_Data(nb)%coupling%snow_cpl(ix) = zero + end if + enddo + enddo + if (IPD_Control%debug) then ! -- diagnostics write(6,'("update_atmos: prsi - min/max/avg",3g16.6)') minval(prsi), maxval(prsi), sum(prsi)/size(prsi) From 75654acb927799a4caad875d0d900e2fdedf336a Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 3 Dec 2019 22:48:19 +0000 Subject: [PATCH 02/20] Properly set kind type of literal constants defining zero and one. --- atmos_model.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atmos_model.F90 b/atmos_model.F90 index bb94bc163..0883d3da9 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -221,7 +221,8 @@ module atmos_model_mod logical,parameter :: flip_vc = .true. #endif - real(kind=IPD_kind_phys), parameter :: zero=0.0, one=1.0 + real(kind=IPD_kind_phys), parameter :: zero = 0.0_IPD_kind_phys, & + one = 1.0_IPD_kind_phys contains From 2870f5e2edfe211a4f926834613aa7115b50da29 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 3 Dec 2019 23:19:57 +0000 Subject: [PATCH 03/20] Initialize to zero canopy resistance output variable in noah/osu land-surface model subdriver. --- gfsphysics/physics/sflx.f | 1 + 1 file changed, 1 insertion(+) diff --git a/gfsphysics/physics/sflx.f b/gfsphysics/physics/sflx.f index 29467fe37..f84006be9 100644 --- a/gfsphysics/physics/sflx.f +++ b/gfsphysics/physics/sflx.f @@ -251,6 +251,7 @@ subroutine sflx & runoff2 = 0.0 runoff3 = 0.0 snomlt = 0.0 + rc = 0.0 ! --- ... define local variable ice to achieve: ! sea-ice case, ice = 1 From 37fb245a137cff7519a5b1c46da5114c1c365c7f Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 3 Dec 2019 23:25:45 +0000 Subject: [PATCH 04/20] Re-implement radiation diagnostic output involving spectral band layer cloud optical depths (0.55 and 10 mu channels) to prevent floating invalid errors due to uninitialized optical depth arrays. --- gfsphysics/GFS_layer/GFS_radiation_driver.F90 | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/gfsphysics/GFS_layer/GFS_radiation_driver.F90 b/gfsphysics/GFS_layer/GFS_radiation_driver.F90 index c7323d6bb..4a3f080f9 100644 --- a/gfsphysics/GFS_layer/GFS_radiation_driver.F90 +++ b/gfsphysics/GFS_layer/GFS_radiation_driver.F90 @@ -2116,18 +2116,41 @@ subroutine GFS_radiation_driver & Diag%fluxr(i,11-j) = Diag%fluxr(i,11-j) + tem0d * Statein%prsi(i,itop+kt) Diag%fluxr(i,14-j) = Diag%fluxr(i,14-j) + tem0d * Statein%prsi(i,ibtc+kb) Diag%fluxr(i,17-j) = Diag%fluxr(i,17-j) + tem0d * Statein%tgrs(i,itop) + enddo + enddo ! Anning adds optical depth and emissivity output - tem1 = 0. - tem2 = 0. - do k=ibtc,itop - tem1 = tem1 + cldtausw(i,k) ! approx .55 mu channel - tem2 = tem2 + cldtaulw(i,k) ! approx 10. mu channel + + if (Model%lsswr .and. (nday > 0)) then + do j = 1, 3 + do i = 1, IM + tem0d = raddt * cldsa(i,j) + itop = mtopa(i,j) - kd + ibtc = mbota(i,j) - kd + tem1 = 0. + do k=ibtc,itop + tem1 = tem1 + cldtausw(i,k) ! approx .55 um channel + enddo + Diag%fluxr(i,43-j) = Diag%fluxr(i,43-j) + tem0d * tem1 enddo - Diag%fluxr(i,43-j) = Diag%fluxr(i,43-j) + tem0d * tem1 - Diag%fluxr(i,46-j) = Diag%fluxr(i,46-j) + tem0d * (1.0-exp(-tem2)) enddo - enddo + endif + + if (Model%lslwr) then + do j = 1, 3 + do i = 1, IM + tem0d = raddt * cldsa(i,j) + itop = mtopa(i,j) - kd + ibtc = mbota(i,j) - kd + tem2 = 0. + do k=ibtc,itop + tem2 = tem2 + cldtaulw(i,k) ! approx 10. um channel + enddo + Diag%fluxr(i,46-j) = Diag%fluxr(i,46-j) + tem0d * (1.0-exp(-tem2)) + enddo + enddo + endif + endif endif ! end_if_lssav From 1c0a9bae0b72a4a2675c5559b572cb1d02274464 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 3 Dec 2019 23:52:13 +0000 Subject: [PATCH 05/20] Reset to zero instantaneous total moisture tendency coupling array at the beginning of each coupling step if coupled with chemistry model. --- gfsphysics/GFS_layer/GFS_physics_driver.F90 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gfsphysics/GFS_layer/GFS_physics_driver.F90 b/gfsphysics/GFS_layer/GFS_physics_driver.F90 index 5b67f7faa..19c40bf24 100644 --- a/gfsphysics/GFS_layer/GFS_physics_driver.F90 +++ b/gfsphysics/GFS_layer/GFS_physics_driver.F90 @@ -3184,9 +3184,13 @@ subroutine GFS_physics_driver & dtdt(1:im,:) = Stateout%gt0(1:im,:) endif ! end if_ldiag3d/cnvgwd - if (Model%ldiag3d) then + if (Model%ldiag3d .or. Model%cplchm) then dqdt(1:im,:,1) = Stateout%gq0(1:im,:,1) - endif ! end if_ldiag3d + endif ! end if_ldiag3d/cplchm + + if (Model%cplchm) then + Coupling%dqdti(1:im,:) = zero + endif ! end if_cplchm #ifdef GFS_HYDRO call get_phi(im, ix, levs, ntrac, Stateout%gt0, Stateout%gq0, & From 735eb9eb9cf0a2dd7a39d890ef3c61259e5ccdcf Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Mon, 19 Oct 2020 17:38:30 -0500 Subject: [PATCH 06/20] Temporarily disable filling export fields during the NUOPC Realize phase since it breaks coupling with aerosol component. --- fv3_cap.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 29b7c361a..b09bec3da 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -954,7 +954,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return end if !jw - call fillExportFields(exportData) +! call fillExportFields(exportData) endif end subroutine InitializeRealize From 59cf366bc2572f36066252404be8bd3794c3aaec Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Mon, 19 Oct 2020 17:52:38 -0500 Subject: [PATCH 07/20] Moving previous commit to a branch. This reverts commit 735eb9eb9cf0a2dd7a39d890ef3c61259e5ccdcf. --- fv3_cap.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index b09bec3da..29b7c361a 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -954,7 +954,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return end if !jw -! call fillExportFields(exportData) + call fillExportFields(exportData) endif end subroutine InitializeRealize From d35c2e0db7b48d91b6cfccb117c73b0f6d2f36fc Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Mon, 7 Feb 2022 17:51:14 +0000 Subject: [PATCH 08/20] add p8b suite files --- ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml | 96 +++++++++++++++++++ ccpp/suites/suite_FV3_GFS_v17_p8b.xml | 95 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml create mode 100644 ccpp/suites/suite_FV3_GFS_v17_p8b.xml diff --git a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml new file mode 100644 index 000000000..be36410d4 --- /dev/null +++ b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml @@ -0,0 +1,96 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_cice + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + unified_ugwp + unified_ugwp_post + GFS_GWD_generic_post + GFS_suite_stateout_update + ozphys_2015 + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + phys_tend + + + + diff --git a/ccpp/suites/suite_FV3_GFS_v17_p8b.xml b/ccpp/suites/suite_FV3_GFS_v17_p8b.xml new file mode 100644 index 000000000..dd8480334 --- /dev/null +++ b/ccpp/suites/suite_FV3_GFS_v17_p8b.xml @@ -0,0 +1,95 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + GFS_radiation_surface + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + unified_ugwp + unified_ugwp_post + GFS_GWD_generic_post + GFS_suite_stateout_update + ozphys_2015 + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + phys_tend + + + + From 76db8e688dd82d0e799955d348d2fe172105490f Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Mon, 7 Feb 2022 23:53:53 +0000 Subject: [PATCH 09/20] fix suite name --- ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml index be36410d4..dfe858bbc 100644 --- a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml +++ b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml @@ -1,6 +1,6 @@ - + From fa996dffba36c18affe42c8d17c20296992ffc69 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Fri, 11 Feb 2022 18:11:23 +0000 Subject: [PATCH 10/20] Enable Thompson microphysics when coupling with chemistry. --- ccpp/data/GFS_typedefs.F90 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index 7b73170ff..d11ac2353 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -7589,9 +7589,7 @@ subroutine interstitial_setup_tracers(Interstitial, Model) endif if (Model%cplchm) then - ! Only Zhao/Carr/Sundqvist and GFDL microphysics schemes are supported - ! when coupling with chemistry. PBL diffusion of aerosols is only supported - ! for GFDL microphysics and MG microphysics. + ! Only the following microphysics schemes are supported with coupled chemistry if (Model%imp_physics == Model%imp_physics_zhao_carr) then Interstitial%nvdiff = 3 elseif (Model%imp_physics == Model%imp_physics_mg) then @@ -7602,8 +7600,14 @@ subroutine interstitial_setup_tracers(Interstitial, Model) endif elseif (Model%imp_physics == Model%imp_physics_gfdl) then Interstitial%nvdiff = 7 + elseif (Model%imp_physics == Model%imp_physics_thompson) then + if (Model%ltaerosol) then + Interstitial%nvdiff = 12 + else + Interstitial%nvdiff = 9 + endif else - write(0,*) "Only Zhao/Carr/Sundqvist and GFDL microphysics schemes are supported when coupling with chemistry" + write(0,*) "Selected microphysics scheme is not supported when coupling with chemistry" stop endif if (Interstitial%trans_aero) Interstitial%nvdiff = Interstitial%nvdiff + Model%ntchm From ea00cbf1885a2392e602cdf0c648d763d90a38c3 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Fri, 25 Feb 2022 01:18:59 +0000 Subject: [PATCH 11/20] Rewind model clock by one time step after run phase 1 to ensure restarts are properly generated. --- fv3_cap.F90 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 87dbe0e69..93988aeb4 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -1105,18 +1105,21 @@ subroutine TimestampExport_phase1(gcomp, rc) ! local variables character(len=*),parameter :: subname='(fv3gfs_cap:TimestampExport_phase1)' - type(ESMF_Clock) :: driverClock, modelClock + type(ESMF_Clock) :: modelClock type(ESMF_State) :: exportState rc = ESMF_SUCCESS ! get driver and model clock - call NUOPC_ModelGet(gcomp, driverClock=driverClock, & - modelClock=modelClock, exportState=exportState, rc=rc) + call NUOPC_ModelGet(gcomp, modelClock=modelClock, exportState=exportState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! reset model clock to initial time - call NUOPC_CheckSetClock(modelClock, driverClock, forceCurrTime=.true., rc=rc) + ! rewind clock by one time step + call ESMF_ClockSet(modelClock, direction=ESMF_DIRECTION_REVERSE, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_ClockAdvance(modelClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_ClockSet(modelClock, direction=ESMF_DIRECTION_FORWARD, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! update timestamp on export Fields From d37ec5b4eeacfc8ff8bbda42c3a1feaf29b3e7c6 Mon Sep 17 00:00:00 2001 From: JessicaMeixner-NOAA Date: Tue, 1 Mar 2022 10:32:27 -0600 Subject: [PATCH 12/20] rename suite files from p8b to p8 --- ...3_GFS_v17_coupled_p8b.xml => suite_FV3_GFS_v17_coupled_p8.xml} | 0 .../{suite_FV3_GFS_v17_p8b.xml => suite_FV3_GFS_v17_p8.xml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ccpp/suites/{suite_FV3_GFS_v17_coupled_p8b.xml => suite_FV3_GFS_v17_coupled_p8.xml} (100%) rename ccpp/suites/{suite_FV3_GFS_v17_p8b.xml => suite_FV3_GFS_v17_p8.xml} (100%) diff --git a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml similarity index 100% rename from ccpp/suites/suite_FV3_GFS_v17_coupled_p8b.xml rename to ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml diff --git a/ccpp/suites/suite_FV3_GFS_v17_p8b.xml b/ccpp/suites/suite_FV3_GFS_v17_p8.xml similarity index 100% rename from ccpp/suites/suite_FV3_GFS_v17_p8b.xml rename to ccpp/suites/suite_FV3_GFS_v17_p8.xml From fa098a3f186364a59dfedfea99e8c88cfab773e0 Mon Sep 17 00:00:00 2001 From: JessicaMeixner-NOAA Date: Tue, 1 Mar 2022 13:02:00 -0600 Subject: [PATCH 13/20] update suite file name in files --- ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml | 2 +- ccpp/suites/suite_FV3_GFS_v17_p8.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml index dfe858bbc..b5730d686 100644 --- a/ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml +++ b/ccpp/suites/suite_FV3_GFS_v17_coupled_p8.xml @@ -1,6 +1,6 @@ - + diff --git a/ccpp/suites/suite_FV3_GFS_v17_p8.xml b/ccpp/suites/suite_FV3_GFS_v17_p8.xml index dd8480334..10d4e8c19 100644 --- a/ccpp/suites/suite_FV3_GFS_v17_p8.xml +++ b/ccpp/suites/suite_FV3_GFS_v17_p8.xml @@ -1,6 +1,6 @@ - + From 4a08834e382b0e198561521711c2e95910875424 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 3 Mar 2022 08:26:37 -0600 Subject: [PATCH 14/20] fix 2phases intermediate restart --- fv3_cap.F90 | 13 +++++-------- module_fcst_grid_comp.F90 | 15 +++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 93988aeb4..87dbe0e69 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -1105,21 +1105,18 @@ subroutine TimestampExport_phase1(gcomp, rc) ! local variables character(len=*),parameter :: subname='(fv3gfs_cap:TimestampExport_phase1)' - type(ESMF_Clock) :: modelClock + type(ESMF_Clock) :: driverClock, modelClock type(ESMF_State) :: exportState rc = ESMF_SUCCESS ! get driver and model clock - call NUOPC_ModelGet(gcomp, modelClock=modelClock, exportState=exportState, rc=rc) + call NUOPC_ModelGet(gcomp, driverClock=driverClock, & + modelClock=modelClock, exportState=exportState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! rewind clock by one time step - call ESMF_ClockSet(modelClock, direction=ESMF_DIRECTION_REVERSE, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_ClockAdvance(modelClock, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_ClockSet(modelClock, direction=ESMF_DIRECTION_FORWARD, rc=rc) + ! reset model clock to initial time + call NUOPC_CheckSetClock(modelClock, driverClock, forceCurrTime=.true., rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! update timestamp on export Fields diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index 7fe309d3f..f662ede5b 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -585,6 +585,13 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) enddo endif endif +! frestart only contains intermediate restart + do i=1,size(frestart) + if(frestart(i) == total_inttime) then + frestart(i) = 0 + exit + endif + enddo ! if to write out restart at the end of forecast restart_endfcst = .false. if ( ANY(frestart(:) == total_inttime) ) restart_endfcst = .true. @@ -1017,11 +1024,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) ! call ESMF_GridCompGet(fcst_comp, localpet=mype, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_ClockGet(clock, advanceCount=NTIMESTEP_ESMF, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - na = NTIMESTEP_ESMF ! !----------------------------------------------------------------------- ! *** call fcst integration subroutines @@ -1034,8 +1036,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !--- intermediate restart if (intrm_rst>0) then - if (na /= num_atmos_calls-1) then - call get_time(Atmos%Time - Atmos%Time_init, seconds) if (ANY(frestart(:) == seconds)) then if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & 'integration lenght=',na*dt_atmos/3600. @@ -1059,7 +1059,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) call mpp_close(unit) endif endif - endif endif if (mype == 0) write(*,*)"PASS: fcstRUN phase 2, na = ",na, ' time is ', mpi_wtime()-tbeg1 From 6e3cdf68fd89c17ba48f72b6c24bf6aaed8a4138 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 3 Mar 2022 14:14:46 -0600 Subject: [PATCH 15/20] make frestart for intermediate restart time --- module_fcst_grid_comp.F90 | 51 ++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index f662ede5b..d5f358ddd 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -585,6 +585,9 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) enddo endif endif +! if to write out restart at the end of forecast + restart_endfcst = .false. + if ( ANY(frestart(:) == total_inttime) ) restart_endfcst = .true. ! frestart only contains intermediate restart do i=1,size(frestart) if(frestart(i) == total_inttime) then @@ -592,9 +595,6 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) exit endif enddo -! if to write out restart at the end of forecast - restart_endfcst = .false. - if ( ANY(frestart(:) == total_inttime) ) restart_endfcst = .true. if (mype == 0) print *,'frestart=',frestart(1:10)/3600, 'restart_endfcst=',restart_endfcst, & 'total_inttime=',total_inttime ! if there is restart writing during integration @@ -1036,29 +1036,30 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !--- intermediate restart if (intrm_rst>0) then - if (ANY(frestart(:) == seconds)) then - if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & - 'integration lenght=',na*dt_atmos/3600. - - timestamp = date_to_string (Atmos%Time) - call atmos_model_restart(Atmos, timestamp) - call write_stoch_restart_atm('RESTART/'//trim(timestamp)//'.atm_stoch.res.nc') - - !----- write restart file ------ - if (mpp_pe() == mpp_root_pe())then - call get_date (Atmos%Time, date(1), date(2), date(3), & - date(4), date(5), date(6)) - call mpp_open( unit, 'RESTART/'//trim(timestamp)//'.coupler.res', nohdrs=.TRUE. ) - write( unit, '(i6,8x,a)' )calendar_type, & - '(Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)' - - write( unit, '(6i6,8x,a)' )date_init, & - 'Model start time: year, month, day, hour, minute, second' - write( unit, '(6i6,8x,a)' )date, & - 'Current model time: year, month, day, hour, minute, second' - call mpp_close(unit) - endif + call get_time(Atmos%Time - Atmos%Time_init, seconds) + if (ANY(frestart(:) == seconds)) then + if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & + 'integration lenght=',na*dt_atmos/3600. + + timestamp = date_to_string (Atmos%Time) + call atmos_model_restart(Atmos, timestamp) + call write_stoch_restart_atm('RESTART/'//trim(timestamp)//'.atm_stoch.res.nc') + + !----- write restart file ------ + if (mpp_pe() == mpp_root_pe())then + call get_date (Atmos%Time, date(1), date(2), date(3), & + date(4), date(5), date(6)) + call mpp_open( unit, 'RESTART/'//trim(timestamp)//'.coupler.res', nohdrs=.TRUE. ) + write( unit, '(i6,8x,a)' )calendar_type, & + '(Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)' + + write( unit, '(6i6,8x,a)' )date_init, & + 'Model start time: year, month, day, hour, minute, second' + write( unit, '(6i6,8x,a)' )date, & + 'Current model time: year, month, day, hour, minute, second' + call mpp_close(unit) endif + endif endif if (mype == 0) write(*,*)"PASS: fcstRUN phase 2, na = ",na, ' time is ', mpi_wtime()-tbeg1 From a84357225e2c75b13a5cfee72d145b5445196cc1 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 3 Mar 2022 14:20:43 -0600 Subject: [PATCH 16/20] set na in phase 2 --- module_fcst_grid_comp.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index d5f358ddd..ce895e33d 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -1037,6 +1037,7 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !--- intermediate restart if (intrm_rst>0) then call get_time(Atmos%Time - Atmos%Time_init, seconds) + na = seconds/dt_atmos - 1 if (ANY(frestart(:) == seconds)) then if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & 'integration lenght=',na*dt_atmos/3600. From d676423efd3fa88bc1defa23076899ccc3fc9a3b Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 3 Mar 2022 21:17:21 -0600 Subject: [PATCH 17/20] clean up num_atmos_calls --- module_fcst_grid_comp.F90 | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index ce895e33d..f507463c0 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -86,7 +86,7 @@ module module_fcst_grid_comp integer :: ngrids, mygrid integer,dimension(:),allocatable :: grid_number_on_all_pets(:) - integer :: num_atmos_calls, intrm_rst + integer :: intrm_rst !----- coupled model data ----- @@ -324,10 +324,8 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! type(ESMF_VM) :: VM type(ESMF_Time) :: CurrTime, StartTime, StopTime - type(ESMF_TimeInterval) :: RunDuration type(ESMF_Config) :: cf - integer :: Run_length integer,dimension(6) :: date, date_end ! character(len=9) :: month @@ -523,19 +521,11 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) endif ! fexist endif ! mype == 0 - RunDuration = StopTime - CurrTime - - CALL ESMF_TimeIntervalGet(RunDuration, S=Run_length, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! call diag_manager_init (TIME_INIT=date) call diag_manager_set_time_end(Time_end) ! Time_step = set_time (dt_atmos,0) - num_atmos_calls = Run_length / dt_atmos - if (mype == 0) write(*,*)'num_atmos_calls=',num_atmos_calls,'time_init=', & - date_init,'time=',date,'time_end=',date_end,'dt_atmos=',dt_atmos, & - 'Run_length=',Run_length + if (mype == 0) write(*,*)'time_init=', date_init,'time=',date,'time_end=',date_end,'dt_atmos=',dt_atmos ! set up forecast time array that controls when to write out restart files frestart = 0 @@ -953,8 +943,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) ! !*** local variables ! - integer :: mype, na - integer(kind=ESMF_KIND_I8) :: ntimestep_esmf + integer :: mype, na, seconds real(kind=8) :: mpi_wtime, tbeg1 ! !----------------------------------------------------------------------- @@ -968,11 +957,9 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) ! call ESMF_GridCompGet(fcst_comp, localpet=mype, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! - call ESMF_ClockGet(clock, advanceCount=NTIMESTEP_ESMF, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - na = NTIMESTEP_ESMF + call get_time(Atmos%Time - Atmos%Time_init, seconds) + na = seconds/dt_atmos ! !----------------------------------------------------------------------- ! *** call fcst integration subroutines @@ -1008,7 +995,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !*** local variables ! integer :: mype, na, date(6), seconds - integer(kind=ESMF_KIND_I8) :: ntimestep_esmf character(len=64) :: timestamp integer :: unit real(kind=8) :: mpi_wtime, tbeg1 @@ -1040,7 +1026,7 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) na = seconds/dt_atmos - 1 if (ANY(frestart(:) == seconds)) then if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & - 'integration lenght=',na*dt_atmos/3600. + 'integration length=',na*dt_atmos/3600. timestamp = date_to_string (Atmos%Time) call atmos_model_restart(Atmos, timestamp) From a23347ecb679c6bbf838204a65fbde77d25e937d Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 4 Mar 2022 08:21:10 -0600 Subject: [PATCH 18/20] compute na once --- module_fcst_grid_comp.F90 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index f507463c0..ce41e0835 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -86,7 +86,7 @@ module module_fcst_grid_comp integer :: ngrids, mygrid integer,dimension(:),allocatable :: grid_number_on_all_pets(:) - integer :: intrm_rst + integer :: intrm_rst, na !----- coupled model data ----- @@ -943,7 +943,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) ! !*** local variables ! - integer :: mype, na, seconds + integer :: mype, seconds real(kind=8) :: mpi_wtime, tbeg1 ! !----------------------------------------------------------------------- @@ -994,7 +994,7 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) ! !*** local variables ! - integer :: mype, na, date(6), seconds + integer :: mype, date(6), seconds character(len=64) :: timestamp integer :: unit real(kind=8) :: mpi_wtime, tbeg1 @@ -1022,8 +1022,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !--- intermediate restart if (intrm_rst>0) then - call get_time(Atmos%Time - Atmos%Time_init, seconds) - na = seconds/dt_atmos - 1 if (ANY(frestart(:) == seconds)) then if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & 'integration length=',na*dt_atmos/3600. From 56f6e682ae6ee313156eabfc58c2ff9f6f9647ce Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 4 Mar 2022 08:33:50 -0600 Subject: [PATCH 19/20] fix seconds --- module_fcst_grid_comp.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index ce41e0835..17cf280d4 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -1022,6 +1022,7 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !--- intermediate restart if (intrm_rst>0) then + call get_time(Atmos%Time - Atmos%Time_init, seconds) if (ANY(frestart(:) == seconds)) then if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & 'integration length=',na*dt_atmos/3600. From 03fb73f422bf4f2a0b58124d8d2198af5a0af643 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 4 Mar 2022 13:49:00 -0600 Subject: [PATCH 20/20] change na to n_atmsteps --- module_fcst_grid_comp.F90 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index 17cf280d4..9c67f254c 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -86,7 +86,7 @@ module module_fcst_grid_comp integer :: ngrids, mygrid integer,dimension(:),allocatable :: grid_number_on_all_pets(:) - integer :: intrm_rst, na + integer :: intrm_rst, n_atmsteps !----- coupled model data ----- @@ -959,7 +959,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return call get_time(Atmos%Time - Atmos%Time_init, seconds) - na = seconds/dt_atmos + n_atmsteps = seconds/dt_atmos ! !----------------------------------------------------------------------- ! *** call fcst integration subroutines @@ -971,7 +971,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) call atmos_model_exchange_phase_1 (Atmos, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (mype == 0) write(*,*)"PASS: fcstRUN phase 1, na = ",na, ' time is ', mpi_wtime()-tbeg1 + if (mype == 0) write(*,*)"PASS: fcstRUN phase 1, n_atmsteps = ",n_atmsteps, ' time is ', mpi_wtime()-tbeg1 ! !----------------------------------------------------------------------- ! @@ -1024,8 +1024,8 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) if (intrm_rst>0) then call get_time(Atmos%Time - Atmos%Time_init, seconds) if (ANY(frestart(:) == seconds)) then - if (mype == 0) write(*,*)'write out restart at na=',na,' seconds=',seconds, & - 'integration length=',na*dt_atmos/3600. + if (mype == 0) write(*,*)'write out restart at n_atmsteps=',n_atmsteps,' seconds=',seconds, & + 'integration length=',n_atmsteps*dt_atmos/3600. timestamp = date_to_string (Atmos%Time) call atmos_model_restart(Atmos, timestamp) @@ -1048,7 +1048,7 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) endif endif - if (mype == 0) write(*,*)"PASS: fcstRUN phase 2, na = ",na, ' time is ', mpi_wtime()-tbeg1 + if (mype == 0) write(*,*)"PASS: fcstRUN phase 2, n_atmsteps = ",n_atmsteps, ' time is ', mpi_wtime()-tbeg1 ! !----------------------------------------------------------------------- !