Skip to content

Commit

Permalink
Merge branch 'ocean/develop' into e3sm/develop
Browse files Browse the repository at this point in the history 
Codes for a semi implicit barotropic mode solver #422
  • Loading branch information
@mark-petersen
mark-petersen committed Jan 13, 2021
2 parents ff3717e + 3d13880 commit 5c8b841
Show file tree
Hide file tree
Showing 12 changed files with 4,099 additions and 21 deletions.
22 changes: 17 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -593,10 +593,21 @@ endif
LIBS += -L$(PNETCDF)/$(PNETCDFLIBLOC) -lpnetcdf
endif

ifneq "$(LAPACK)" ""
LIBS += -L$(LAPACK)
LIBS += -llapack
LIBS += -lblas
ifeq "$(USE_LAPACK)" "true"
ifndef LAPACK
$(error LAPACK is not set. Please set LAPACK to the LAPACK install directory when USE_LAPACK=true)
endif

ifneq (, $(shell ls $(LAPACK)/liblapack.*))
LIBS += -L$(LAPACK)
else ifneq (, $(shell ls $(LAPACK)/lib/liblapack.*))
LIBS += -L$(LAPACK)/lib
else
$(error liblapack.* does NOT exist in $(LAPACK) or $(LAPACK)/lib)
endif
LIBS += -llapack
LIBS += -lblas
override CPPFLAGS += -DUSE_LAPACK
endif

RM = rm -f
Expand Down Expand Up @@ -1045,8 +1056,9 @@ errmsg:
@echo " USE_PIO2=true - links with the PIO 2 library. Default is to use the PIO 1.x library."
@echo " PRECISION=single - builds with default single-precision real kind. Default is to use double-precision."
@echo " SHAREDLIB=true - generate position-independent code suitable for use in a shared library. Default is false."
@echo " USE_LAPACK=true - builds and links with LAPACK / BLAS libraries. Default is to not use LAPACK."
@echo ""
@echo "Ensure that NETCDF, PNETCDF, PIO, and PAPI (if USE_PAPI=true) are environment variables"
@echo "Ensure that NETCDF, PNETCDF, PIO, LAPACK (if USE_LAPACK=true), and PAPI (if USE_PAPI=true) are environment variables"
@echo "that point to the absolute paths for the libraries."
@echo ""
ifdef CORE
Expand Down
150 changes: 144 additions & 6 deletions src/core_ocean/Registry.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@
/>
<nml_option name="config_time_integrator" type="character" default_value="split_explicit" units="unitless"
description="Time integration method."
possible_values="'split_explicit', 'RK4', 'unsplit_explicit'"
possible_values="'split_explicit', 'RK4', 'unsplit_explicit', 'semi_implicit'"
/>
</nml_record>
<nml_record name="hmix" mode="forward">
Expand Down Expand Up @@ -1044,23 +1044,25 @@
possible_values="any positive real"
/>
</nml_record>
<nml_record name="split_explicit_ts" mode="forward">
<nml_record name="split_timestep_share" mode="forward">
<nml_option name="config_n_ts_iter" type="integer" default_value="2" units="unitless"
description="number of large iterations over stages 1-3"
possible_values="any positive integer, but typically 1, 2, or 3"
/>
<nml_option name="config_n_bcl_iter_beg" type="integer" default_value="1" units="unitless"
description="number of iterations of stage 1 (baroclinic solve) on the first split-explicit iteration"
description="number of iterations of stage 1 (baroclinic solve) on the first split timestepping iteration"
possible_values="any positive integer, but typically 1, 2, or 3"
/>
<nml_option name="config_n_bcl_iter_mid" type="integer" default_value="2" units="unitless"
description="number of iterations of stage 1 (baroclinic solve) on any split-explicit iterations between first and last"
description="number of iterations of stage 1 (baroclinic solve) on any split timestepping iterations between first and last"
possible_values="any positive integer, but typically 1, 2, or 3"
/>
<nml_option name="config_n_bcl_iter_end" type="integer" default_value="2" units="unitless"
description="number of iterations of stage 1 (baroclinic solve) on the last split-explicit iteration"
description="number of iterations of stage 1 (baroclinic solve) on the last split timestepping iteration"
possible_values="any positive integer, but typically 1, 2, or 3"
/>
</nml_record>
<nml_record name="split_explicit_ts" mode="forward">
<nml_option name="config_btr_dt" type="character" default_value="0000_00:00:15" units="unitless"
description="Timestep to use for the barotropic mode in the split explicit time integrator"
possible_values="Any time stamp in 'YYYY-MM-DD_hh:mm:ss' format. Items can be removed from the left if they are unused."
Expand Down Expand Up @@ -1094,6 +1096,24 @@
possible_values=".true. or .false."
/>
</nml_record>
<nml_record name="semi_implicit_ts" mode="forward">
<nml_option name="config_btr_si_preconditioner" type="character" default_value="ras" units="unitless"
description="Type of preconditioner for the barotropic mode solver"
possible_values="ras, block_jacobi, jacobi, none"
/>
<nml_option name="config_btr_si_tolerance" type="real" default_value="1.0e-9" units="unitless"
description="Tolerance for the barotropic mode solver"
possible_values="any positive real, but typically less than 1.0e-9"
/>
<nml_option name="config_n_btr_si_outer_iter" type="integer" default_value="2" units="unitless"
description="number of outer iterations"
possible_values="any positive integer, but typically 2"
/>
<nml_option name="config_btr_si_partition_match_mode" type="logical" default_value=".false." units="unitless"
description="If true, the semi-implicit method uses the Jacobi preconditioner with the bit-for-bit all-reduce. This is less performant, so should only be used for testing."
possible_values=".true. or .false."
/>
</nml_record>
<nml_record name="ALE_vertical_grid" mode="forward">
<nml_option name="config_vert_coord_movement" type="character" default_value="uniform_stretching" units="unitless"
description="Determines the vertical coordinate movement type. 'uniform_stretching' distributes SSH perturbations through all vertical levels (z-star vertical coordinate); 'fixed' places them all in the top level (z-level vertical coordinate); 'user_specified' allows the input file to determine the distribution using the variable vertCoordMovementWeights (weighted z-star vertical coordinate); and 'impermeable_interfaces' makes the vertical transport between layers zero, i.e. $w^t=0$ (idealized isopycnal)."
Expand Down Expand Up @@ -1288,6 +1308,7 @@
<package name="variableShortwave" description="This package includes variables required to compute spatially variable shortwave extinction coefficients"/>

<package name="splitTimeIntegrator" description="This package includes variables required for either the split or unsplit explicit time integrators."/>
<package name="semiImplicitTimePKG" description="This package includes variables required for semi-implicit time integrators."/>
<package name="thicknessFilter" description="This package includes variables required for frequency filtered thickness."/>
<package name="windStressBulkPKG" description="This package includes varibles required for bulk wind stress forcing."/>
<package name="variableBottomDragPKG" description="This package includes varibles required for variable bottom drag."/>
Expand Down Expand Up @@ -2317,7 +2338,7 @@
description="salinity averaged over ocean surface layer (generally 0.1 of the ocean boundary layer)"
/>
</var_array>
<var name="normalVelocitySurfaceLayer" type="real" dimensions="nEdges Time" nits="m s^{-1}"
<var name="normalVelocitySurfaceLayer" type="real" dimensions="nEdges Time" units="m s^{-1}"
description="normal velocity averaged over ocean surface layer (generally 0.1 of the ocean boundary layer)"
packages="forwardMode;analysisMode"
/>
Expand Down Expand Up @@ -2784,6 +2805,123 @@
<var name="pressureAdjustedSSH" type="real" dimensions="nCells Time" units="m"
description="sea surface height adjusted by sea surface pressure"
/>
<!-- FIELD semi_implicit time stepping -->
<var name="barotropicCoriolisTerm" type="real" dimensions="nEdges Time" units="m s^{-2}"
description="f * uPerp for the semi-implicit time stepping"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_r0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_r1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_rh0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_rh1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_r00" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_ph0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_ph1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_v0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_v1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_s0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_s1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_sh0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_sh1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_w0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_w1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_wh0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_wh1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_q0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_q1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_qh0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_qh1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_z0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_z1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_zh0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_zh1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_t0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_t1" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
<var name="CGvec_y0" type="real" dimensions="nCells Time" units="unitless"
description="A vector used in the semi-implicit barotropic mode solver"
packages="semiImplicitTimePKG"
/>
</var_struct>
<var_struct name="shortwave" time_levs="1">
<!-- **********************************************************************
Expand Down
3 changes: 2 additions & 1 deletion src/core_ocean/analysis_members/mpas_ocn_high_frequency_output.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,8 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{
call mpas_pool_get_array(highFrequencyOutputAMPool, 'vertVelSFC', vertVelSFC)

! split explicit specific arrays
if ( config_time_integrator == trim('split_explicit')) then
call mpas_pool_get_config(ocnConfigs, 'config_time_integrator', config_time_integrator)
if ( config_time_integrator == trim('split_explicit') .or. config_time_integrator == trim('semi_implicit') ) then
call mpas_pool_get_array(statePool, 'normalBaroclinicVelocity', normalBaroclinicVelocity, 1)
call mpas_pool_get_array(statePool, 'normalBarotropicVelocity', normalBarotropicVelocity, 1)
endif
Expand Down
11 changes: 9 additions & 2 deletions src/core_ocean/driver/mpas_ocn_core_interface.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{
logical, pointer :: initModeActive
logical, pointer :: thicknessFilterActive
logical, pointer :: splitTimeIntegratorActive
logical, pointer :: semiImplicitTimePKGActive
logical, pointer :: windStressBulkPKGActive
logical, pointer :: variableBottomDragPKGActive
logical, pointer :: tracerBudgetActive
Expand Down Expand Up @@ -195,16 +196,22 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{

!
! test for integration scheme
! (TDR: this makes no sense, if split or unsplit then splitTimeIntegratorActive = .true.)
!
call mpas_pool_get_package(packagePool, 'splitTimeIntegratorActive', splitTimeIntegratorActive)
call mpas_pool_get_config(configPool, 'config_time_integrator', config_time_integrator)
if ( forwardModeActive ) then
if ( config_time_integrator == trim('split_explicit') &
.or. config_time_integrator == trim('unsplit_explicit') ) then
.or. config_time_integrator == trim('unsplit_explicit') &
.or. config_time_integrator == trim('semi_implicit') ) then
splitTimeIntegratorActive = .true.
end if
endif
call mpas_pool_get_package(packagePool, 'semiImplicitTimePKGActive', semiImplicitTimePKGActive)
if ( forwardModeActive ) then
if (config_time_integrator == trim('semi_implicit') ) then
semiImplicitTimePKGActive = .true.
end if
endif

!
! test for time filtering scheme
Expand Down
6 changes: 5 additions & 1 deletion src/core_ocean/mode_forward/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@
OBJS = mpas_ocn_forward_mode.o \
mpas_ocn_time_integration.o \
mpas_ocn_time_integration_rk4.o \
mpas_ocn_time_integration_si.o \
mpas_ocn_time_integration_split.o

all: forward_mode

forward_mode: $(OBJS)

mpas_ocn_time_integration.o: mpas_ocn_time_integration_rk4.o mpas_ocn_time_integration_split.o
mpas_ocn_time_integration.o: mpas_ocn_time_integration_rk4.o mpas_ocn_time_integration_si.o mpas_ocn_time_integration_split.o

mpas_ocn_time_integration_rk4.o:

mpas_ocn_time_integration_si.o:

mpas_ocn_time_integration_split.o:

mpas_ocn_forward_mode.o: mpas_ocn_time_integration.o \
mpas_ocn_time_integration_rk4.o \
mpas_ocn_time_integration_si.o \
mpas_ocn_time_integration_split.o

clean:
Expand Down
21 changes: 20 additions & 1 deletion src/core_ocean/mode_forward/mpas_ocn_forward_mode.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ module ocn_forward_mode

use ocn_time_integration
use ocn_time_integration_split
use ocn_time_integration_si
use ocn_tendency
use ocn_diagnostics
use ocn_test
Expand Down Expand Up @@ -262,7 +263,13 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{

call ocn_init_routines_compute_max_level(domain)

call ocn_time_integration_split_init(domain)
call mpas_pool_get_config(domain % configs, 'config_time_integrator', config_time_integrator)

if (trim(config_time_integrator) == 'semi_implicit' ) then
call ocn_time_integration_si_init(domain)
else
call ocn_time_integration_split_init(domain)
endif

call mpas_log_write(' Vertical coordinate movement is: ' // trim(config_vert_coord_movement))

Expand Down Expand Up @@ -380,6 +387,18 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{

endif

! Preconditioner for the semi-implicit barotropic mode solver
if (config_time_integrator == 'semi_implicit' ) then

call mpas_log_write(' Building a preconditioning matrix: ')
call mpas_timer_start("preconditioning matrix build")

! Building a preconditioning matrix
call ocn_time_integrator_si_preconditioner(domain, dt)

call mpas_timer_stop("preconditioning matrix build")
endif

call mpas_dmpar_exch_group_create(domain, 'subcycleFields')
call mpas_dmpar_exch_group_add_field(domain, 'subcycleFields', 'sshSubcycle')
call mpas_dmpar_exch_group_add_field(domain, 'subcycleFields', 'normalBarotropicVelocitySubcycle')
Expand Down
10 changes: 8 additions & 2 deletions src/core_ocean/mode_forward/mpas_ocn_time_integration.F
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ module ocn_time_integration
use ocn_config
use ocn_time_integration_rk4
use ocn_time_integration_split
use ocn_time_integration_si

implicit none
private
Expand All @@ -60,7 +61,7 @@ module ocn_time_integration
!
!--------------------------------------------------------------------

logical :: rk4On, splitOn
logical :: rk4On, splitOn, simpOn

contains

Expand Down Expand Up @@ -109,6 +110,8 @@ subroutine ocn_timestep(domain, dt, timeStamp)!{{{
call ocn_time_integrator_rk4(domain, dt)
elseif (splitOn) then
call ocn_time_integrator_split(domain, dt)
elseif (simpOn) then
call ocn_time_integrator_si(domain, dt)
endif

block => domain % blocklist
Expand Down Expand Up @@ -143,16 +146,19 @@ subroutine ocn_timestep_init(err)!{{{

rk4On = .false.
splitOn = .false.
simpOn = .false.

if (trim(config_time_integrator) == 'RK4') then
rk4On = .true.
elseif (trim(config_time_integrator) == 'split_explicit' &
.or.trim(config_time_integrator) == 'unsplit_explicit') then
splitOn = .true.
elseif (trim(config_time_integrator) == 'semi_implicit' ) then
simpOn = .true.
else
err = 1
call mpas_log_write('Incorrect choice for config_time_integrator:' // trim(config_time_integrator) // &
' choices are: RK4, split_explicit, unsplit_explicit', MPAS_LOG_CRIT)
' choices are: RK4, split_explicit, unsplit_explicit, semi_implicit', MPAS_LOG_CRIT)
endif


Expand Down
Loading

0 comments on commit 5c8b841

Please sign in to comment.