diff --git a/configuration/scripts/tests/CTest/CTestConfig.cmake b/configuration/scripts/tests/CTest/CTestConfig.cmake deleted file mode 100644 index 51bb4ecfa..000000000 --- a/configuration/scripts/tests/CTest/CTestConfig.cmake +++ /dev/null @@ -1,6 +0,0 @@ -set(CTEST_PROJECT_NAME "myCICE") -set(CTEST_DROP_METHOD "http") -set(CTEST_DROP_SITE "my.cdash.org") -set(CTEST_DROP_LOCATION "/submit.php?project=myCICE") -set(CTEST_DROP_SITE_CDASH TRUE) -set(CTEST_NIGHTLY_START_TIME "01:00:00 CET") diff --git a/configuration/scripts/tests/CTest/CTestTestfile.cmake b/configuration/scripts/tests/CTest/CTestTestfile.cmake deleted file mode 100644 index 8a31a709e..000000000 --- a/configuration/scripts/tests/CTest/CTestTestfile.cmake +++ /dev/null @@ -1,6 +0,0 @@ - -# Add your custom executables here in the form: -# add_test( [ ...]) -# -# Make your test return 0 for success, nonzero for failure. - diff --git a/configuration/scripts/tests/CTest/gen_ctestfile.csh b/configuration/scripts/tests/CTest/gen_ctestfile.csh deleted file mode 100755 index e2179d5dd..000000000 --- a/configuration/scripts/tests/CTest/gen_ctestfile.csh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/csh -f - -# This script is passed the test name, and whether or not bfbcomp is to be performed. -# It then builds the CTestTestfile.cmake file per the inputs - -# $1 = $tsdir (the directory where CTestTestfile.cmake is to be put) -# $2 = $testname_noid (the name of the test) -# $3 = $bfbcomp -# $4 = $spval (used to test for bfbcomp) - -# Every test needs to have the "build" phase -echo "add_test($2_build grep "\""PASS.*$2 .*build"\"" results.log)" >> $1/CTestTestfile.cmake - -# If this is a restart test, add the 'run-initial' and 'run-restart' tests -if ( "$2" =~ *"_restart_"* ) then - echo "add_test($2_run_initial grep "\""PASS.*$2 .*run-initial"\"" results.log)" >> $1/CTestTestfile.cmake - echo "add_test($2_run_restart grep "\""PASS.*$2 .*run-restart"\"" results.log)" >> $1/CTestTestfile.cmake - echo "add_test($2_exact_restart grep "\""PASS.*$2 .*exact-restart"\"" results.log)" >> $1/CTestTestfile.cmake -else - echo "add_test($2_run grep "\""PASS.*$2 .*run"\"" results.log)" >> $1/CTestTestfile.cmake - echo "add_test($2_compare grep "\""PASS.*$2 .*compare"\"" results.log)" >> $1/CTestTestfile.cmake - # Check for bfbcomp in sets - if ( $3 != $4 ) then - echo "add_test($2_bfbcomp grep "\""PASS.*$2 .*bfbcomp"\"" results.log)" >> $1/CTestTestfile.cmake - endif -endif - diff --git a/configuration/scripts/tests/CTest/parse_timings.csh b/configuration/scripts/tests/CTest/parse_timings.csh deleted file mode 100755 index 9532a7dcb..000000000 --- a/configuration/scripts/tests/CTest/parse_timings.csh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/csh -f - -# This script parses the timings from the tests in the test suite and writes them to the CTest -# Test.xml file prior to submitting to CDash. - -# Loop through each line of the Test.xml file -set CTEST_TAG="`head -n 1 Testing/TAG`" -set testfile="`ls Testing/${CTEST_TAG}/Test.xml`" -set outfile="Testing/${CTEST_TAG}/Test.xml.generated" -set save_time=0 -foreach line ("`cat $testfile`") - if ( "$line" =~ *"Test Status"* ) then - if ( "$line" =~ *"passed"* ) then - set save_time=1 - else - set save_time=0 - endif - endif - if ( "$line" =~ *"FullName"* ) then - if ( $save_time == 1 ) then - if ( "$line" =~ *"_run<"* ) then - set save_time=1 - else if ("$line" =~ *"_run-initial<"*) then - set save_time=2 - else if ("$line" =~ *"run-restart"*) then - set save_time=3 - else - set save_time=0 - endif - # Grab the case name - set casename=`echo $line | grep -oP '(?<=\.\/).*?(?=)'` - set casename=`echo $casename | sed 's/_run\>\|_run-initial\>\|_run-restart\>//'` - else - set save_time=0 - endif - endif - if ( "$line" =~ *"Execution Time"* && $save_time > 0 ) then - # Find the case runlog - #set runlog=`ls ./${casename}.*/logs/*runlog*` - foreach file (`ls ./${casename}.*/logs/*runlog*`) - set runlog="$file" - if ( $save_time == 2 ) then - break - endif - end - foreach line1 ("`cat $runlog`") - if ( "$line1" =~ *"Timer 2:"*) then - set runtime=`echo $line1 | grep -oP "\d+\.(\d+)?" | sort -n | tail -1` - endif - end - set local_runtime=`echo $line | grep -oP "\d+\.(\d+)?" | sort -n | tail -1` - # Grab the leading whitespace - # Replace the timing in Test.xml with the timing from the runlog file - set line=`echo "$line" | sed "s/^\(.*\)${local_runtime}\(<\/Value>\)/\1${runtime}<\/Value>/"` - set save_time=0 - endif - echo "$line" >> $outfile -end - -mv $testfile ${testfile}.bak -mv $outfile $testfile diff --git a/configuration/scripts/tests/CTest/run_ctest.csh b/configuration/scripts/tests/CTest/run_ctest.csh deleted file mode 100755 index 425665683..000000000 --- a/configuration/scripts/tests/CTest/run_ctest.csh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/csh -f - -set initargv = ( $argv[*] ) - -set dash = "-" -set submit_only=0 - -# Check if any of the results could not find the baseline dataset -grep --quiet 'baseline-does-not-exist' results.log -if ($status == 0) then - echo "Tests were not able to find the baseline datasets. No results" - echo "will be posted to CDash" - grep 'baseline-does-not-exist' results.log - exit -1 -endif - -# Read in command line arguments -set argv = ( $initargv[*] ) - -while (1) - if ( $#argv < 1 ) break; - if ("$argv[1]" == "${dash}submit") then - set submit_only=1 - shift argv - continue - endif -end - -if ( $submit_only == 0 ) then - ctest -S steer.cmake -else - # Find the filename to submit to CDash - set CTEST_TAG="`head -n 1 Testing/TAG.submit`" - -cat > submit.cmake << EOF0 -cmake_minimum_required(VERSION 2.8) -set(CTEST_DASHBOARD_ROOT "\$ENV{PWD}") -set(CTEST_SOURCE_DIRECTORY "\$ENV{PWD}") -set(CTEST_BINARY_DIRECTORY "\$ENV{PWD}") -message("source directory = \${CTEST_SOURCE_DIRECTORY}") - -include( CTestConfig.cmake ) - -ctest_start("Experimental") -ctest_submit(FILES "`pwd`/Testing/${CTEST_TAG}/Test.xml") -EOF0 - ctest -S submit.cmake -endif - -if ( $submit_only == 0 ) then - if ( -f Testing/TAG ) then - set file='Testing/TAG' - set CTEST_TAG="`head -n 1 $file`" - - # Check to see if ctest_submit was successful - set success=0 - set submit_file="Testing/Temporary/LastSubmit_$CTEST_TAG.log" - foreach line ("`cat $submit_file`") - if ( "$line" =~ *'Submission successful'* ) then - set success=1 - endif - end - - if ( $success == 0 ) then - echo "" - set xml_file="Testing/$CTEST_TAG/Test.xml" - cp Testing/TAG Testing/TAG.submit - tar czf cice_ctest.tgz run_ctest.csh CTestConfig.cmake Testing/TAG.submit \ - Testing/${CTEST_TAG}/Test.xml - echo "CTest submission failed. To try the submission again run " - echo " ./run_ctest.csh -submit" - echo "If you wish to submit the test results from another server, copy the " - echo "cice_ctest.tgz file to another server and run " - echo " ./run_ctest.csh -submit" - else - echo "Submit Succeeded" - endif - else - echo "No Testing/TAG file exists. Ensure that ctest is installed on this system." - endif -endif diff --git a/configuration/scripts/tests/CTest/steer.cmake b/configuration/scripts/tests/CTest/steer.cmake deleted file mode 100644 index ddeb9bedf..000000000 --- a/configuration/scripts/tests/CTest/steer.cmake +++ /dev/null @@ -1,43 +0,0 @@ -# ----------------------------------------------------------- -# -- Get environment -# ----------------------------------------------------------- - -## -- Set hostname -## -------------------------- -find_program(HOSTNAME_CMD NAMES hostname) -exec_program(${HOSTNAME_CMD} ARGS OUTPUT_VARIABLE HOSTNAME) - -set(CTEST_SITE "${HOSTNAME}") - -## -- Set site / build name -## -------------------------- - -find_program(UNAME NAMES uname) -macro(getuname name flag) - exec_program("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}") -endmacro(getuname) - -getuname(osname -s) -getuname(osrel -r) -getuname(cpu -m) - -find_program(GIT_CMD NAMES git) -exec_program(${GIT_CMD} ARGS rev-parse --short HEAD OUTPUT_VARIABLE GIT_COMMIT_HASH) - -find_program(IFORT_CMD NAMES ifort) -exec_program(${IFORT_CMD} ARGS --version | head -n 1 | awk '{print $3}' OUTPUT_VARIABLE COMPILER_VERSION) - -set(CTEST_BUILD_NAME "${osname}-${cpu}-intel${COMPILER_VERSION}-${GIT_COMMIT_HASH}") - -message("build name = ${CTEST_BUILD_NAME}") - -set(CTEST_DASHBOARD_ROOT "$ENV{PWD}") -set(CTEST_SOURCE_DIRECTORY "$ENV{PWD}") -set(CTEST_BINARY_DIRECTORY "$ENV{PWD}") -message("source directory = ${CTEST_SOURCE_DIRECTORY}") - -ctest_start(${MODEL} TRACK ${MODEL}) -ctest_test( BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) -message("Parsing timings into Test.xml") -execute_process(COMMAND "./parse_timings.csh") -ctest_submit( RETURN_VALUE res) diff --git a/doc/source/cice_index.rst b/doc/source/cice_index.rst index 9a620aab8..1a05b4fd4 100644 --- a/doc/source/cice_index.rst +++ b/doc/source/cice_index.rst @@ -5,10 +5,10 @@ Index of primary variables and parameters ========================================== -This index defines many of the symbols used frequently in the ice model +This index defines many of the symbols used frequently in the CICE model code. Values appearing in this list are fixed or recommended; most namelist parameters are indicated ( :math:`E_\circ`) with their default -values. For other namelist options, see Section :ref:`tab-namelist`. All +values. For other namelist options, see Section :ref:`tabnamelist`. All quantities in the code are expressed in MKS units (temperatures may take either Celsius or Kelvin units). diff --git a/doc/source/developer_guide/dg_about.rst b/doc/source/developer_guide/dg_about.rst index a88c94175..62f1ec484 100755 --- a/doc/source/developer_guide/dg_about.rst +++ b/doc/source/developer_guide/dg_about.rst @@ -1,31 +1,18 @@ :tocdepth: 3 -.. _about_dev: +.. _dev_about: About Development ================== -The Icepack model consists of three different parts, the column physics -code, the icepack driver, and the scripts. Development of each of these -pieces will be described below separately. +The CICE model consists of four different parts, the CICE dynamics and supporting infrastructure, +the CICE driver code, the Icepack column physics code, and the scripts. Development of each of these +pieces is described separately. -Subroutine calls and other linkages into Icepack from the host model should only -need to access the **icepack\_intfc\*.F90** interface modules within the -``columnphysics/`` directory. -The Icepack driver in the ``configuration/driver/`` directory is based on the CICE -model and provides an example of the sea ice host model capabilities needed for inclusion -of Icepack. In particular, host models will need to include code equivalent to that -in the modules **icedrv\_\*_column.F90**. Calls into the Icepack interface routines -are primarily from **icedrv\_step\_mod.F90** but there are others (search the driver code -for ``intfc``). +Guiding principles for the creation of CICE include the following: + - CICE can be run in stand-alone or coupled modes. A top layer driver, coupling layer, + or model cap can be used to drive the CICE model. + - The Icepack column physics modules are independent, consist of methods that operate + on individual gridcells, and contain so underlying infrastructure. CICE must call + into the Icepack using interfaces and approaches specified by Icepack. -Guiding principles for the creation of Icepack include the following: - - The column physics modules shall be independent of all sea ice model infrastructural - elements that may vary from model to model. Examples include input/output, timers, - references to CPUs or computational tasks, initialization other than that necessary for - strictly physical reasons, and anything related to a horizontal grid. - - The column physics modules shall not call or reference any routines or code that - reside outside of the **columnphysics/** directory. - - Any capabilities required by a host sea ice model (e.g. calendar variables, tracer - flags, diagnostics) shall be implemented in the driver and passed into or out of the - column physics modules via array arguments. diff --git a/doc/source/developer_guide/dg_driver.rst b/doc/source/developer_guide/dg_driver.rst index b8389b90f..076c77252 100755 --- a/doc/source/developer_guide/dg_driver.rst +++ b/doc/source/developer_guide/dg_driver.rst @@ -2,45 +2,89 @@ .. _dev_driver: -Driver Implementation -======================== - -The icepack driver is Fortran source code and exists to test the column physics -in a stand-alone mode for some simple column configurations. - -File List -------------------- - -The icepack driver consists of the following files - -| **configuration/driver/** driver for testing Icepack in stand-alone mode -| **icedrv_MAIN.F90** main program -| **icedrv_InitMod.F90** routines for initializing a run -| **icedrv_RunMod.F90** main driver routines for time stepping -| **icedrv_arrays_column.F90** essential arrays to describe the state of the ice -| **icedrv_calendar.F90** keeps track of what time it is -| **icedrv_constants.F90** physical and numerical constants and parameters -| **icedrv_diagnostics.F90** miscellaneous diagnostic and debugging routines -| **icedrv_diagnostics_bgc.F90** diagnostic routines for biogeochemistry -| **icedrv_domain_size.F90** domain sizes -| **icedrv_flux.F90** fluxes needed/produced by the model -| **icedrv_forcing.F90** routines to read and interpolate forcing data for stand-alone model runs -| **icedrv_forcing_bgc.F90** routines to read and interpolate forcing data for bgc stand-alone model runs -| **icedrv_init.F90** general initialization routines -| **icedrv_init_column.F90** initialization routines specific to the column physics -| **icedrv_restart.F90** driver for reading/writing restart files -| **icedrv_restart_bgc.F90** restart routines specific to the column physics -| **icedrv_restart_shared.F90** code shared by all restart options -| **icedrv_state.F90** essential arrays to describe the state of the ice -| **icedrv_step.F90** routines for time stepping the major code components -| **icedrv_system.F90** overall system management calls - -Overview ------------- - -The icepack driver exists to test the column physics. At the present time, it is hardwired -to run 4 different gridcells on one processor with the same forcing used for all gridcells. -There is no MPI and no threading built into the icepack driver. There is limited IO capabilities, -no history files, and no netcdf restart files. The model generally runs very quickly. - -Forcing data and details on these data are available in :ref:`force`. + +Driver and Coupling Implementation +==================================== + +The driver and coupling layer is found in **cicecore/drivers/**. The standalone driver is found +under **cicecore/drivers/cice/** and other high level coupling layers are found in other directories. +In general, CICE will build with only one of these drivers, depending how the model is run and +coupled. Within the **cicecore/drivers/cice/** directory, the following files are found, + +**CICE.F90** is the top level program file and that calls CICE_Initialize, CICE_Run, and CICE_Finalize methods. +**CICE_InitMod.F90** contains the CICE_Initialize method and other next level source code. +**CICE_RunMod.F90** contains the CICE_Run method and other next level source code. +**CICE_FinalMod.F90 ** contains the CICE_Finalize method and other next level source code. + +Other **cicecore/drivers/** directories are similarly implemented with a top level coupling layer, +that is largely specified by an external coupled system and then some version of the **CICE_InitMod.F90**, +**CICE_RunMod.F90**, and **CICE_FinalMod.F90** files. + + +Calling Sequence +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The initialize calling sequence looks something like:: + + call init_communicate ! initial setup for message passing + call init_fileunits ! unit numbers + call icepack_configure() ! initialize icepack + call input_data ! namelist variables + call init_zbgc ! vertical biogeochemistry namelist + call init_domain_blocks ! set up block decomposition + call init_grid1 ! domain distribution + call init_ice_timers ! initialize all timers + call init_grid2 ! grid variables + call init_calendar ! initialize some calendar stuff + call init_hist (dt) ! initialize output history file + if (kdyn == 2) then + call init_eap (dt_dyn) ! define eap dynamics parameters, variables + else ! for both kdyn = 0 or 1 + call init_evp (dt_dyn) ! define evp dynamics parameters, variables + endif + call init_coupler_flux ! initialize fluxes exchanged with coupler + call init_thermo_vertical ! initialize vertical thermodynamics + call icepack_init_itd(ncat, hin_max) ! ice thickness distribution + call calendar(time) ! determine the initial date + call init_forcing_ocn(dt) ! initialize sss and sst from data + call init_state ! initialize the ice state + call init_transport ! initialize horizontal transport + call ice_HaloRestore_init ! restored boundary conditions + call init_restart ! initialize restart variables + call init_diags ! initialize diagnostic output points + call init_history_therm ! initialize thermo history variables + call init_history_dyn ! initialize dynamic history variables + call init_shortwave ! initialize radiative transfer + call init_forcing_atmo ! initialize atmospheric forcing (standalone) + +See a **CICE_InitMod.F90** file for the latest. + +The run sequence within a time loop looks something like:: + + call init_mass_diags ! diagnostics per timestep + call init_history_therm + call init_history_bgc + + do iblk = 1, nblocks + if (calc_Tsfc) call prep_radiation (dt, iblk) + call step_therm1 (dt, iblk) ! vertical thermodynamics + call biogeochemistry (dt, iblk) ! biogeochemistry + call step_therm2 (dt, iblk) ! ice thickness distribution thermo + enddo ! iblk + + call update_state (dt, daidtt, dvidtt, dagedtt, offset) + + do k = 1, ndtd + call step_dyn_horiz (dt_dyn) + do iblk = 1, nblocks + call step_dyn_ridge (dt_dyn, ndtd, iblk) + enddo + call update_state (dt_dyn, daidtd, dvidtd, dagedtd, offset) + enddo + + do iblk = 1, nblocks + call step_radiation (dt, iblk) + call coupling_prep (iblk) + enddo ! iblk + +See a **CICE_RunMod.F90** file for the latest. \ No newline at end of file diff --git a/doc/source/developer_guide/dg_dynamics.rst b/doc/source/developer_guide/dg_dynamics.rst new file mode 100755 index 000000000..d86b9145f --- /dev/null +++ b/doc/source/developer_guide/dg_dynamics.rst @@ -0,0 +1,130 @@ +:tocdepth: 3 + +.. _dev_dynamics: + + +Dynamics and Infrastructure Implementation +================================================ + +The CICE **cicecore/** directory consists of the non icepack source code. Within that +directory there are the following subdirectories + +**cicecore/cicedynB/analysis** contains higher level history and diagnostic routines. + +**cicecore/cicedynB/dynamics** contains all the dynamical evp, eap, and transport routines. + +**cicecore/cicedynB/general** contains routines associated with forcing, flux calculation, +initialization, and model timestepping. + +**cicecore/cicedynB/infrastructure** contains most of the low-level infrastructure associated +with communication (halo updates, gather, scatter, global sums, etc) and I/O reading and writing +binary and netcdf files. + +**cicecore/drivers/** contains subdirectories that support stand-alone drivers and other high level +coupling layers. + +**cicecore/shared/** contains some basic methods related to grid decomposition, time managers, constants, +kinds, and restart capabilities. + + +Dynamics +~~~~~~~~~~~~~~ + +Dyanamical Solvers +************************ + +The dynamics solvers are found in **cicecore/cicedynB/dynamics/**. A couple of different solvers are +available including EVP, revised EVP, and EAP. The dynamics solver is specified in namelist with the +``kdyn`` variable. ``kdyn=1`` is evp, ``kdyn=2`` is eap, and revised evp requires the ``revised_evp`` +namelist flag be set to true. + + +Transport +************** + +The transport (advection) methods are found in **cicecore/cicedynB/dynamics/**. Two methods are supported, +upwind and remap. These are set in namelist via the advection variable. + + +Infrastructure +~~~~~~~~~~~~~~~~~~~~ + +Kinds +********* + +**cicecore/shared/ice_kinds_mod.F90** defines the kinds datatypes used in CICE. These kinds are +used throughout CICE code to define variable types. The CICE kinds are adopted from the kinds +defined in Icepack for consistency in interfaces. + +Constants +************* + +**cicecore/shared/ice_constants.F90** defines several model constants. Some are hardwired parameters +while others have internal defaults and can be set thru namelist. + +Static Array Allocation +************************** + +CICE is implemented using mainly static arrays. CICE arrays tend to be defined based on the +block size and tracer count. Those block sizes and tracer counts are defined in the case +**cice.settings** file, are passed to the build script as CPPs, like +:: + +ftn -c ... -DNXGLOB=100 -DNYGLOB=116 -DBLCKX=25 -DBLCKY=29 -DMXBLCKS=4 -DNICELYR=7 -DNSNWLYR=1 -DNICECAT=5 -DTRAGE=1 -DTRFY=1 -DTRLVL=1 -DTRPND=1 -DTRBRI=0 -DNTRAERO=1 -DTRZS=0 -DNBGCLYR=7 -DTRALG=0 -DTRBGCZ=0 -DTRDOC=0 -DTRDOC=0 -DTRDIC=0 -DTRDON=0 -DTRFED=0 -DTRFEP=0 -DTRZAERO=0 -DTRBGCS=0 -DNUMIN=11 -DNUMAX=99 ... + +and then those CPPs replace variable strings in the file **cicecore/shared/ice_domain_size.F90**. +Data in **ice_domain_size.F90** provides parameters for static array allocations throughout CICE. + +Time Manager +**************** + +Time manager data is module data in **cicecore/shared/ice_calendar.F90**. Much of the time manager +data is public and operated on during the model timestepping. The model timestepping actually takes +place in the **CICE_RunMod.F90** file which is part of the driver code and tends to look like this:: + + call ice_step + istep = istep + 1 ! update time step counters + istep1 = istep1 + 1 + time = time + dt ! determine the time and date + + + +Communication +******************** + +Two low-level communications packages, mpi and serial, are provided as part of CICE. This software +provides a middle layer between the model and the underlying libraries. Only the CICE mpi or +serial directories are compiled with CICE, not both. + +**cicedynB/infrastructure/comm/mpi/** +is based on MPI and provides various methods to do halo updates, global sums, gather/scatter, broadcasts +and similar using some fairly generic interfaces to isolate the MPI calls in the code. + +**cicedynB/infrastructure/comm/serial/** support the same interfaces, but operates +in shared memory mode with no MPI. The serial library will be used, by default in the CICE scripts, +if the number of MPI tasks is set to 1. The serial library allows the model to be run on a single +core or with OpenMP parallelism only without requiring an MPI library. + +I/O +*********** + +There are three low-level IO packages in CICE, io_netcdf, io_binary, and io_pio. This software +provides a middle layer between the model and the underlying IO writing. +Only one of the three IO directories can be built with CICE. The CICE scripts will build with the io_netcdf +by default, but other options can be selecting by setting ``ICE_IOTYPE`` in **cice.settings** in the +case. This has to be set before CICE is built. + +**cicedynB/infrastructure/io/io_netcdf/** is the +default for the standalone CICE model, and it supports writing history and restart files in netcdf +format using standard netcdf calls. It does this by writing from and reading to the root task and +gathering and scattering fields from the root task to support model parallelism. + +**cicedynB/infrastructure/io/io_binary/** supports files in binary format using a gather/scatter +approach and reading to and writing from the root task. + +**cicedynB/infrastructure/io/io_pio/** support reading and writing through the pio interface. pio +is a parallel io library (https://github.com/NCAR/ParallelIO) that supports reading and writing of +binary and netcdf file through various interfaces including netcdf and pnetcdf. pio is generally +more parallel in memory even when using serial netcdf than the standard gather/scatter methods, +and it provides parallel read/write capabilities by optionally linking and using pnetcdf. + diff --git a/doc/source/developer_guide/dg_icepack.rst b/doc/source/developer_guide/dg_icepack.rst new file mode 100755 index 000000000..7d71ed1ca --- /dev/null +++ b/doc/source/developer_guide/dg_icepack.rst @@ -0,0 +1,18 @@ +:tocdepth: 3 + +.. _dev_icepack: + +Icepack +================== + +The CICE model calls the Icepack columnphysics source code. The Icepack model is documented +separately, see https://github.com/cice-consortium/icepack. + +More specifically, the CICE model uses methods defined in **icepack_intfc.F90**. It uses +the init, query, and write methods to set, get, and document icepack values. And it follows +the icepack_warnings methodology where icepack_warnings_aborted is checked and +icepack_warnings_print is called after every call to an icepack method. It does not directly +"use" icepack data and access icepack data only thru interfaces. + + + diff --git a/doc/source/user_guide/ug_adding.rst b/doc/source/developer_guide/dg_other.rst similarity index 66% rename from doc/source/user_guide/ug_adding.rst rename to doc/source/developer_guide/dg_other.rst index 86bc22304..17c4fec53 100644 --- a/doc/source/user_guide/ug_adding.rst +++ b/doc/source/developer_guide/dg_other.rst @@ -2,14 +2,34 @@ .. _adding: -Adding things +Other things ============= + +Reproducible Sums +---------------------- + +The ‘reproducible’ option (`DITTO`) makes diagnostics bit-for-bit when +varying the number of processors. (The simulation results are +bit-for-bit regardless, because they do not require global sums or +max/mins as do the diagnostics.) This was done mainly by increasing the +precision for the global reduction calculations, except for regular +double-precision (r8) calculations involving MPI; MPI can not handle +MPI\_REAL16 on some architectures. Instead, these cases perform sums or +max/min calculations across the global block structure, so that the +results are bit-for-bit as long as the block distribution is the same +(the number of processors can be different). + +A more flexible option is available for double-precision MPI +calculations, using the namelist variable `bfbflag`. When true, this flag +produces bit-for-bit identical diagnostics with different tasks, +threads, blocks and grid decompositions. + + .. _addtimer: -~~~~~~ -Timers -~~~~~~ +Adding Timers +----------------- Timing any section of code, or multiple sections, consists of defining the timer and then wrapping the code with start and stop commands for @@ -27,9 +47,8 @@ desired, by including the block ID in the timer calls. .. _addhist: -~~~~~~~~~~~~~~ -History fields -~~~~~~~~~~~~~~ +Adding History fields +------------------------- To add a variable to be printed in the history output, search for ‘example’ in **ice\_history\_shared.F90**: @@ -58,17 +77,8 @@ section :ref:`history`. .. _addtrcr: -~~~~~~~ -Tracers -~~~~~~~ - -Each optional tracer has its own module, **ice\_[tracer].F90**, which -also contains as much of the additional tracer code as possible, and for -backward compatibility of binary restart files, each new tracer has its -own binary restart file. We recommend that the logical namelist variable -`tr\_[tracer]` be used for all calls involving the new tracer outside of -**ice\_[tracer].F90**, in case other users do not want to use that -tracer. +Adding Tracers +--------------------- A number of optional tracers are available in the code, including ice age, first-year ice area, melt pond area and volume, brine height, @@ -96,18 +106,16 @@ dependencies (e.g., :math:`a_{lvl}` and :math:`a_{pnd}` in To add a tracer, follow these steps using one of the existing tracers as a pattern. -#. **ice\_domain\_size.F90**: increase `max\_ntrcr` (can also add option - to **comp\_ice** and **bld/Macros.\***) +- **ice\_domain\_size.F90**: increase `max\_ntrcr` via cpps in the build. -#. **ice\_state.F90**: declare `nt\_[tracer]` and `tr\_[tracer]` +- **ice\_state.F90**: declare `nt\_[tracer]` and `tr\_[tracer]` -#. **ice\_[tracer].F90**: create initialization, physics, restart - routines +- create initialization, physics, and restart routines. The restart and history + routine will be in CICE. The physics will be in Icepack. -#. **ice\_fileunits.F90**: add new dump and restart file units +- **ice\_fileunits.F90**: add new dump and restart file units -#. **ice\_init.F90**: (some of this may be done in **ice\_[tracer].F90** - instead) +- to control the new tracer - add new module and `tr\_[tracer]` to list of used modules and variables @@ -126,26 +134,28 @@ a pattern. ice volume, 2 for snow volume, 2+nt\_[tracer] for dependence on other tracers) -#. **ice\_itd.F90**, **ice\_mechred.F90**: Account for new dependencies +- **ice\_itd.F90**, **ice\_mechred.F90**: Account for new dependencies if needed. -#. **CICE\_InitMod.F90**: initialize tracer (includes reading restart +- **CICE\_InitMod.F90**: initialize tracer (includes reading restart file) -#. **CICE\_RunMod.F90**, **ice\_step\_mod.F90**: +- **CICE\_RunMod.F90**, **ice\_step\_mod.F90**: - call routine to write tracer restart data - - call physics routines in **ice\_[tracer].F90** (often called from + - call physics routines as needed (often called from **ice\_step\_mod.F90**) -#. **ice\_restart.F90**: define restart variables (for binary,  and PIO) +- **ice\_restart.F90**: define restart variables (for binary,  and PIO) -#. **ice\_history\_[tracer].F90**: add history variables +- **ice\_history\_[tracer].F90**: add history variables (Section :ref:`addhist`) -#. **ice\_in**: add namelist variables to *tracer\_nml* and +- **ice\_in**: add namelist variables to *tracer\_nml* and *icefields\_nml* -#. If strict conservation is necessary, add diagnostics as noted for +- If strict conservation is necessary, add diagnostics as noted for topo ponds in Section :ref:`ponds`. + +See also Icepack documentation. diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index cc320b459..a5beb7d58 100755 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -5,8 +5,8 @@ Scripts Implementation ======================== -The scripts are the third part of the icepack package. They support setting up -cases, building, and running the icepack stand-alone model. +The scripts are the third part of the cice package. They support setting up +cases, building, and running the cice stand-alone model. File List -------------- @@ -15,21 +15,22 @@ The directory structure under configure/scripts is as follows. | **configuration/scripts/** | **Makefile** primary makefile -| **icepack.batch.csh** creates batch scripts for particular machines -| **icepack.build** compiles the code -| **icepack.launch.csh** creates script logic that runs the executable -| **icepack.run.setup.csh** sets up the run scripts -| **icepack.run.suite.csh** sets up the test suite -| **icepack.settings** defines environment, model configuration and run settings -| **icepack.test.setup.csh** creates configurations for testing the model -| **icepack_decomp.csh** defines the grid size -| **icepack_in** namelist input data +| **cice.batch.csh** creates batch scripts for particular machines +| **cice.build** compiles the code +| **cice.decomp.csh** computes a decomposition given a grid and task/thread count +| **cice.launch.csh** creates script logic that runs the executable +| **cice.run.setup.csh** sets up the run scripts +| **cice.settings** defines environment, model configuration and run settings +| **cice.test.setup.csh** creates configurations for testing the model +| **ice_in** namelist input data | **machines/** machine specific files to set env and Macros | **makdep.c** determines module dependencies -| **options/** other namelist configurations available from the icepack.setup command line +| **options/** other namelist configurations available from the cice.setup command line | **parse_namelist.sh** replaces namelist with command-line configuration -| **parse_namelist_from_settings.sh** replaces namelist with values from icepack.settings +| **parse_namelist_from_settings.sh** replaces namelist with values from cice.settings | **parse_settings.sh** replaces settings with command-line configuration +| **setup_run_dirs.csh creates the case run directories +| **set_version_number.csh** updates the model version number from the **cice.setup** command line | **tests/** scripts for configuring and running basic tests .. _dev_strategy: @@ -37,39 +38,40 @@ The directory structure under configure/scripts is as follows. Strategy ----------- -The icepack scripts are implemented such that everything is resolved after -**icepack.setup** is called. This is done by both copying specific files -into the case directory and running scripts as part of the **icepack.setup** +The cice scripts are implemented such that everything is resolved after +**cice.setup** is called. This is done by both copying specific files +into the case directory and running scripts as part of the **cice.setup** command line to setup various files. -**icepack.setup** drives the case setup. It is written in csh. All supporting -scripts are relatively simple csh or sh scripts. +**cice.setup** drives the case setup. It is written in csh. All supporting +scripts are relatively simple csh or sh scripts. See :ref:`scripts` for additional +details. + +The file **cice.settings** specifies a set of env defaults for the case. The file +**ice_in** defines the namelist input for the cice driver. -The file **icepack.settings** specifies a set of env defaults for the case. The file -**icepack_in** defines the namelist input for the icepack driver. .. _dev_options: Preset Case Options --------------------- - -``icepack.setup -s`` option allows the user to choose some predetermined icepack +The ``cice.setup --set`` option allows the user to choose some predetermined cice settings and namelist. Those options are defined in **configurations/scripts/options/** -and the files are prefixed by either set_env, set_nml, or test_nml. When **icepack.setup** +and the files are prefixed by either set_env or set_nml. When **cice.setup** is executed, the appropriate files are read from **configurations/scripts/options/** -and the **icepack.settings** and/or **icepack_in** files are updated in the case directory +and the **cice.settings** and/or **ice_in** files are updated in the case directory based on the values in those files. The filename suffix determines the name of the -s option. So, for instance, - ``icepack.setup -s diag1,debug,bgcISPOL`` + ``cice.setup -s diag1,debug,bgcISPOL`` will search for option files with suffixes of diag1, debug, and bgcISPOL and then apply those settings. **parse_namelist.sh**, **parse_settings.sh**, and **parse_namelist_from_settings.sh** -are the three scripts that modify **icepack_in** and **icepack.settings**. +are the three scripts that modify **ice_in** and **cice.settings**. To add new options, just add new files to the **configurations/scripts/options/** directory with appropriate names and syntax. The set_nml file syntax is the same as namelist @@ -84,7 +86,7 @@ Machines Machine specific information is contained in **configuration/scripts/machines**. That directory contains a Macros file and an env file for each supported machine. One other files will need to be -changed to support a port, that is **configuration/scripts/icepack.batch.csh**. +changed to support a port, that is **configuration/scripts/cice.batch.csh**. To port to a new machine, see :ref:`porting`. .. _dev_testing: @@ -93,17 +95,30 @@ Test scripts ------------- Under **configuration/scripts/tests** are several files including the scripts to -setup the smoke and restart tests (**test_smoke.script**, **test_restart.script*). -A baseline test script (**baseline.script**) is also there to setup the regression +setup the various tests, such as smoke and restart tests (**test_smoke.script**, **test_restart.script**) +and the files the describe with options files are needed for each test (ie. **test_smoke.files**, **test_restart.files**). +A baseline test script (**baseline.script**) is also there to setup the general regression and comparison testing. That directory also contains the preset test suites (ie. **base_suite.ts**) and a file that supports post-processing on the model -output (**timeseries.csh**). +output (**timeseries.csh**). There is also a script **report_results.csh** that pushes results +from test suites back to the CICE-Consortium test results wiki page. + +The directory **configuration/scripts/tests/QC** contains scripts related to the non bit-for-bit +compliance testing described in :ref:`compliance`. + +To add a new test (for example newtest), several files may be needed, + +- **configuration/scripts/tests/test_newtest.script** defines how to run the test. This chunk + of script will be incorporated into the case test script +- **configuration/scripts/tests/test_newtest.files** list the set of options files found in + **configuration/scripts/options/** needed to + run this test. Those files will be copied into the test directory when the test is invoked + so they are available for the **test_newtest.script** to use. +- some new files may be needed in **configuration/scripts/options/**. These could be relatively + generic **set_nml** or **set_env** files, or they could be test specific files typically carrying + a prefix of **test_nml**. + +Generating a new test, particularly the **test_newtest.script** usually takes some iteration before +it's working properly. -There is a subdirectory, **configuration/scripts/tests/CTest**, that supports the -CTest scripts. These scripts allow test reporting to CDash. -To add a new test, a file associated with that test will need to be added to the -**configuration/scripts/tests** directory similar to **test_smoke.script** -and **test_restart.script**. In addition, some new options files in -**configuration/scripts/options** may need to be added similar to **test_nml.restart1**, -**test_nml.restart2**, and **set_nml.restart**. diff --git a/doc/source/developer_guide/index.rst b/doc/source/developer_guide/index.rst index 3ea52748d..1b2c2c916 100755 --- a/doc/source/developer_guide/index.rst +++ b/doc/source/developer_guide/index.rst @@ -12,6 +12,10 @@ Developer Guide :maxdepth: 3 dg_about.rst + dg_dynamics.rst dg_driver.rst + dg_icepack.rst dg_scripts.rst dg_documentation.rst + dg_other.rst + diff --git a/doc/source/user_guide/ug_citing.rst b/doc/source/intro/citing.rst similarity index 85% rename from doc/source/user_guide/ug_citing.rst rename to doc/source/intro/citing.rst index 1abf4279f..cb4c648ff 100644 --- a/doc/source/user_guide/ug_citing.rst +++ b/doc/source/intro/citing.rst @@ -13,3 +13,6 @@ DOI:10.5281/zenodo.1205674 (https://zenodo.org/record/1205674) This DOI can be used to cite all CICE versions and the URL will default to the most recent version. However, each released version of CICE will also receive its own, unique DOI that can be used for citations as well. + +Please also make the CICE-Consortium aware of any publications and model use. + diff --git a/doc/source/intro/index.rst b/doc/source/intro/index.rst index 5f7a9d452..32d943e50 100755 --- a/doc/source/intro/index.rst +++ b/doc/source/intro/index.rst @@ -15,5 +15,6 @@ Introduction - CICE quickstart.rst major_updates.rst acknowledgements.rst + citing.rst copyright.rst diff --git a/doc/source/intro/quickstart.rst b/doc/source/intro/quickstart.rst index 5f24046b4..123be8aed 100755 --- a/doc/source/intro/quickstart.rst +++ b/doc/source/intro/quickstart.rst @@ -12,6 +12,8 @@ Download the model from the CICE-Consortium repository, Instructions for working in github with CICE (and Icepack) can be found in the `CICE Git and Workflow Guide `_. +You will probably have to download some inputdata, see the `CICE wiki `_ or :ref:`force`. + From your main CICE directory, execute:: ./cice.setup -c ~/mycase1 -g gx3 -m testmachine -s diag1,thread -p 8x1 @@ -82,13 +84,13 @@ cice.build in the list are edited, cice.build should be rerun. The casescripts directory holds scripts used to create the case and can largely be ignored. In general, when cice.build is executed, the model will build from scratch due to the large -dependence on cpps. To change this behavior, edit the env variable ICE_CLEANBUILD in +dependence on cpps. To change this behavior, edit the env variable ``ICE_CLEANBUILD`` in cice.settings. The cice.submit script just submits the cice.run script. You can use cice.submit or just submit the cice.run script on the command line. -The model will run in the directory defined by the env variable CICE_RUNDIR in cice.settings. +The model will run in the directory defined by the env variable ``ICE_RUNDIR`` in cice.settings. Build and run logs will be copied into the case logs directory when complete. To port, an env.machine and Macros.machine file have to be added to scripts/machines and the cice.run.setup.csh file needs to be modified. @@ -97,7 +99,7 @@ To port, an env.machine and Macros.machine file have to be added to scripts/mach - Edit the env and Macros file - cd to consortium/scripts - Edit the cice.run.setup.csh script to add a section for your machine for the batch settings and for the job launch settings - - Download and untar the 1997 dataset to the location defined by ICE_MACHINE_INPUTDATA in the env file + - Download and untar the 1997 dataset to the location defined by ``ICE_MACHINE_INPUTDATA`` in the env file - Create a file in your home directory called .cice_proj and add your preferred account name to the first line. - You can now create a case and test. If there are problems, you can manually edit the env, Macros, and cice.run files in the case directory until things are working properly. Then you can copy the env and Macros files back to consortium/scripts/machines. You will have to manually modify the cice.run.setup.csh script if there any changes needed there. diff --git a/doc/source/user_guide/index.rst b/doc/source/user_guide/index.rst index 85c66d6c5..0b9656e20 100644 --- a/doc/source/user_guide/index.rst +++ b/doc/source/user_guide/index.rst @@ -11,10 +11,9 @@ User Guide .. toctree:: :maxdepth: 3 - ug_citing.rst ug_implementation.rst ug_running.rst - ug_performance.rst - ug_adding.rst - ug_troubleshooting.rst ug_testing.rst + ug_case_settings.rst + ug_troubleshooting.rst + diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst new file mode 100755 index 000000000..c3280e475 --- /dev/null +++ b/doc/source/user_guide/ug_case_settings.rst @@ -0,0 +1,422 @@ +:tocdepth: 3 + +.. _case_settings: + +Case Settings +===================== + +There are two important files that define the case, **cice.settings** and +**ice_in**. **cice.settings** is a list of env variables that define many +values used to setup, build and run the case. **ice_in** is the input namelist file +for CICE. Variables in both files are described below. + +.. _tabsettings: + +Table of CICE Settings +-------------------------- + +The **cice.settings** file is reasonably well self documented. Several of +the variables defined in the file are not used in CICE. They exist +to support the CICE model. + +.. csv-table:: *CICE settings* + :header: "variable", "options/format", "description", "recommended value" + :widths: 15, 15, 25, 20 + + "ICE_CASENAME", " ", "case name", "set by cice.setup" + "ICE_SANDBOX", " ", "sandbox directory", "set by cice.setup" + "ICE_MACHINE", " ", "machine name", "set by cice.setup" + "ICE_COMPILER", " ", "environment name", "set by cice.setup" + "ICE_MACHCOMP", " ", "machine_environment name", "set by cice.setup" + "ICE_SCRIPTS", " ", "scripts directory", "set by cice.setup" + "ICE_CASEDIR", " ", "case directory", "set by cice.setup" + "ICE_RUNDIR", " ", "run directory", "set by cice.setup" + "ICE_OBJDIR", " ", "compile directory", "${ICE_RUNDIR}/compile" + "ICE_RSTDIR", " ", "unused", "${ICE_RUNDIR}/restart" + "ICE_HSTDIR", " ", "unused", "${ICE_RUNDIR}/history" + "ICE_LOGDIR", " ", "log directory", "${ICE_CASEDIR}/logs" + "ICE_DRVOPT", " ", "unused", "cice" + "ICE_CONSTOPT", " ", "unused", "cice" + "ICE_IOTYPE", " ", "unused", "netcdf" + "ICE_CLEANBUILD", "true,false", "automatically clean before building", "true" + "ICE_GRID", "col", "grid", "set by cice.setup" + "ICE_NXGLOB", "integer", "number of gridcells", "set by cice.setup" + "ICE_NTASKS", "integer", "number of tasks, must be set to 1", "set by cice.setup" + "ICE_NTHRDS", "integer", "number of threads per task, must be set to 1", "set by cice.setup" + "ICE_BLCKX", "integer", "block size in x direction", "set by cice.setup" + "ICE_BLCKY", "integer", "block size in y direction", "set by cice.setup" + "ICE_MXBLCKS", "integer", "maximum number of blocks per task", "set by cice.setup" + "ICE_TEST", " ", "test setting if using a test", "set by cice.setup" + "ICE_TESTNAME", " ", "test name if using a test", "set by cice.setup" + "ICE_BASELINE", " ", "baseline directory name, associated with cice.setup -bdir ", "set by cice.setup" + "ICE_BASEGEN", " ", "baseline directory name for regression generation, associated with cice.setup -bgen ", "set by cice.setup" + "ICE_BASECOM", " ", "baseline directory name for regression comparison, associated with cice.setup -bcmp ", "set by cice.setup" + "ICE_BFBCOMP", " ", "location of case for comparison, associated with cice.setup -td", "set by cice.setup" + "ICE_SPVAL", " ", "special value for cice.settings strings", "set by cice.setup" + "ICE_RUNLENGTH", " ", "batch run length default", "set by cice.setup" + "ICE_ACCOUNT", " ", "batch account number", "set by cice.setup or by default" + "ICE_THREADED", "true,false", "force threading in compile, will always compile threaded if NTHRDS is gt 1", "false" + "NICELYR", "integer", "number of vertical layers in the ice", "7" + "NSNWLYR", "integer", "number of vertical layers in the snow", "1" + "NICECAT", "integer", "number of ice thickness categories", "5" + "TRAGE", "0,1", "ice age tracer", "1" + "TRFY", "0,1", "first-year ice area tracer", "1" + "TRLVL", "0,1", "deformed ice tracer", "1" + "TRPND", "0,1", "melt pond tracer", "1" + "NTRAERO", "integer", "number of aerosol tracers", "1" + "TRBRI", "0,1", "brine height tracer", "0" + "TRZS", "0,1", "zsalinity tracer, needs TRBRI=1", "0" + "TRBGCS", "0,1", "skeletal layer tracer, needs TRBGCZ=0", "0" + "TRBGCZ", "0,1", "zbgc tracers, needs TRBGCS=0 and TRBRI=1", "0" + "NBGCLYR", "integer", "number of zbgc layers", "7" + "TRZAERO", "0-6", "number of z aerosol tracers", "0" + "TRALG", "0,1,2,3", "number of algal tracers", "0" + "TRDOC", "0,1,2,3", "number of dissolved organic carbon", "0" + "TRDIC", "0,1", "number of dissolved inorganic carbon", "0" + "TRDON", "0,1", "number of dissolved organic nitrogen", "0" + "TRFEP", "0,1,2", "number of particulate iron tracers", "0" + "TRFED", "0,1,2", "number of dissolved iron tracers", "0" + "CAM_ICE", " ", "unused", "no" + "DITTO", "no,yes", "turn on bit-for-bit global sums via real16", "no" + "BARRIERS", "no,yes", "turn on barriers between global scatters and gathers", "no" + "ICE_BLDDEBUG", "true,false", "turn on compile debug flags", "false" + "NUMIN", "integer", "smallest unit number assigned to CICE files", "11" + "NUMAX", "integer", "largest unit number assigned to CICE files", "99" + + + +.. _tabnamelist: + + +Table of namelist options +------------------------------- + +.. csv-table:: Table 8 + :header: "variable", "options/format", "description", "recommended value" + :widths: 15, 15, 30, 15 + + "*setup_nml*", "", "", "" + "", "", "*Time, Diagnostics*", "" + "``days_per_year``", "``360`` or ``365``", "number of days in a model year", "365" + "``use_leap_years``", "true/false", "if true, include leap days", "" + "``year_init``", "yyyy", "the initial year, if not using restart", "" + "``istep0``", "integer", "initial time step number", "0" + "``dt``", "seconds", "thermodynamics time step length", "3600." + "``npt``", "integer", "total number of time steps to take", "" + "``ndtd``", "integer", "number of dynamics/advection/ridging/steps per thermo timestep", "1" + "", "", "*Initialization/Restarting*", "" + "``runtype``", "``initial``", "start from ``ice_ic``", "" + "", "``continue``", "restart using ``pointer_file``", "" + "``ice_ic``", "``default``", "latitude and sst dependent", "default" + "", "``none``", "no ice", "" + "", "path/file", "restart file name", "" + "``restart``", "true/false", "initialize using restart file", "``.true.``" + "``use_restart_time``", "true/false", "set initial date using restart file", "``.true.``" + "``restart_format``", "nc", "read/write  restart files (use with PIO)", "" + "", "bin", "read/write binary restart files", "" + "``lcdf64``", "true/false", "if true, use 64-bit  format", "" + "``restart_dir``", "path/", "path to restart directory", "" + "``restart_ext``", "true/false", "read/write halo cells in restart files", "" + "``restart_file``", "filename prefix", "output file for restart dump", "‘iced’" + "``pointer_file``", "pointer filename", "contains restart filename", "" + "``dumpfreq``", "``y``", "write restart every ``dumpfreq_n`` years", "y" + "", "``m``", "write restart every ``dumpfreq_n`` months", "" + "", "``d``", "write restart every ``dumpfreq_n`` days", "" + "``dumpfreq_n``", "integer", "frequency restart data is written", "1" + "``dump_last``", "true/false", "if true, write restart on last time step of simulation", "" + "", "", "*Model Output*", "" + "``bfbflag``", "true/false", "for bit-for-bit diagnostic output", "" + "``diagfreq``", "integer", "frequency of diagnostic output in ``dt``", "24" + "", "*e.g.*, 10", "once every 10 time steps", "" + "``diag_type``", "``stdout``", "write diagnostic output to stdout", "" + "", "``file``", "write diagnostic output to file", "" + "``diag_file``", "filename", "diagnostic output file (script may reset)", "" + "``print_global``", "true/false", "print diagnostic data, global sums", "``.false.``" + "``print_points``", "true/false", "print diagnostic data for two grid points", "``.false.``" + "``latpnt``", "real", "latitude of (2) diagnostic points", "" + "``lonpnt``", "real", "longitude of (2) diagnostic points", "" + "``dbug``", "true/false", "if true, write extra diagnostics", "``.false.``" + "``histfreq``", "string array", "defines output frequencies", "" + "", "``y``", "write history every ``histfreq_n`` years", "" + "", "``m``", "write history every ``histfreq_n`` months", "" + "", "``d``", "write history every ``histfreq_n`` days", "" + "", "``h``", "write history every ``histfreq_n`` hours", "" + "", "``1``", "write history every time step", "" + "", "``x``", "unused frequency stream (not written)", "" + "``histfreq_n``", "integer array", "frequency history output is written", "" + "", "0", "do not write to history", "" + "``hist_avg``", "true", "write time-averaged data", "``.true.``" + "", "false", "write snapshots of data", "" + "``history_dir``", "path/", "path to history output directory", "" + "``history_file``", "filename prefix", "output file for history", "‘iceh’" + "``write_ic``", "true/false", "write initial condition", "" + "``incond_dir``", "path/", "path to initial condition directory", "" + "``incond_file``", "filename prefix", "output file for initial condition", "‘iceh’" + "``runid``", "string", "label for run (currently CESM only)", "" + "", "", "", "" + "*grid_nml*", "", "", "" + "", "", "*Grid*", "" + "``grid_format``", "``nc``", "read  grid and kmt files", "‘bin’" + "", "``bin``", "read direct access, binary file", "" + "``grid_type``", "``rectangular``", "defined in *rectgrid*", "" + "", "``displaced_pole``", "read from file in *popgrid*", "" + "", "``tripole``", "read from file in *popgrid*", "" + "", "``regional``", "read from file in *popgrid*", "" + "``grid_file``", "filename", "name of grid file to be read", "‘grid’" + "``kmt_file``", "filename", "name of land mask file to be read", "‘kmt’" + "``gridcpl_file``", "filename", "input file for coupling grid info", "" + "``kcatbound``", "``0``", "original category boundary formula", "0" + "", "``1``", "new formula with round numbers", "" + "", "``2``", "WMO standard categories", "" + "", "``-1``", "one category", "" + "", "", "", "" + "*domain_nml*", "", "", "" + "", "", "*Domain*", "" + "``nprocs``", "integer", "number of processors to use", "" + "``processor_shape``", "``slenderX1``", "1 processor in the y direction (tall, thin)", "" + "", "``slenderX2``", "2 processors in the y direction (thin)", "" + "", "``square-ice``", "more processors in x than y, :math:`\sim` square", "" + "", "``square-pop``", "more processors in y than x, :math:`\sim` square", "" + "``distribution_type``", "``cartesian``", "distribute blocks in 2D Cartesian array", "" + "", "``roundrobin``", "1 block per proc until blocks are used", "" + "", "``sectcart``", "blocks distributed to domain quadrants", "" + "", "``sectrobin``", "several blocks per proc until used", "" + "", "``rake``", "redistribute blocks among neighbors", "" + "", "``spacecurve``", "distribute blocks via space-filling curves", "" + "``distribution_weight``", "``block``", "full block size sets ``work_per_block``", "" + "", "``latitude``", "latitude/ocean sets ``work_per_block``", "" + "``ew_boundary_type``", "``cyclic``", "periodic boundary conditions in x-direction", "" + "", "``open``", "Dirichlet boundary conditions in x", "" + "``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "" + "", "``open``", "Dirichlet boundary conditions in y", "" + "", "``tripole``", "U-fold tripole boundary conditions in y", "" + "", "``tripoleT``", "T-fold tripole boundary conditions in y", "" + "``maskhalo_dyn``", "true/false", "mask unused halo cells for dynamics", "" + "``maskhalo_remap``", "true/false", "mask unused halo cells for transport", "" + "``maskhalo_bound``", "true/false", "mask unused halo cells for boundary updates", "" + "", "", "", "" + "*tracer_nml*", "", "", "" + "", "", "*Tracers*", "" + "``tr_iage``", "true/false", "ice age", "" + "``restart_age``", "true/false", "restart tracer values from file", "" + "``tr_FY``", "true/false", "first-year ice area", "" + "``restart_FY``", "true/false", "restart tracer values from file", "" + "``tr_lvl``", "true/false", "level ice area and volume", "" + "``restart_lvl``", "true/false", "restart tracer values from file", "" + "``tr_pond_cesm``", "true/false", "CESM melt ponds", "" + "``restart_pond_cesm``", "true/false", "restart tracer values from file", "" + "``tr_pond_topo``", "true/false", "topo melt ponds", "" + "``restart_pond_topo``", "true/false", "restart tracer values from file", "" + "``tr_pond_lvl``", "true/false", "level-ice melt ponds", "" + "``restart_pond_lvl``", "true/false", "restart tracer values from file", "" + "``tr_aero``", "true/false", "aerosols", "" + "``restart_aero``", "true/false", "restart tracer values from file", "" + "*thermo_nml*", "", "", "" + "", "", "*Thermodynamics*", "" + "``kitd``", "``0``", "delta function ITD approximation", "1" + "", "``1``", "linear remapping ITD approximation", "" + "``ktherm``", "``0``", "zero-layer thermodynamic model", "" + "", "``1``", "Bitz and Lipscomb thermodynamic model", "" + "", "``2``", "mushy-layer thermodynamic model", "" + "``conduct``", "``MU71``", "conductivity :cite:`MU71`", "" + "", "``bubbly``", "conductivity :cite:`PETB07`", "" + "``a_rapid_mode``", "real", "brine channel diameter", "0.5x10 :math:`^{-3}` m" + "``Rac_rapid_mode``", "real", "critical Rayleigh number", "10" + "``aspect_rapid_mode``", "real", "brine convection aspect ratio", "1" + "``dSdt_slow_mode``", "real", "drainage strength parameter", "-1.5x10 :math:`^{-7}` m/s/K" + "``phi_c_slow_mode``", ":math:`0<\phi_c < 1`", "critical liquid fraction", "0.05" + "``phi_i_mushy``", ":math:`0<\phi_i < 1`", "solid fraction at lower boundary", "0.85" + "", "", "", "" + "*dynamics_nml*", "", "", "" + "", "", "*Dynamics*", "" + "``kdyn``", "``0``", "dynamics OFF", "1" + "", "``1``", "EVP dynamics", "" + "", "``2``", "EAP dynamics", "" + "``revised_evp``", "true/false", "use revised EVP formulation", "" + "``ndte``", "integer", "number of EVP subcycles", "120" + "``advection``", "``remap``", "linear remapping advection", "‘remap’" + "", "``upwind``", "donor cell advection", "" + "``kstrength``", "``0``", "ice strength formulation :cite:`Hibler79`", "1" + "", "``1``", "ice strength formulation :cite:`Rothrock75`", "" + "``krdg_partic``", "``0``", "old ridging participation function", "1" + "", "``1``", "new ridging participation function", "" + "``krdg_redist``", "``0``", "old ridging redistribution function", "1" + "", "``1``", "new ridging redistribution function", "" + "``mu_rdg``", "real", "e-folding scale of ridged ice", "" + "``Cf``", "real", "ratio of ridging work to PE change in ridging", "17." + "", "", "", "" + "*shortwave_nml*", "", "", "" + "", "", "*Shortwave*", "" + "``shortwave``", "``default``", "NCAR CCSM3 distribution method", "" + "", "``dEdd``", "Delta-Eddington method", "" + "``albedo_type``", "``default``", "NCAR CCSM3 albedos", "‘default’" + "", "``constant``", "four constant albedos", "" + "``albicev``", ":math:`0<\alpha <1`", "visible ice albedo for thicker ice", "" + "``albicei``", ":math:`0<\alpha <1`", "near infrared ice albedo for thicker ice", "" + "``albsnowv``", ":math:`0<\alpha <1`", "visible, cold snow albedo", "" + "``albsnowi``", ":math:`0<\alpha <1`", "near infrared, cold snow albedo", "" + "``ahmax``", "real", "albedo is constant above this thickness", "0.3 m" + "``R_ice``", "real", "tuning parameter for sea ice albedo from Delta-Eddington shortwave", "" + "``R_pnd``", "real", "... for ponded sea ice albedo …", "" + "``R_snw``", "real", "... for snow (broadband albedo) …", "" + "``dT_mlt``", "real", ":math:`\Delta` temperature per :math:`\Delta` snow grain radius", "" + "``rsnw_mlt``", "real", "maximum melting snow grain radius", "" + "``kalg``", "real", "absorption coefficient for algae", "" + "", "", "", "" + "*ponds_nml*", "", "", "" + "", "", "*Melt Ponds*", "" + "``hp1``", "real", "critical ice lid thickness for topo ponds", "0.01 m" + "``hs0``", "real", "snow depth of transition to bare sea ice", "0.03 m" + "``hs1``", "real", "snow depth of transition to pond ice", "0.03 m" + "``dpscale``", "real", "time scale for flushing in permeable ice", ":math:`1\times 10^{-3}`" + "``frzpnd``", "``hlid``", "Stefan refreezing with pond ice thickness", "‘hlid’" + "", "``cesm``", "CESM refreezing empirical formula", "" + "``rfracmin``", ":math:`0 \le r_{min} \le 1`", "minimum melt water added to ponds", "0.15" + "``rfracmax``", ":math:`0 \le r_{max} \le 1`", "maximum melt water added to ponds", "1.0" + "``pndaspect``", "real", "aspect ratio of pond changes (depth:area)", "0.8" + "", "", "", "" + "*zbgc_nml*", "", "", "" + "", "", "*Biogeochemistry*", "" + "``tr_brine``", "true/false", "brine height tracer", "" + "``tr_zaero``", "true/false", "vertical aerosol tracers", "" + "``modal_aero``", "true/false", "modal aersols", "" + "``restore_bgc``", "true/false", "restore bgc to data", "" + "``solve_zsal`", "true/false", "update salinity tracer profile", "" + "``bgc_data_dir``", "path/", "data directory for bgc", "" + "``skl_bgc``", "true/false", "biogeochemistry", "" + "``sil_data_type``", "``default``", "default forcing value for silicate", "" + "", "``clim``", "silicate forcing from ocean climatology :cite:`GLBA06`", "" + "``nit_data_type``", "``default``", "default forcing value for nitrate", "" + "", "``clim``", "nitrate forcing from ocean climatology :cite:`GLBA06`", "" + "", "``sss``", "nitrate forcing equals salinity", "" + "``fe_data_type``", "``default``", "default forcing value for iron", "" + "", "``clim``", "iron forcing from ocean climatology", "" + "``bgc_flux_type``", "``Jin2006``", "ice–ocean flux velocity of :cite:`JDWSTWLG06`", "" + "", "``constant``", "constant ice–ocean flux velocity", "" + "``restart_bgc``", "true/false", "restart tracer values from file", "" + "``tr_bgc_C_sk``", "true/false", "algal carbon tracer", "" + "``tr_bgc_chl_sk``", "true/false", "algal chlorophyll tracer", "" + "``tr_bgc_Am_sk``", "true/false", "ammonium tracer", "" + "``tr_bgc_Sil_sk``", "true/false", "silicate tracer", "" + "``tr_bgc_DMSPp_sk``", "true/false", "particulate DMSP tracer", "" + "``tr_bgc_DMSPd_sk``", "true/false", "dissolved DMSP tracer", "" + "``tr_bgc_DMS_sk``", "true/false", "DMS tracer", "" + "``phi_snow``", "real", "snow porosity for brine height tracer", "" + "", "", "", "" + "*forcing_nml*", "", "", "" + "", "", "*Forcing*", "" + "``formdrag``", "true/false", "calculate form drag", "" + "``atmbndy``", "``default``", "stability-based boundary layer", "‘default’" + "", "``constant``", "bulk transfer coefficients", "" + "``fyear_init``", "yyyy", "first year of atmospheric forcing data", "" + "``ycycle``", "integer", "number of years in forcing data cycle", "" + "``atm_data_format``", "``nc``", "read  atmo forcing files", "" + "", "``bin``", "read direct access, binary files", "" + "``atm_data_type``", "``default``", "constant values defined in the code", "" + "", "``LYq``", "AOMIP/Large-Yeager forcing data", "" + "", "``monthly``", "monthly forcing data", "" + "", "``ncar``", "NCAR bulk forcing data", "" + "", "``oned``", "column forcing data", "" + "``atm_data_dir``", "path/", "path to atmospheric forcing data directory", "" + "``calc_strair``", "true", "calculate wind stress and speed", "" + "", "false", "read wind stress and speed from files", "" + "``highfreq``", "true/false", "high-frequency atmo coupling", "" + "``natmiter``", "integer", "number of atmo boundary layer iterations", "" + "``calc_Tsfc``", "true/false", "calculate surface temperature", "``.true.``" + "``precip_units``", "``mks``", "liquid precipitation data units", "" + "", "``mm_per_month``", "", "" + "", "``mm_per_sec``", "(same as MKS units)", "" + "``tfrz_option``", "``minus1p8``", "constant ocean freezing temperature (:math:`-1.8^{\circ} C`)", "" + "", "``linear_salt``", "linear function of salinity (ktherm=1)", "" + "", "``mushy_layer``", "matches mushy-layer thermo (ktherm=2)", "" + "``ustar_min``", "real", "minimum value of ocean friction velocity", "0.0005 m/s" + "``fbot_xfer_type``", "``constant``", "constant ocean heat transfer coefficient", "" + "", "``Cdn_ocn``", "variable ocean heat transfer coefficient", "" + "``update_ocn_f``", "true", "include frazil water/salt fluxes in ocn fluxes", "" + "", "false", "do not include (when coupling with POP)", "" + "``l_mpond_fresh``", "true", "retain (topo) pond water until ponds drain", "" + "", "false", "release (topo) pond water immediately to ocean", "" + "``oceanmixed_ice``", "true/false", "active ocean mixed layer calculation", "``.true.`` (if uncoupled)" + "``ocn_data_format``", "``nc``", "read  ocean forcing files", "" + "", "``bin``", "read direct access, binary files", "" + "``sss_data_type``", "``default``", "constant values defined in the code", "" + "", "``clim``", "climatological data", "" + "", "``near``", "POP ocean forcing data", "" + "``sst_data_type``", "``default``", "constant values defined in the code", "" + "", "``clim``", "climatological data", "" + "", "``ncar``", "POP ocean forcing data", "" + "``ocn_data_dir``", "path/", "path to oceanic forcing data directory", "" + "``oceanmixed_file``", "filename", "data file containing ocean forcing data", "" + "``restore_sst``", "true/false", "restore sst to data", "" + "``trestore``", "integer", "sst restoring time scale (days)", "" + "``restore_ice``", "true/false", "restore ice state along lateral boundaries", "" + "", "", "", "" + "*icefields_tracer_nml*", "", "", "" + "", "", "*History Fields*", "" + "``f_``", "string", "frequency units for writing ```` to history", "" + "", "``y``", "write history every ``histfreq_n`` years", "" + "", "``m``", "write history every ``histfreq_n`` months", "" + "", "``d``", "write history every ``histfreq_n`` days", "" + "", "``h``", "write history every ``histfreq_n`` hours", "" + "", "``1``", "write history every time step", "" + "", "``x``", "do not write ```` to history", "" + "", "``md``", "*e.g.,* write both monthly and daily files", "" + "``f__ai``", "", "grid cell average of ```` (:math:`\times a_i`)", "" + + + +.. _tuning: + +BGC Tuning Parameters +------------------------ + +Biogeochemical tuning parameters are specified as namelist options in +**ice\_in**. Table :ref:`tab-bio-tracers2` provides a list of parameters +used in the reaction equations, their representation in the code, a +short description of each and the default values. Please keep in mind +that there has only been minimal tuning of the model. + +.. _tab-bio-tracers2: + +.. csv-table:: *Biogeochemical Reaction Parameters* + :header: "Text Variable", "Variable in code", "Description", "Value", "units" + :widths: 7, 20, 15, 15, 15 + + ":math:`f_{graze}`", "fr\_graze(1:3)", "fraction of growth grazed", "0, 0.1, 0.1", "1" + ":math:`f_{res}`", "fr\_resp", "fraction of growth respired", "0.05", "1" + ":math:`l_{max}`", "max\_loss", "maximum tracer loss fraction", "0.9", "1" + ":math:`m_{pre}`", "mort\_pre(1:3)", "maximum mortality rate", "0.007, 0.007, 0.007", "day\ :math:`^{-1}`" + ":math:`m_{T}`", "mort\_Tdep(1:3)", "mortality temperature decay", "0.03, 0.03, 0.03", ":math:`^o`\ C\ :math:`^{-1}`" + ":math:`T_{max}`", "T\_max", "maximum brine temperature", "0", ":math:`^o`\ C" + ":math:`k_{nitr}`", "k\_nitrif", "nitrification rate", "0", "day\ :math:`^{-1}`" + ":math:`f_{ng}`", "fr\_graze\_e", "fraction of grazing excreted", "0.5", "1" + ":math:`f_{gs}`", "fr\_graze\_s", "fraction of grazing spilled", "0.5", "1" + ":math:`f_{nm}`", "fr\_mort2min", "fraction of mortality to :math:`{\mbox{NH$_4$}}`", "0.5", "1" + ":math:`f_{dg}`", "f\_don", "frac. spilled grazing to :math:`{\mbox{DON}}`", "0.6", "1" + ":math:`k_{nb}`", "kn\_bac :math:`^a`", "bacterial degradation of :math:`{\mbox{DON}}`", "0.03", "day\ :math:`^{-1}`" + ":math:`f_{cg}`", "f\_doc(1:3)", "fraction of mortality to :math:`{\mbox{DOC}}`", "0.4, 0.4, 0.2 ", "1" + ":math:`R_{c:n}^c`", "R\_C2N(1:3)", "algal carbon to nitrogen ratio", "7.0, 7.0, 7.0", "mol/mol" + ":math:`k_{cb}`", "k\_bac1:3\ :math:`^a`", "bacterial degradation of DOC", "0.03, 0.03, 0.03", "day\ :math:`^{-1}`" + ":math:`\tau_{fe}`", "t\_iron\_conv", "conversion time pFe :math:`\leftrightarrow` dFe", "3065.0 ", "day" + ":math:`r^{max}_{fed:doc}`", "max\_dfe\_doc1", "max ratio of dFe to saccharids", "0.1852", "nM Fe\ :math:`/\mu`\ M C" + ":math:`f_{fa}`", "fr\_dFe ", "fraction of remin. N to dFe", "0.3", "1" + ":math:`R_{fe:n}`", "R\_Fe2N(1:3)", "algal Fe to N ratio", "0.023, 0.023, 0.7", "mmol/mol" + ":math:`R_{s:n}`", "R\_S2N(1:3)", "algal S to N ratio", "0.03, 0.03, 0.03", "mol/mol" + ":math:`f_{sr}`", "fr\_resp\_s", "resp. loss as DMSPd", "0.75", "1" + ":math:`\tau_{dmsp}`", "t\_sk\_conv", "Stefels rate", "3.0", "day" + ":math:`\tau_{dms}`", "t\_sk\_ox", "DMS oxidation rate", "10.0", "day" + ":math:`y_{dms}`", "y\_sk\_DMS", "yield for DMS conversion", "0.5", "1" + ":math:`K_{{\mbox{NO$_3$}}}`", "K\_Nit(1:3)", ":math:`{\mbox{NO$_3$}}` half saturation constant", "1,1,1", "mmol/m\ :math:`^{3}`" + ":math:`K_{{\mbox{NH$_4$}}}`", "K\_Am(1:3)", ":math:`{\mbox{NH$_4$}}` half saturation constant", "0.3, 0.3, 0.3", "mmol/m\ :math:`^{-3}`" + ":math:`K_{{\mbox{SiO$_3$}}}`", "K\_Sil(1:3)", "silicate half saturation constant", "4.0, 0, 0", "mmol/m\ :math:`^{-3}`" + ":math:`K_{{\mbox{fed}}}`", "K\_Fe(1:3)", "iron half saturation constant", "1.0, 0.2, 0.1", ":math:`\mu`\ mol/m\ :math:`^{-3}`" + ":math:`op_{min}`", "op\_dep\_min", "boundary for light attenuation", "0.1", "1" + ":math:`chlabs`", "chlabs(1:3)", "light absorption length per chla conc.", "0.03, 0.01, 0.05", "1\ :math:`/`\ m\ :math:`/`\ (mg\ :math:`/`\ m\ :math:`^{3}`)" + ":math:`\alpha`", "alpha2max\_low(1:3)", "light limitation factor", "0.25, 0.25, 0.25", "m\ :math:`^2`/W" + ":math:`\beta`", "beta2max(1:3)", "light inhibition factor", "0.018, 0.0025, 0.01", "m\ :math:`^2`/W" + ":math:`\mu_{max}`", "mu\_max(1:3)", "maximum algal growth rate", "1.44, 0.851, 0.851", "day\ :math:`^{-1}`" + ":math:`\mu_T`", "grow\_Tdep(1:3)", "temperature growth factor", "0.06, 0.06, 0.06", "day\ :math:`^{-1}`" + ":math:`f_{sal}`", "fsal", "salinity growth factor", "1", "1" + ":math:`R_{si:n}`", "R\_Si2N(1:3)", "algal silicate to nitrogen", "1.8, 0, 0", "mol/mol" + +:math:`^a` only (1:2) of DOC and DOC parameters have physical meaning diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index 5aac6242d..113794a52 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -1,24 +1,24 @@ :tocdepth: 3 -Numerical implementation +Implementation ======================== CICE is written in FORTRAN90 and runs on platforms using UNIX, LINUX, -and other operating systems. The code is parallelized via grid -decomposition with MPI or OpenMP threads and includes some optimizations +and other operating systems. The code is based on a two-dimensional +horizontal orthogonal grid that is broken into two-dimensional horizontal +blocks and parallelized over blocks +with MPI and OpenMP threads. The code also includes some optimizations for vector architectures. -A second, “external” layer of parallelization involves message passing -between CICE and the flux coupler, which may be running on different -processors in a distributed system. The parallelization scheme for CICE -was designed so that MPI could be used for the coupling along with MPI, -OpenMP or no parallelization internally. The internal parallelization -method is set at compile time with the `NTASK` and `THRD` definitions in the -compile script. Message passing between the ice model and the CESM flux -coupler is accomplished with MPI, regardless of the type of internal -parallelization used for CICE, although the ice model may be coupled to -another system without using MPI. +CICE consists of source code under the **cicecore/** directory that supports +model dynamics and top-level control. The column physics source code is +under the Icepack directory and this is implemented as a submodule in +github from a separate repository (`CICE `) +There is also a **configuration/** directory that includes scripts +for configuring CICE cases. + +.. _coupling: .. _dirstructure: @@ -26,324 +26,51 @@ another system without using MPI. Directory structure ~~~~~~~~~~~~~~~~~~~ -The present code distribution includes make files, several scripts and -some input files. The main directory is **cice/**, and a run directory -(**rundir/**) is created upon initial execution of the script -**comp\_ice**. One year of atmospheric forcing data is also available -from the code distribution web site (see the **README** file for -details). - -basic information - -**bld/** makefiles - -**Macros.**\ :math:`\langle`\ OS\ :math:`\rangle`.\ :math:`\langle`\ SITE\ :math:`\rangle`.\ :math:`\langle`\ machine\ :math:`\rangle` - macro definitions for the given operating system, used by - **Makefile**.\ :math:`\langle` \ OS\ :math:`\rangle` - -**Makefile.**\ :math:`\langle`\ OS\ :math:`\rangle` - primary makefile for the given operating system - (**:math:`\langle`\ std\ :math:`\rangle`** works for most systems) - -**makedep.c** - perl script that determines module dependencies - -script that sets up the run directory and compiles the code - -modules based on “shared" code in CESM - -**shr\_orb\_mod.F90** - orbital parameterizations - -documentation - -**cicedoc.pdf** - this document - -**PDF/** - PDF documents of numerous publications related to CICE - -institution-specific modules - -**cice/** - official driver for CICE v5 (LANL) - - **CICE.F90** - main program - - **CICE\_FinalMod.F90** - routines for finishing and exiting a run - - **CICE\_InitMod.F90** - routines for initializing a run - - **CICE\_RunMod.F90** - main driver routines for time stepping - - **CICE\_RunMod.F90\_debug** - debugging version of **CICE\_RunMod.F90** - - **ice\_constants.F90** - physical and numerical constants and parameters - -sample diagnostic output files - -input files that may be modified for other CICE configurations - -**col/** - column configuration files - - **ice\_in** - namelist input data (data paths depend on particular system) - -**gx1/** - :math:`\left<1^\circ\right>` displaced pole grid files - - **global\_gx1.grid** - :math:`\left<1^\circ\right>` displaced pole grid (binary) - - **global\_gx1.kmt** - :math:`\left<1^\circ\right>` land mask (binary) - - **ice.restart\_file** - pointer for restart file name - - **ice\_in** - namelist input data (data paths depend on particular system) - - **ice\_in\_v4.1** - namelist input data for default CICE v4.1 configuration - - **iced\_gx1\_v5.nc** -  restart file used for initial condition - -**gx3/** - :math:`\left<3^\circ\right>` displaced pole grid files - - **global\_gx3.grid** - :math:`\left<3^\circ\right>` displaced pole grid (binary) - - **global\_gx3.kmt** - :math:`\left<3^\circ\right>` land mask (binary) - - **global\_gx3.grid.nc** - :math:`\left<3^\circ\right>` displaced pole grid () - - **global\_gx3.kmt.nc** - :math:`\left<3^\circ\right>` land mask () - - **ice.restart\_file** - pointer for restart file name - - **ice\_in** - namelist input data (data paths depend on particular system) - - **iced\_gx3\_v5.nc** -  restart file used for initial condition - -convert\_restarts.f90 - Fortran code to convert restart files from v4.1 to v5 (4 ice layers) - -**run\_ice.**\ :math:`\langle`\ OS\ :math:`\rangle`.\ :math:`\langle`\ SITE\ :math:`\rangle`.\ :math:`\langle`\ machine\ :math:`\rangle` - sample script for running on the given operating system - -binary history and restart modules - -**ice\_history\_write.F90** - subroutines with binary output - -**ice\_restart.F90** - read/write binary restart files - - history and restart modules - -**ice\_history\_write.F90** - subroutines with  output - -**ice\_restart.F90** - read/write   restart files - -parallel I/O history and restart modules - -**ice\_history\_write.F90** - subroutines with   output using PIO - -**ice\_pio.F90** - subroutines specific to PIO - -**ice\_restart.F90** - read/write  restart files using PIO - -modules that require MPI calls - -**ice\_boundary.F90** - boundary conditions - -**ice\_broadcast.F90** - routines for broadcasting data across processors - -**ice\_communicate.F90** - routines for communicating between processors - -**ice\_exit.F90** - aborts or exits the run - -**ice\_gather\_scatter.F90** - gathers/scatters data to/from one processor from/to all processors - -**ice\_global\_reductions.F90** - global sums, minvals, maxvals, etc., across processors - -**ice\_timers.F90** - timing routines - -same modules as in **mpi/** but without MPI calls - -general CICE source code +The present code distribution includes source code and scripts. Forcing +data is available from the ftp site. The directory structure of CICE is +as follows -handles most work associated with the aerosol tracers +**LICENSE.pdf** + license and policy for using and sharing the code -handles most work associated with the age tracer +**DistributionPolicy.pdf** + license and policy for using and sharing the code -skeletal layer biogeochemistry +**README.md** + basic information and pointers -stability-based parameterization for calculation of turbulent -ice–atmosphere fluxes +**icepack/** + subdirectory for the Icepack model. The Icepack subdirectory includes Icepack specific scripts, drivers, and documentation. CICE only uses the columnphysics source code under **icepack/columnphysics/**. -for decomposing global domain into blocks +**cicecore/** + directory for CICE source code. -evolves the brine height tracer +**cicecore/cicedynB/** + directory for routines associated with the dynamics core. -keeps track of what time it is +**cicecore/driver/** + directory for top level CICE drivers and coupling layers. -miscellaneous diagnostic and debugging routines +**cicecore/shared/** + directory for CICE source code that is independent of the dynamical core. -for distributing blocks across processors +**cicecore/version.txt** + file that indicates the CICE model version. -decompositions, distributions and related parallel processing info +**configuration/scripts/** + directory of support scripts, see :ref:`dev_scripts` -domain and block sizes +**doc/** + documentation -elastic-anisotropic-plastic dynamics component +**cice.setup** + main CICE script for creating cases -elastic-viscous-plastic dynamics component +A case (compile) directory is created upon initial execution of the script +**icepack.setup** at the user-specified location provided after the -c flag. +Executing the command ``./icepack.setup -h`` provides helpful information for +this tool. -code shared by EVP and EAP dynamics - -unit numbers for I/O - -handles most work associated with the first-year ice area tracer - -fluxes needed/produced by the model - -routines to read and interpolate forcing data for stand-alone ice model -runs - -grid and land masks - -initialization and accumulation of history output variables - -history output of biogeochemistry variables - -history output of form drag variables - -history output of ridging variables - -history output of melt pond variables - -code shared by all history modules - -namelist and initializations - -utilities for managing ice thickness distribution - -basic definitions of reals, integers, etc. - -handles most work associated with the level ice area and volume tracers - -mechanical redistribution component (ridging) - -CESM melt pond parameterization - -level-ice melt pond parameterization - -topo melt pond parameterization - -mixed layer ocean model - -orbital parameters for Delta-Eddington shortwave parameterization - -utilities for reading and writing files - -driver for reading/writing restart files - -code shared by all restart options - -basic restoring for open boundary conditions - -shortwave and albedo parameterizations - -space-filling-curves distribution method - -essential arrays to describe the state of the ice - -routines for time stepping the major code components - -zero-layer thermodynamics of :cite:`Semtner76` - -multilayer thermodynamics of :cite:`BL99` - -thermodynamic changes mostly related to ice thickness distribution - -mushy-theory thermodynamics of:cite:`THB13` - -code shared by all thermodynamics parameterizations - -vertical growth rates and fluxes - -driver for horizontal advection - -horizontal advection via incremental remapping - -driver for ice biogeochemistry and brine tracer motion - -parameters and shared code for biogeochemistry and brine height - -execution or “run” directory created when the code is compiled using the -**comp\_ice** script (gx3) - -**cice** - code executable - -**compile/** - directory containing object files, etc. - -**grid** - horizontal grid file from **cice/input\_templates/gx3/** - -**ice.log.[ID]** - diagnostic output file - -**ice\_in** - namelist input data from **cice/input\_templates/gx3/** - -**history/iceh.[timeID].nc** - output history file - -**kmt** - land mask file from **cice/input\_templates/gx3/** - -**restart/** - restart directory - - **iced\_gx3\_v5.nc** - initial condition from **cice/input\_templates/gx3/** - - **ice.restart\_file** - restart pointer from **cice/input\_templates/gx3/** - -**run\_ice** - batch run script file from **cice/input\_templates/** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grid, boundary conditions and masks @@ -640,9 +367,158 @@ or southern hemispheres, respectively. Special constants (`spval` and `spval\_dbl`, each equal to :math:`10^{30}`) are used to indicate land points in the history files and diagnostics. -~~~~~~~~~~~~~~~~~~~ -Test configurations -~~~~~~~~~~~~~~~~~~~ + +.. _performance: + +*************** +Performance +*************** + +Namelist options (*domain\_nml*) provide considerable flexibility for +finding the most efficient processor and block configuration. Some of +these choices are illustration in :ref:`fig-distrb`. `processor\_shape` +chooses between tall, thin processor domains (`slenderX1` or `slenderX2`, +often better for sea ice simulations on global grids where nearly all of +the work is at the top and bottom of the grid with little to do in +between) and close-to-square domains, which maximize the volume to +surface ratio (and therefore on-processor computations to message +passing, if there were ice in every grid cell). In cases where the +number of processors is not a perfect square (4, 9, 16...), the +`processor\_shape` namelist variable allows the user to choose how the +processors are arranged. Here again, it is better in the sea ice model +to have more processors in x than in y, for example, 8 processors +arranged 4x2 (`square-ice`) rather than 2x4 (`square-pop`). The latter +option is offered for direct-communication compatibility with POP, in +which this is the default. + +The user provides the total number of processors and the block +dimensions in the setup script (**comp\_ice**). When moving toward +smaller, more numerous blocks, there is a point where the code becomes +less efficient; blocks should not have fewer than about 20 grid cells in +each direction. Squarish blocks optimize the volume-to-surface ratio for +communications. + +.. _fig-distrb: + +.. figure:: ./figures/distrb.png + :scale: 50% + + Figure 9 + +:ref:`fig-distrb` : Distribution of 256 blocks across 16 processors, +represented by colors, on the gx1 grid: (a) cartesian, slenderX1, (b) +cartesian, slenderX2, (c) cartesian, square-ice (square-pop is +equivalent here), (d) rake with block weighting, (e) rake with +latitude weighting, (f) spacecurve. Each block consists of 20x24 grid +cells, and white blocks consist entirely of land cells. + +The `distribution\_type` options allow standard Cartesian distribution of +blocks, redistribution via a ‘rake’ algorithm for improved load +balancing across processors, and redistribution based on space-filling +curves. There are also three additional distribution types +(‘roundrobin,’ ‘sectrobin,’ ‘sectcart’) that improve land-block +elimination rates and also allow more flexibility in the number of +processors used. The rake and space-filling curve algorithms are +primarily helpful when using squarish processor domains where some +processors (located near the equator) would otherwise have little work +to do. Processor domains need not be rectangular, however. + +`distribution\_wght` chooses how the work-per-block estimates are +weighted. The ‘block’ option is the default in POP, which uses a lot of +array syntax requiring calculations over entire blocks (whether or not +land is present), and is provided here for direct-communication +compatibility with POP. The ‘latitude’ option weights the blocks based +on latitude and the number of ocean grid cells they contain. + +The rake distribution type is initialized as a standard, Cartesian +distribution. Using the work-per-block estimates, blocks are “raked" +onto neighboring processors as needed to improve load balancing +characteristics among processors, first in the x direction and then in +y. + +Space-filling curves reduce a multi-dimensional space (2D, in our case) +to one dimension. The curve is composed of a string of blocks that is +snipped into sections, again based on the work per processor, and each +piece is placed on a processor for optimal load balancing. This option +requires that the block size be chosen such that the number of blocks in +the x direction equals the number of blocks in the y direction, and that +number must be factorable as :math:`2^n 3^m 5^p` where :math:`n, m, p` +are integers. For example, a 16x16 array of blocks, each containing +20x24 grid cells, fills the gx1 grid (:math:`n=4, m=p=0`). If either of +these conditions is not met, a Cartesian distribution is used instead. + +While the Cartesian distribution groups sets of blocks by processor, the +‘roundrobin’ distribution loops through the blocks and processors +together, putting one block on each processor until the blocks are gone. +This provides good load balancing but poor communication characteristics +due to the number of neighbors and the amount of data needed to +communicate. The ‘sectrobin’ and ‘sectcart’ algorithms loop similarly, +but put groups of blocks on each processor to improve the communication +characteristics. In the ‘sectcart’ case, the domain is divided into two +(east-west) halves and the loops are done over each, sequentially. +:ref:`fig-distribscorecard` provides an overview of the pros and cons +for the distribution types. + +.. _fig-distribscorecard: + +.. figure:: ./figures/scorecard.png + :scale: 20% + + Figure 10 + +:ref:`fig-distribscorecard` : Scorecard for block distribution choices in +CICE, courtesy T. Craig. For more information, see +http://www.cesm.ucar.edu/events/ws.2012/Presentations/SEWG2/craig.pdf + +The `maskhalo` options in the namelist improve performance by removing +unnecessary halo communications where there is no ice. There is some +overhead in setting up the halo masks, which is done during the +timestepping procedure as the ice area changes, but this option +usually improves timings even for relatively small processor counts. +T. Craig has found that performance improved by more than 20% for +combinations of updated decompositions and masked haloes, in CESM’s +version of CICE. A practical guide for choosing a CICE grid +decomposition, based on experience in CESM, is available: +http://oceans11.lanl.gov/drupal/CICE/DecompositionGuide + +Throughout the code, (i, j) loops have been combined into a single loop, +often over just ocean cells or those containing sea ice. This was done +to reduce unnecessary operations and to improve vector performance. + +:ref:`fig-timings` illustrates the computational expense of various +options, relative to the total time (excluding initialization) of a +7-layer configuration using BL99 thermodynamics, EVP dynamics, and the +‘ccsm3’ shortwave parameterization on the gx1 grid, run for one year +from a no-ice initial condition. The block distribution consisted of +20 \ :math:`\times` 192 blocks spread over 32 processors (‘slenderX2’) +with no threads and -O2 optimization. Timings varied by about +:math:`\pm3`\ % in identically configured runs due to machine load. +Extra time required for tracers has two components, that needed to carry +the tracer itself (advection, category conversions) and that needed for +the calculations associated with the particular tracer. The age tracers +(FY and iage) require very little extra calculation, so their timings +represent essentially the time needed just to carry an extra tracer. The +topo melt pond scheme is slightly faster than the others because it +calculates pond area and volume once per grid cell, while the others +calculate it for each thickness category. + +.. _fig-timings: + +.. figure:: ./figures/histograms.png + :scale: 20% + + Figure 11 + +:ref:`fig-timings` : Change in ‘TimeLoop’ timings from the 7-layer +configuration using BL99 thermodynamics and EVP dynamics. Timings +were made on a nondedicated machine, with variations of about +:math:`\pm3`\ % in identically configured runs (light grey). Darker +grey indicates the time needed for extra required options; The +Delta-Eddington radiation scheme is required for all melt pond +schemes and the aerosol tracers, and the level-ice pond +parameterization additionally requires the level-ice tracers. + + .. _init: diff --git a/doc/source/user_guide/ug_performance.rst b/doc/source/user_guide/ug_performance.rst deleted file mode 100644 index df76dbfee..000000000 --- a/doc/source/user_guide/ug_performance.rst +++ /dev/null @@ -1,2694 +0,0 @@ -:tocdepth: 3 - -User Guide -========== - ------------------------- -Numerical implementation ------------------------- - -CICE is written in FORTRAN90 and runs on platforms using UNIX, LINUX, -and other operating systems. The code is parallelized via grid -decomposition with MPI or OpenMP threads and includes some optimizations -for vector architectures. - -A second, “external” layer of parallelization involves message passing -between CICE and the flux coupler, which may be running on different -processors in a distributed system. The parallelization scheme for CICE -was designed so that MPI could be used for the coupling along with MPI, -OpenMP or no parallelization internally. The internal parallelization -method is set at compile time with the `NTASK` and `THRD` definitions in the -compile script. Message passing between the ice model and the CESM flux -coupler is accomplished with MPI, regardless of the type of internal -parallelization used for CICE, although the ice model may be coupled to -another system without using MPI. - -.. _dirstructure: - -~~~~~~~~~~~~~~~~~~~ -Directory structure -~~~~~~~~~~~~~~~~~~~ - -The present code distribution includes make files, several scripts and -some input files. The main directory is **cice/**, and a run directory -(**rundir/**) is created upon initial execution of the script -**comp\_ice**. One year of atmospheric forcing data is also available -from the code distribution web site (see the **README** file for -details). - -basic information - -**bld/** makefiles - -**Macros.**\ :math:`\langle`\ OS\ :math:`\rangle`.\ :math:`\langle`\ SITE\ :math:`\rangle`.\ :math:`\langle`\ machine\ :math:`\rangle` - macro definitions for the given operating system, used by - **Makefile**.\ :math:`\langle` \ OS\ :math:`\rangle` - -**Makefile.**\ :math:`\langle`\ OS\ :math:`\rangle` - primary makefile for the given operating system - (**:math:`\langle`\ std\ :math:`\rangle`** works for most systems) - -**makedep.c** - perl script that determines module dependencies - -script that sets up the run directory and compiles the code - -modules based on “shared" code in CESM - -**shr\_orb\_mod.F90** - orbital parameterizations - -documentation - -**cicedoc.pdf** - this document - -**PDF/** - PDF documents of numerous publications related to CICE - -institution-specific modules - -**cice/** - official driver for CICE v5 (LANL) - - **CICE.F90** - main program - - **CICE\_FinalMod.F90** - routines for finishing and exiting a run - - **CICE\_InitMod.F90** - routines for initializing a run - - **CICE\_RunMod.F90** - main driver routines for time stepping - - **CICE\_RunMod.F90\_debug** - debugging version of **CICE\_RunMod.F90** - - **ice\_constants.F90** - physical and numerical constants and parameters - -sample diagnostic output files - -input files that may be modified for other CICE configurations - -**col/** - column configuration files - - **ice\_in** - namelist input data (data paths depend on particular system) - -**gx1/** - :math:`\left<1^\circ\right>` displaced pole grid files - - **global\_gx1.grid** - :math:`\left<1^\circ\right>` displaced pole grid (binary) - - **global\_gx1.kmt** - :math:`\left<1^\circ\right>` land mask (binary) - - **ice.restart\_file** - pointer for restart file name - - **ice\_in** - namelist input data (data paths depend on particular system) - - **ice\_in\_v4.1** - namelist input data for default CICE v4.1 configuration - - **iced\_gx1\_v5.nc** -  restart file used for initial condition - -**gx3/** - :math:`\left<3^\circ\right>` displaced pole grid files - - **global\_gx3.grid** - :math:`\left<3^\circ\right>` displaced pole grid (binary) - - **global\_gx3.kmt** - :math:`\left<3^\circ\right>` land mask (binary) - - **global\_gx3.grid.nc** - :math:`\left<3^\circ\right>` displaced pole grid () - - **global\_gx3.kmt.nc** - :math:`\left<3^\circ\right>` land mask () - - **ice.restart\_file** - pointer for restart file name - - **ice\_in** - namelist input data (data paths depend on particular system) - - **iced\_gx3\_v5.nc** -  restart file used for initial condition - -convert\_restarts.f90 - Fortran code to convert restart files from v4.1 to v5 (4 ice layers) - -**run\_ice.**\ :math:`\langle`\ OS\ :math:`\rangle`.\ :math:`\langle`\ SITE\ :math:`\rangle`.\ :math:`\langle`\ machine\ :math:`\rangle` - sample script for running on the given operating system - -binary history and restart modules - -**ice\_history\_write.F90** - subroutines with binary output - -**ice\_restart.F90** - read/write binary restart files - - history and restart modules - -**ice\_history\_write.F90** - subroutines with  output - -**ice\_restart.F90** - read/write   restart files - -parallel I/O history and restart modules - -**ice\_history\_write.F90** - subroutines with   output using PIO - -**ice\_pio.F90** - subroutines specific to PIO - -**ice\_restart.F90** - read/write  restart files using PIO - -modules that require MPI calls - -**ice\_boundary.F90** - boundary conditions - -**ice\_broadcast.F90** - routines for broadcasting data across processors - -**ice\_communicate.F90** - routines for communicating between processors - -**ice\_exit.F90** - aborts or exits the run - -**ice\_gather\_scatter.F90** - gathers/scatters data to/from one processor from/to all processors - -**ice\_global\_reductions.F90** - global sums, minvals, maxvals, etc., across processors - -**ice\_timers.F90** - timing routines - -same modules as in **mpi/** but without MPI calls - -general CICE source code - -handles most work associated with the aerosol tracers - -handles most work associated with the age tracer - -skeletal layer biogeochemistry - -stability-based parameterization for calculation of turbulent -ice–atmosphere fluxes - -for decomposing global domain into blocks - -evolves the brine height tracer - -keeps track of what time it is - -miscellaneous diagnostic and debugging routines - -for distributing blocks across processors - -decompositions, distributions and related parallel processing info - -domain and block sizes - -elastic-anisotropic-plastic dynamics component - -elastic-viscous-plastic dynamics component - -code shared by EVP and EAP dynamics - -unit numbers for I/O - -handles most work associated with the first-year ice area tracer - -fluxes needed/produced by the model - -routines to read and interpolate forcing data for stand-alone ice model -runs - -grid and land masks - -initialization and accumulation of history output variables - -history output of biogeochemistry variables - -history output of form drag variables - -history output of ridging variables - -history output of melt pond variables - -code shared by all history modules - -namelist and initializations - -utilities for managing ice thickness distribution - -basic definitions of reals, integers, etc. - -handles most work associated with the level ice area and volume tracers - -mechanical redistribution component (ridging) - -CESM melt pond parameterization - -level-ice melt pond parameterization - -topo melt pond parameterization - -mixed layer ocean model - -orbital parameters for Delta-Eddington shortwave parameterization - -utilities for reading and writing files - -driver for reading/writing restart files - -code shared by all restart options - -basic restoring for open boundary conditions - -shortwave and albedo parameterizations - -space-filling-curves distribution method - -essential arrays to describe the state of the ice - -routines for time stepping the major code components - -zero-layer thermodynamics of :cite:`Semtner76` - -multilayer thermodynamics of :cite:`BL99` - -thermodynamic changes mostly related to ice thickness distribution - -mushy-theory thermodynamics of:cite:`THB13` - -code shared by all thermodynamics parameterizations - -vertical growth rates and fluxes - -driver for horizontal advection - -horizontal advection via incremental remapping - -driver for ice biogeochemistry and brine tracer motion - -parameters and shared code for biogeochemistry and brine height - -execution or “run” directory created when the code is compiled using the -**comp\_ice** script (gx3) - -**cice** - code executable - -**compile/** - directory containing object files, etc. - -**grid** - horizontal grid file from **cice/input\_templates/gx3/** - -**ice.log.[ID]** - diagnostic output file - -**ice\_in** - namelist input data from **cice/input\_templates/gx3/** - -**history/iceh.[timeID].nc** - output history file - -**kmt** - land mask file from **cice/input\_templates/gx3/** - -**restart/** - restart directory - - **iced\_gx3\_v5.nc** - initial condition from **cice/input\_templates/gx3/** - - **ice.restart\_file** - restart pointer from **cice/input\_templates/gx3/** - -**run\_ice** - batch run script file from **cice/input\_templates/** - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Grid, boundary conditions and masks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The spatial discretization is specialized for a generalized orthogonal -B-grid as in :cite:`Murray96` or -:cite:`SKM95`. The ice and snow area, volume and energy are -given at the center of the cell, velocity is defined at the corners, and -the internal ice stress tensor takes four different values within a grid -cell; bilinear approximations are used for the stress tensor and the ice -velocity across the cell, as described in :cite:`HD02`. -This tends to avoid the grid decoupling problems associated with the -B-grid. EVP is available on the C-grid through the MITgcm code -distribution, http://mitgcm.org/viewvc/MITgcm/MITgcm/pkg/seaice/. - -Since ice thickness and thermodynamic variables such as temperature are given -in the center of each cell, the grid cells are referred to as “T cells.” -We also occasionally refer to “U cells,” which are centered on the -northeast corner of the corresponding T cells and have velocity in the -center of each. The velocity components are aligned along grid lines. - -The user has several choices of grid routines: *popgrid* reads grid -lengths and other parameters for a nonuniform grid (including tripole -and regional grids), and *rectgrid* creates a regular rectangular grid, -including that used for the column configuration. The input files -**global\_gx3.grid** and **global\_gx3.kmt** contain the -:math:`\left<3^\circ\right>` POP grid and land mask; -**global\_gx1.grid** and **global\_gx1.kmt** contain the -:math:`\left<1^\circ\right>` grid and land mask. These are binary -unformatted, direct access files produced on an SGI (Big Endian). If you -are using an incompatible (Little Endian) architecture, choose -`rectangular` instead of `displaced\_pole` in **ice\_in**, or follow -procedures as for conejo -(:math:`\langle`\ **OS**\ :math:`\rangle.\langle`\ **SITE**\ :math:`\rangle.\langle`\ **machine**\ :math:`\rangle` -= Linux.LANL.conejo). There are versions of the gx3 grid files -available. - -In CESM, the sea ice model may exchange coupling fluxes using a -different grid than the computational grid. This functionality is -activated using the namelist variable `gridcpl\_file`. - -*********************** -Grid domains and blocks -*********************** - -In general, the global gridded domain is -`nx\_global` :math:`\times`\ `ny\_global`, while the subdomains used in the -block distribution are `nx\_block` :math:`\times`\ `ny\_block`. The -physical portion of a subdomain is indexed as [`ilo:ihi`, `jlo:jhi`], with -nghost “ghost” or “halo" cells outside the domain used for boundary -conditions. These parameters are illustrated in :ref:`fig-grid` in one -dimension. The routines *global\_scatter* and *global\_gather* -distribute information from the global domain to the local domains and -back, respectively. If MPI is not being used for grid decomposition in -the ice model, these routines simply adjust the indexing on the global -domain to the single, local domain index coordinates. Although we -recommend that the user choose the local domains so that the global -domain is evenly divided, if this is not possible then the furthest east -and/or north blocks will contain nonphysical points (“padding”). These -points are excluded from the computation domain and have little effect -on model performance. - -.. _fig-grid: - -.. figure:: ./figures/grid.png - :scale: 20% - - Figure 8 - -:ref:`fig-grid` : Grid parameters for a sample one-dimensional, 20-cell -global domain decomposed into four local subdomains. Each local -domain has one ghost (halo) cell on each side, and the physical -portion of the local domains are labeled `ilo:ihi`. The parameter -`nx\_block` is the total number of cells in the local domain, including -ghost cells, and the same numbering system is applied to each of the -four subdomains. - -The user chooses a block size `BLCKX` :math:`\times`\ `BLCKY` and the -number of processors `NTASK` in **comp\_ice**. Parameters in the -*domain\_nml* namelist in **ice\_in** determine how the blocks are -distributed across the processors, and how the processors are -distributed across the grid domain. Recommended combinations of these -parameters for best performance are given in Section :ref:`performance`. -The script **comp\_ice** computes the maximum number of blocks on each -processor for typical Cartesian distributions, but for non-Cartesian -cases `MXBLCKS` may need to be set in the script. The code will print this -information to the log file before aborting, and the user will need to -adjust `MXBLCKS` in **comp\_ice** and recompile. The code will also print -a warning if the maximum number of blocks is too large. Although this is -not fatal, it does require excess memory. - -A loop at the end of routine *create\_blocks* in module -**ice\_blocks.F90** will print the locations for all of the blocks on -the global grid if dbug is set to be true. Likewise, a similar loop at -the end of routine *create\_local\_block\_ids* in module -**ice\_distribution.F90** will print the processor and local block -number for each block. With this information, the grid decomposition -into processors and blocks can be ascertained. The dbug flag must be -manually set in the code in each case (independently of the dbug flag in -**ice\_in**), as there may be hundreds or thousands of blocks to print -and this information should be needed only rarely. This information is -much easier to look at using a debugger such as Totalview. - -Alternatively, a new variable is provided in the history files, `blkmask`, -which labels the blocks in the grid decomposition according to `blkmask` = -`my\_task` + `iblk/100`. - -************* -Tripole grids -************* - -The tripole grid is a device for constructing a global grid with a -normal south pole and southern boundary condition, which avoids placing -a physical boundary or grid singularity in the Arctic Ocean. Instead of -a single north pole, it has two “poles” in the north, both located on -land, with a line of grid points between them. This line of points is -called the “fold,” and it is the “top row” of the physical grid. One -pole is at the left-hand end of the top row, and the other is in the -middle of the row. The grid is constructed by “folding” the top row, so -that the left-hand half and the right-hand half of it coincide. Two -choices for constructing the tripole grid are available. The one first -introduced to CICE is called “U-fold”, which means that the poles and -the grid cells between them are U cells on the grid. Alternatively the -poles and the cells between them can be grid T cells, making a “T-fold.” -Both of these options are also supported by the OPA/NEMO ocean model, -which calls the U-fold an “f-fold” (because it uses the Arakawa C-grid -in which U cells are on T-rows). The choice of tripole grid is given by -the namelist variable `ns\_boundary\_type`, ‘tripole’ for the U-fold and -‘tripoleT’ for the T-fold grid. - -In the U-fold tripole grid, the poles have U-index -:math:`{\tt nx\_global}/2` and `nx\_global` on the top U-row of the -physical grid, and points with U-index i and :math:`{\tt nx\_global-i}` -are coincident. Let the fold have U-row index :math:`n` on the global -grid; this will also be the T-row index of the T-row to the south of the -fold. There are ghost (halo) T- and U-rows to the north, beyond the -fold, on the logical grid. The point with index i along the ghost T-row -of index :math:`n+1` physically coincides with point -:math:`{\tt nx\_global}-{\tt i}+1` on the T-row of index :math:`n`. The -ghost U-row of index :math:`n+1` physically coincides with the U-row of -index :math:`n-1`. - -In the T-fold tripole grid, the poles have T-index 1 and and -:math:`{\tt nx\_global}/2+1` on the top T-row of the physical grid, and -points with T-index i and :math:`{\tt nx\_global}-{\tt i}+2` are -coincident. Let the fold have T-row index :math:`n` on the global grid. -It is usual for the northernmost row of the physical domain to be a -U-row, but in the case of the T-fold, the U-row of index :math:`n` is -“beyond” the fold; although it is not a ghost row, it is not physically -independent, because it coincides with U-row :math:`n-1`, and it -therefore has to be treated like a ghost row. Points i on U-row -:math:`n` coincides with :math:`{\tt nx\_global}-{\tt i}+1` on U-row -:math:`n-1`. There are still ghost T- and U-rows :math:`n+1` to the -north of U-row :math:`n`. Ghost T-row :math:`n+1` coincides with T-row -:math:`n-1`, and ghost U-row :math:`n+1` coincides with U-row -:math:`n-2`. - -The tripole grid thus requires two special kinds of treatment for -certain rows, arranged by the halo-update routines. First, within rows -along the fold, coincident points must always have the same value. This -is achieved by averaging them in pairs. Second, values for ghost rows -and the “quasi-ghost” U-row on the T-fold grid are reflected copies of -the coincident physical rows. Both operations involve the tripole -buffer, which is used to assemble the data for the affected rows. -Special treatment is also required in the scattering routine, and when -computing global sums one of each pair of coincident points has to be -excluded. - -.. _bio-grid: - -******** -Bio-grid -******** - -The bio-grid is a vertical grid used for solving the brine height -variable :math:`h_b`. In the future, it will also be used for -discretizing the vertical transport equations of biogeochemical tracers. -The bio-grid is a non-dimensional vertical grid which takes the value -zero at :math:`h_b` and one at the ice–ocean interface. The number of -grid levels is specified during compilation in **comp\_ice** by setting -the variable `NBGCLYR` equal to an integer (:math:`n_b`) . - -Ice tracers and microstructural properties defined on the bio-grid are -referenced in two ways: as `bgrid` :math:`=n_b+2` points and as -igrid\ :math:`=n_b+1` points. For both bgrid and igrid, the first and -last points reference :math:`h_b` and the ice–ocean interface, -respectively, and so take the values :math:`0` and :math:`1`, -respectively. For bgrid, the interior points :math:`[2, n_b+1]` are -spaced at :math:`1/n_b` intervals beginning with `bgrid(2)` :math:` = -1/(2n_b)`. The `igrid` interior points :math:`[2, n_b]` are also -equidistant with the same spacing, but physically coincide with points -midway between those of `bgrid`. - -******************** -Column configuration -******************** - -A column modeling capability is available. Because of the boundary -conditions and other spatial assumptions in the model, this is not a -single column, but a small array of columns (minimum grid size is 5x5). -However, the code is set up so that only the single, central column is -used (all other columns are designated as land). The column is located -near Barrow (71.35N, 156.5W). Options for choosing the column -configuration are given in **comp\_ice** (choose `RES col`) and in the -namelist file, **input\_templates/col/ice\_in**. Here, `istep0` and the -initial conditions are set such that the run begins September 1 with no -ice. The grid type is rectangular, dynamics are turned off (`kdyn` = 0) and -one processor is used. - -History variables available for column output are ice and snow -temperature, `Tinz` and `Tsnz`. These variables also include thickness -category as a fourth dimension. - -******************* -Boundary conditions -******************* - -Much of the infrastructure used in CICE, including the boundary -routines, is adopted from POP. The boundary routines perform boundary -communications among processors when MPI is in use and among blocks -whenever there is more than one block per processor. - -Open/cyclic boundary conditions are the default in CICE; the physical -domain can still be closed using the land mask. In our bipolar, -displaced-pole grids, one row of grid cells along the north and south -boundaries is located on land, and along east/west domain boundaries not -masked by land, periodic conditions wrap the domain around the globe. -CICE can be run on regional grids with open boundary conditions; except -for variables describing grid lengths, non-land halo cells along the -grid edge must be filled by restoring them to specified values. The -namelist variable `restore\_ice` turns this functionality on and off; the -restoring timescale `trestore` may be used (it is also used for restoring -ocean sea surface temperature in stand-alone ice runs). This -implementation is only intended to provide the “hooks" for a more -sophisticated treatment; the rectangular grid option can be used to test -this configuration. The ‘displaced\_pole’ grid option should not be used -unless the regional grid contains land all along the north and south -boundaries. The current form of the boundary condition routines does not -allow Neumann boundary conditions, which must be set explicitly. This -has been done in an unreleased branch of the code; contact Elizabeth for -more information. - -For exact restarts using restoring, set `restart\_ext` = true in namelist -to use the extended-grid subroutines. - -On tripole grids, the order of operations used for calculating elements -of the stress tensor can differ on either side of the fold, leading to -round-off differences. Although restarts using the extended grid -routines are exact for a given run, the solution will differ from -another run in which restarts are written at different times. For this -reason, explicit halo updates of the stress tensor are implemented for -the tripole grid, both within the dynamics calculation and for restarts. -This has not been implemented yet for tripoleT grids, pending further -testing. - -***** -Masks -***** - -A land mask hm (:math:`M_h`) is specified in the cell centers, with 0 -representing land and 1 representing ocean cells. A corresponding mask -uvm (:math:`M_u`) for velocity and other corner quantities is given by - -.. math:: - M_u(i,j)=\min\{M_h(l),\,l=(i,j),\,(i+1,j),\,(i,j+1),\,(i+1,j+1)\}. - -The logical masks `tmask` and `umask` (which correspond to the real masks -`hm` and `uvm`, respectively) are useful in conditional statements. - -In addition to the land masks, two other masks are implemented in -*evp\_prep* in order to reduce the dynamics component’s work on a global -grid. At each time step the logical masks `ice\_tmask` and `ice\_umask` are -determined from the current ice extent, such that they have the value -“true” wherever ice exists. They also include a border of cells around -the ice pack for numerical purposes. These masks are used in the -dynamics component to prevent unnecessary calculations on grid points -where there is no ice. They are not used in the thermodynamics -component, so that ice may form in previously ice-free cells. Like the -land masks `hm` and `uvm`, the ice extent masks `ice\_tmask` and `ice\_umask` -are for T cells and U cells, respectively. - -Improved parallel performance may result from utilizing halo masks for -boundary updates of the full ice state, incremental remapping transport, -or for EVP or EAP dynamics. These options are accessed through the -logical namelist flags `maskhalo\_bound`, `maskhalo\_remap`, and -`maskhalo\_dyn`, respectively. Only the halo cells containing needed -information are communicated. - -Two additional masks are created for the user’s convenience: `lmask\_n` -and `lmask\_s` can be used to compute or write data only for the northern -or southern hemispheres, respectively. Special constants (`spval` and -`spval\_dbl`, each equal to :math:`10^{30}`) are used to indicate land -points in the history files and diagnostics. - -~~~~~~~~~~~~~~~~~~~ -Test configurations -~~~~~~~~~~~~~~~~~~~ - -.. _init: - -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Initialization and coupling -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ice model’s parameters and variables are initialized in several -steps. Many constants and physical parameters are set in -**ice\_constants.F90**. Namelist variables (:ref:`tabnamelist`), -whose values can be altered at run time, are handled in *input\_data* -and other initialization routines. These variables are given default -values in the code, which may then be changed when the input file -**ice\_in** is read. Other physical constants, numerical parameters, and -variables are first set in initialization routines for each ice model -component or module. Then, if the ice model is being restarted from a -previous run, core variables are read and reinitialized in -*restartfile*, while tracer variables needed for specific configurations -are read in separate restart routines associated with each tracer or -specialized parameterization. Finally, albedo and other quantities -dependent on the initial ice state are set. Some of these parameters -will be described in more detail in :ref:`tabnamelist`. - -The restart files supplied with the code release include the core -variables on the default configuration, that is, with seven vertical -layers and the ice thickness distribution defined by `kcatbound` = 0. -Restart information for some tracers is also included in the  restart -files. - -Three namelist variables control model initialization, `ice\_ic`, `runtype`, -and `restart`, as described in :ref:`tab-ic`. It is possible to do an -initial run from a file **filename** in two ways: (1) set runtype = -‘initial’, restart = true and ice\_ic = **filename**, or (2) runtype = -‘continue’ and pointer\_file = **./restart/ice.restart\_file** where -**./restart/ice.restart\_file** contains the line -“./restart/[filename]". The first option is convenient when repeatedly -starting from a given file when subsequent restart files have been -written. With this arrangement, the tracer restart flags can be set to -true or false, depending on whether the tracer restart data exist. With -the second option, tracer restart flags are set to ‘continue’ for all -active tracers. - -An additional namelist option, `restart\_ext` specifies whether halo cells -are included in the restart files. This option is useful for tripole and -regional grids, but can not be used with PIO. - -MPI is initialized in *init\_communicate* for both coupled and -stand-alone MPI runs. The ice component communicates with a flux coupler -or other climate components via external routiines that handle the -variables listed in :ref:`tab-flux-cpl`. For stand-alone runs, -routines in **ice\_forcing.F90** read and interpolate data from files, -and are intended merely to provide guidance for the user to write his or -her own routines. Whether the code is to be run in stand-alone or -coupled mode is determined at compile time, as described below. - -:ref:`tab-ic` : *Ice initial state resulting from combinations of* -`ice\_ic`, `runtype` and `restart`. :math:`^a`\ *If false, restart is reset to -true.* :math:`^b`\ *restart is reset to false.* :math:`^c`\ ice\_ic *is -reset to ‘none.’* - -.. _tab-ic: - -.. table:: Table 4 - - +----------------+--------------------------+--------------------------------------+----------------------------------------+ - | ice\_ic | | | | - +================+==========================+======================================+========================================+ - | | initial/false | initial/true | continue/true (or false\ :math:`^a`) | - +----------------+--------------------------+--------------------------------------+----------------------------------------+ - | none | no ice | no ice\ :math:`^b` | restart using **pointer\_file** | - +----------------+--------------------------+--------------------------------------+----------------------------------------+ - | default | SST/latitude dependent | SST/latitude dependent\ :math:`^b` | restart using **pointer\_file** | - +----------------+--------------------------+--------------------------------------+----------------------------------------+ - | **filename** | no ice\ :math:`^c` | start from **filename** | restart using **pointer\_file** | - +----------------+--------------------------+--------------------------------------+----------------------------------------+ - -.. _parameters: - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Choosing an appropriate time step -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The time step is chosen based on stability of the transport component -(both horizontal and in thickness space) and on resolution of the -physical forcing. CICE allows the dynamics, advection and ridging -portion of the code to be run with a shorter timestep, -:math:`\Delta t_{dyn}` (`dt\_dyn`), than the thermodynamics timestep -:math:`\Delta t` (`dt`). In this case, `dt` and the integer ndtd are -specified, and `dt\_dyn` = `dt/ndtd`. - -A conservative estimate of the horizontal transport time step bound, or -CFL condition, under remapping yields - -.. math:: - \Delta t_{dyn} < {\min\left(\Delta x, \Delta y\right)\over 2\max\left(u, v\right)}. - -Numerical estimates for this bound for several POP grids, assuming -:math:`\max(u, v)=0.5` m/s, are as follows: - -.. csv-table:: - :widths: 20,40,40,40,40 - - grid label,N pole singularity,dimensions,min :math:`\sqrt{\Delta x\cdot\Delta y}`,max :math:`\Delta t_{dyn}` - gx3,Greenland,:math:`100\times 116`,:math:`39\times 10^3` m,10.8hr - gx1,Greenland,:math:`320\times 384`,:math:`18\times 10^3` m,5.0hr - p4,Canada,:math:`900\times 600`,:math:`6.5\times 10^3` m,1.8hr - -As discussed in section :ref:`mech-red` and -:cite:`LHMJ07`, the maximum time step in practice is -usually determined by the time scale for large changes in the ice -strength (which depends in part on wind strength). Using the strength -parameterization of :cite:`Rothrock75`, as in -Equation :eq:`roth-strength0`, limits the time step to :math:`\sim`\ 30 -minutes for the old ridging scheme (`krdg\_partic` = 0), and to -:math:`\sim`\ 2 hours for the new scheme (`krdg\_partic` = 1), assuming -:math:`\Delta x` = 10 km. Practical limits may be somewhat less, -depending on the strength of the atmospheric winds. - -Transport in thickness space imposes a similar restraint on the time -step, given by the ice growth/melt rate and the smallest range of -thickness among the categories, -:math:`\Delta t<\min(\Delta H)/2\max(f)`, where :math:`\Delta H` is the -distance between category boundaries and :math:`f` is the thermodynamic -growth rate. For the 5-category ice thickness distribution used as the -default in this distribution, this is not a stringent limitation: -:math:`\Delta t < 19.4` hr, assuming :math:`\max(f) = 40` cm/day. - -In the classic EVP or EAP approach (`kdyn` = 1 or 2, `revised\_evp` = false), -the dynamics component is subcycled ndte (:math:`N`) times per dynamics -time step so that the elastic waves essentially disappear before the -next time step. The subcycling time step (:math:`\Delta -t_e`) is thus - -.. math:: - dte = dt\_dyn/ndte. - -A second parameter, :math:`E_\circ` (`eyc`), defines the elastic wave -damping timescale :math:`T`, described in Section :ref:`dynam`, as -`eyc`\ * `dt\_dyn`. The forcing terms are not updated during the subcycling. -Given the small step (`dte`) at which the EVP dynamics model is subcycled, -the elastic parameter :math:`E` is also limited by stability -constraints, as discussed in :cite:`HD97`. Linear stability -analysis for the dynamics component shows that the numerical method is -stable as long as the subcycling time step :math:`\Delta t_e` -sufficiently resolves the damping timescale :math:`T`. For the stability -analysis we had to make several simplifications of the problem; hence -the location of the boundary between stable and unstable regions is -merely an estimate. In practice, the ratio -:math:`\Delta t_e ~:~ T ~:~ \Delta t`  = 1 : 40 : 120 provides both -stability and acceptable efficiency for time steps (:math:`\Delta t`) on -the order of 1 hour. - -For the revised EVP approach (`kdyn` = 1, `revised\_evp` = true), the -relaxation parameter `arlx1i` effectively sets the damping timescale in -the problem, and `brlx` represents the effective subcycling -:cite:`BFLM13`. In practice the parameters :math:`S_e>0.5` -and :math:`\xi<1` are set, along with an estimate of the ice strength -per unit mass, and the damping and subcycling parameters are then -calculated. With the addition of the revised EVP approach to CICE, the -code now uses these parameters internally for both classic and revised -EVP configurations (see Section :ref:`revp`). - -Note that only :math:`T` and :math:`\Delta t_e` figure into the -stability of the dynamics component; :math:`\Delta t` does not. Although -the time step may not be tightly limited by stability considerations, -large time steps (*e.g.,* :math:`\Delta t=1` day, given daily forcing) -do not produce accurate results in the dynamics component. The reasons -for this error are discussed in :cite:`HD97`; see -:cite:`HZ99` for its practical effects. The thermodynamics -component is stable for any time step, as long as the surface -temperature :math:`T_{sfc}` is computed internally. The -numerical constraint on the thermodynamics time step is associated with -the transport scheme rather than the thermodynamic solver. - -~~~~~~~~~~~~ -Model output -~~~~~~~~~~~~ - -.. _history: - -************* -History files -************* - -Model output data is averaged over the period(s) given by `histfreq` and -`histfreq\_n`, and written to binary or  files prepended by `history\_file` -in **ice\_in**. That is, if `history\_file` = ‘iceh’ then the filenames -will have the form **iceh.[timeID].nc** or **iceh.[timeID].da**, -depending on the output file format chosen in **comp\_ice** (set -`IO\_TYPE`). The  history files are CF-compliant; header information for -data contained in the  files is displayed with the command `ncdump -h -filename.nc`. Parallel  output is available using the PIO library; the -attribute `io\_flavor` distinguishes output files written with PIO from -those written with standard netCDF. With binary files, a separate header -file is written with equivalent information. Standard fields are output -according to settings in the **icefields\_nml** namelist in **ice\_in**. -The user may add (or subtract) variables not already available in the -namelist by following the instructions in section :ref:`addhist`. - -With this release, the history module has been divided into several -modules based on the desired formatting and on the variables -themselves. Parameters, variables and routines needed by multiple -modules is in **ice\_history\_shared.F90**, while the primary routines -for initializing and accumulating all of the history variables are in -**ice\_history.F90**. These routines call format-specific code in the -**io\_binary**, **io\_netcdf** and **io\_pio** directories. History -variables specific to certain components or parameterizations are -collected in their own history modules (**ice\_history\_bgc.F90**, -**ice\_history\_drag.F90**, **ice\_history\_mechred.F90**, -**ice\_history\_pond.F90**). - -The history modules allow output at different frequencies. Five output -frequencies (1, `h`, `d`, `m`, `y`) are available simultaneously during a run. -The same variable can be output at different frequencies (say daily and -monthly) via its namelist flag, `f\_` :math:`\left<{var}\right>`, which -is now a character string corresponding to `histfreq` or ‘x’ for none. -(Grid variable flags are still logicals, since they are written to all -files, no matter what the frequency is.) If there are no namelist flags -with a given `histfreq` value, or if an element of `histfreq\_n` is 0, then -no file will be written at that frequency. The output period can be -discerned from the filenames. - -For example, in namelist: - -:: - - `histfreq` = ’1’, ’h’, ’d’, ’m’, ’y’ - `histfreq\_n` = 1, 6, 0, 1, 1 - `f\_hi` = ’1’ - `f\_hs` = ’h’ - `f\_Tsfc` = ’d’ - `f\_aice` = ’m’ - `f\_meltb` = ’mh’ - `f\_iage` = ’x’ - -Here, `hi` will be written to a file on every timestep, `hs` will be -written once every 6 hours, `aice` once a month, `meltb` once a month AND -once every 6 hours, and `Tsfc` and `iage` will not be written. - -From an efficiency standpoint, it is best to set unused frequencies in -`histfreq` to ‘x’. Having output at all 5 frequencies takes nearly 5 times -as long as for a single frequency. If you only want monthly output, the -most efficient setting is `histfreq` = ’m’,’x’,’x’,’x’,’x’. The code counts -the number of desired streams (`nstreams`) based on `histfreq`. - -The history variable names must be unique for netcdf, so in cases where -a variable is written at more than one frequency, the variable name is -appended with the frequency in files after the first one. In the example -above, `meltb` is called `meltb` in the monthly file (for backward -compatibility with the default configuration) and `meltb\_h` in the -6-hourly file. - -Using the same frequency twice in `histfreq` will have unexpected -consequences and currently will cause the code to abort. It is not -possible at the moment to output averages once a month and also once -every 3 months, for example. - -If `write\_ic` is set to true in **ice\_in**, a snapshot of the same set -of history fields at the start of the run will be written to the history -directory in **iceh\_ic.[timeID].nc(da)**. Several history variables are -hard-coded for instantaneous output regardless of the averaging flag, at -the frequency given by their namelist flag. - -The normalized principal components of internal ice stress are computed -in *principal\_stress* and written to the history file. This calculation -is not necessary for the simulation; principal stresses are merely -computed for diagnostic purposes and included here for the user’s -convenience. - -Several history variables are available in two forms, a value -representing an average over the sea ice fraction of the grid cell, and -another that is multiplied by :math:`a_i`, representing an average over -the grid cell area. Our naming convention attaches the suffix “\_ai" to -the grid-cell-mean variable names. - -**************** -Diagnostic files -**************** - -Like `histfreq`, the parameter `diagfreq` can be used to regulate how often -output is written to a log file. The log file unit to which diagnostic -output is written is set in **ice\_fileunits.F90**. If `diag\_type` = -‘stdout’, then it is written to standard out (or to **ice.log.[ID]** if -you redirect standard out as in **run\_ice**); otherwise it is written -to the file given by `diag\_file`. In addition to the standard diagnostic -output (maximum area-averaged thickness, velocity, average albedo, total -ice area, and total ice and snow volumes), the namelist options -`print\_points` and `print\_global` cause additional diagnostic information -to be computed and written. `print\_global` outputs global sums that are -useful for checking global conservation of mass and energy. -`print\_points` writes data for two specific grid points. Currently, one -point is near the North Pole and the other is in the Weddell Sea; these -may be changed in **ice\_in**. - -Timers are declared and initialized in **ice\_timers.F90**, and the code -to be timed is wrapped with calls to *ice\_timer\_start* and -*ice\_timer\_stop*. Finally, *ice\_timer\_print* writes the results to -the log file. The optional “stats" argument (true/false) prints -additional statistics. Calling *ice\_timer\_print\_all* prints all of -the timings at once, rather than having to call each individually. -Currently, the timers are set up as in :ref:`timers`. -Section :ref:`addtimer` contains instructions for adding timers. - -The timings provided by these timers are not mutually exclusive. For -example, the column timer (5) includes the timings from 6–10, and -subroutine *bound* (timer 15) is called from many different places in -the code, including the dynamics and advection routines. - -The timers use *MPI\_WTIME* for parallel runs and the F90 intrinsic -*system\_clock* for single-processor runs. - -:ref:`timers` : *CICE timers* - -.. _timers: - -.. table:: Table 5 - - +--------------+-------------+----------------------------------------------------+ - | **Timer** | | | - +--------------+-------------+----------------------------------------------------+ - | **Index** | **Label** | | - +--------------+-------------+----------------------------------------------------+ - | 1 | Total | the entire run | - +--------------+-------------+----------------------------------------------------+ - | 2 | Step | total minus initialization and exit | - +--------------+-------------+----------------------------------------------------+ - | 3 | Dynamics | EVP | - +--------------+-------------+----------------------------------------------------+ - | 4 | Advection | horizontal transport | - +--------------+-------------+----------------------------------------------------+ - | 5 | Column | all vertical (column) processes | - +--------------+-------------+----------------------------------------------------+ - | 6 | Thermo | vertical thermodynamics | - +--------------+-------------+----------------------------------------------------+ - | 7 | Shortwave | SW radiation and albedo | - +--------------+-------------+----------------------------------------------------+ - | 8 | Meltponds | melt ponds | - +--------------+-------------+----------------------------------------------------+ - | 9 | Ridging | mechanical redistribution | - +--------------+-------------+----------------------------------------------------+ - | 10 | Cat Conv | transport in thickness space | - +--------------+-------------+----------------------------------------------------+ - | 11 | Coupling | sending/receiving coupler messages | - +--------------+-------------+----------------------------------------------------+ - | 12 | ReadWrite | reading/writing files | - +--------------+-------------+----------------------------------------------------+ - | 13 | Diags | diagnostics (log file) | - +--------------+-------------+----------------------------------------------------+ - | 14 | History | history output | - +--------------+-------------+----------------------------------------------------+ - | 15 | Bound | boundary conditions and subdomain communications | - +--------------+-------------+----------------------------------------------------+ - | 16 | BGC | biogeochemistry | - +--------------+-------------+----------------------------------------------------+ - -************* -Restart files -************* - -CICE now provides restart data in binary unformatted or  formats, via -the `IO\_TYPE` flag in **comp\_ice** and namelist variable -`restart\_format`. Restart and history files must use the same format. As -with the history output, there is also an option for writing parallel -restart files using PIO. - -The restart files created by CICE contain all of the variables needed -for a full, exact restart. The filename begins with the character string -‘iced.’, and the restart dump frequency is given by the namelist -variables `dumpfreq` and `dumpfreq\_n`. The pointer to the filename from -which the restart data is to be read for a continuation run is set in -`pointer\_file`. The code assumes that auxiliary binary tracer restart -files will be identified using the same pointer and file name prefix, -but with an additional character string in the file name that is -associated with each tracer set. All variables are included in  restart -files. - -Additional namelist flags provide further control of restart behavior. -`dump\_last` = true causes a set of restart files to be written at the end -of a run when it is otherwise not scheduled to occur. The flag -`use\_restart\_time` enables the user to choose to use the model date -provided in the restart files. If `use\_restart\_time` = false then the -initial model date stamp is determined from the namelist parameters. -lcdf64 = true sets 64-bit  output, allowing larger file sizes with -version 3. - -Routines for gathering, scattering and (unformatted) reading and writing -of the “extended" global grid, including the physical domain and ghost -(halo) cells around the outer edges, allow exact restarts on regional -grids with open boundary conditions, and they will also simplify -restarts on the various tripole grids. They are accessed by setting -`restart\_ext` = true in namelist. Extended grid restarts are not -available when using PIO; in this case extra halo update calls fill -ghost cells for tripole grids (do not use PIO for regional grids). - -Two restart files are included with the CICE v5 code distribution, for -the gx3 and gx1 grids. The were created using the default model -configuration (settings as in **comp\_ice** and **ice\_in**), but -initialized with no ice. The gx3 case was run for 1 year using the 1997 -forcing data provided with the code. The gx1 case was run for 20 years, -so that the date of restart in the file is 1978-01-01. Note that the -restart dates provided in the restart files can be overridden using the -namelist variables `use\_restart\_time`, `year\_init` and `istep0`. The -forcing time can also be overridden using `fyear\_init`. - -Several changes in CICE v5 have made restarting from v4.1 restart files -difficult. First, the ice and snow enthalpy state variables are now -carried as tracers instead of separate arrays, and salinity has been -added as a necessary restart field. Second, the default number of ice -layers has been increased from 4 to 7. Third, netcdf format is now used -for all I/O; it is no longer possible to have history output as  and -restart output in binary format. However, some facilities are included -with CICE v5 for converting v4.1 restart files to the new file structure -and format, provided that the same number of ice layers and basic -physics packages will be used for the new runs. See Section -:ref:`restarttrouble` for details. - --------------------- -Execution procedures --------------------- - -To compile and execute the code: in the source directory, - -#. Download the forcing data used for testing from the CICE-Consortium github page, - https://github.com/CICE-Consortium . - -#. Create **Macros.\*** and **run\_ice.\*** files for your particular - platform, if they do not already exist (type ‘uname -s’ at the prompt - to get :math:`\langle`\ OS\ :math:`\rangle`). - -#. Alter directories in the script **comp\_ice**. - -#. Run **comp\_ice** to set up the run directory and make the executable - ‘**cice**’. - -#. | To clean the compile directory and start fresh, simply execute - ‘/bin/rm -rf compile’ from the run directory. - -In the run directory, - -#. Alter `atm\_data\_dir` and `ocn\_data\_dir` in the namelist file - **ice\_in**. - -#. Alter the script **run\_ice** for your system. - -#. Execute **run\_ice**. - -If this fails, see Section :ref:`setup`. - -This procedure creates the output log file **ice.log.[ID]**, and if -`npt` is long enough compared with `dumpfreq` and `histfreq`, dump files -**iced.[timeID]** and   (or binary) history output files -**iceh\_[timeID].nc (.da)**. Using the :math:`\left<3^\circ\right>` -grid, the log file should be similar to -**ice.log.\ :math:`\langle`\ OS\ :math:`\rangle`**, provided for the -user’s convenience. These log files were created using MPI on 4 -processors on the :math:`\left<3^\circ\right>` grid. - -Several options are available in **comp\_ice** for configuring the run, -shown in :ref:`comp-ice`. If `NTASK` = 1, then the **serial/** -code is used, otherwise the code in **mpi/** is used. Loops over blocks -have been threaded throughout the code, so that their work will be -divided among `OMP\_NUM\_THREADS` if `THRD` is ‘yes.’ Note that the value of -`NTASK` in **comp\_ice** must equal the value of `nprocs` in **ice\_in**. -Generally the value of `MXBLCKS` computed by **comp\_ice** is sufficient, -but sometimes it will need to be set explicitly, as discussed in -Section :ref:`performance`. To conserve memory, match the tracer requests -in **comp\_ice** with those in **ice\_in**. CESM uses 3 aerosol tracers; -the number given in **comp\_ice** must be less than or equal to the -maximum allowed in **ice\_domain\_size.F90**. - -The scripts define a number of environment variables, mostly as -directories that you will need to edit for your own environment. -`$SYSTEM\_USERDIR`, which on machines at Oak Ridge National Laboratory -points automatically to scratch space, is intended to be a disk where -the run directory resides. `SHRDIR` is a path to the CESM shared code. - -:ref:`comp-ice` : Configuration options available in **comp_ice**. - -.. _comp-ice: - -.. table:: Table 6 - - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - | variable | options | description | - +=====================+======================================+====================================================================================+ - |RES | col, gx3, gx1 | grid resolution | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NTASK | (integer) | total number of processors | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |BLCKX | (integer) | number of grid cells on each block in the x-direction :math:`^\dagger` | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |BLCKY | (integer) | number of grid cells on each block in the y-direction :math:`^\dagger` | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |MXBLCKS | (integer) | maximum number of blocks per processor | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NICELYR | (integer) | number of vertical layers in the ice | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NSNWLYR | (integer) | number of vertical layers in the snow | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NICECAT | (integer) | number of ice thickness categories | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |TRAGE | 0 or 1 | set to 1 for ice age tracer | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |TRFY | 0 or 1 | set to 1 for first-year ice age tracer | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |TRLVL | 0 or 1 | set to 1 for level and deformed ice tracers | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |TRPND | 0 or 1 | set to 1 for melt pond tracers | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NTRAERO | 0 or 1 | number of aerosol tracers | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |TRBRINE | set to 1 for brine height tracer | | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NBGCLYR | (integer) | number of vertical layers for biogeochemical transport | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |IO_TYPE | none/netcdf/pio | use ‘none’ if  library is unavailable,‘pio’ for PIO | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |DITTO | yes/no | for reproducible diagnostics | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |BARRIERS | yes/no | flushes MPI buffers during global scatters and gathers | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |THRD | yes/no | set to yes for OpenMP threaded parallelism | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |OMP_NUM_THREADS | (integer) | the number of OpenMP threads requested | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NUMIN | (integer) | smallest unit number assigned to CICE files | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - |NUMAX | (integer) | largest unit number assigned to CICE files | - +---------------------+--------------------------------------+------------------------------------------------------------------------------------+ - -The ‘reproducible’ option (`DITTO`) makes diagnostics bit-for-bit when -varying the number of processors. (The simulation results are -bit-for-bit regardless, because they do not require global sums or -max/mins as do the diagnostics.) This was done mainly by increasing the -precision for the global reduction calculations, except for regular -double-precision (r8) calculations involving MPI; MPI can not handle -MPI\_REAL16 on some architectures. Instead, these cases perform sums or -max/min calculations across the global block structure, so that the -results are bit-for-bit as long as the block distribution is the same -(the number of processors can be different). - -A more flexible option is available for double-precision MPI -calculations, using the namelist variable `bfbflag`. When true, this flag -produces bit-for-bit identical diagnostics with different tasks, -threads, blocks and grid decompositions. - -CICE namelist variables available for changes after compile time appear -in **ice.log.\*** with values read from the file **ice\_in**; their -definitions are given in Section :ref:`index`. For example, to run for a -different length of time, say three days, set `npt` = 72 in **ice\_in**. -At present, the user supplies the time step `dt`, the number of -dynamics/advection/ridging subcycles `ndtd`, and for classic EVP, the -number of EVP subcycles `ndte`; `dte` is then calculated in subroutine -*init\_evp*. The primary reason for doing it this way is to ensure that -`ndte` is an integer. (This is done differently for `revised\_evp` = true.; -see Section :ref:`dynam`). - -To restart from a previous run, set restart = true in **ice\_in**. There -are two ways of restarting from a given file. The restart pointer file -**ice.restart\_file** (created by the previous run) contains the name of -the last written data file (**iced.[timeID]**). Alternatively, a -filename can be assigned to ice\_ic in **ice\_in**. Consult -Section :ref:`init` for more details. Restarts are exact for MPI or -single processor runs. - -~~~~~~~ -Scripts -~~~~~~~ - -~~~~~~~~~~~ -Directories -~~~~~~~~~~~ - -~~~~~~~~~~~~~~~~~~~ -Local modifications -~~~~~~~~~~~~~~~~~~~ - -~~~~~~~~~~~~ -Forcing data -~~~~~~~~~~~~ - - -.. _performance: - ------------ -Performance ------------ - -Namelist options (*domain\_nml*) provide considerable flexibility for -finding the most efficient processor and block configuration. Some of -these choices are illustration in :ref:`fig-distrb`. `processor\_shape` -chooses between tall, thin processor domains (`slenderX1` or `slenderX2`, -often better for sea ice simulations on global grids where nearly all of -the work is at the top and bottom of the grid with little to do in -between) and close-to-square domains, which maximize the volume to -surface ratio (and therefore on-processor computations to message -passing, if there were ice in every grid cell). In cases where the -number of processors is not a perfect square (4, 9, 16...), the -`processor\_shape` namelist variable allows the user to choose how the -processors are arranged. Here again, it is better in the sea ice model -to have more processors in x than in y, for example, 8 processors -arranged 4x2 (`square-ice`) rather than 2x4 (`square-pop`). The latter -option is offered for direct-communication compatibility with POP, in -which this is the default. - -The user provides the total number of processors and the block -dimensions in the setup script (**comp\_ice**). When moving toward -smaller, more numerous blocks, there is a point where the code becomes -less efficient; blocks should not have fewer than about 20 grid cells in -each direction. Squarish blocks optimize the volume-to-surface ratio for -communications. - -.. _fig-distrb: - -.. figure:: ./figures/distrb.png - :scale: 50% - - Figure 9 - -:ref:`fig-distrb` : Distribution of 256 blocks across 16 processors, -represented by colors, on the gx1 grid: (a) cartesian, slenderX1, (b) -cartesian, slenderX2, (c) cartesian, square-ice (square-pop is -equivalent here), (d) rake with block weighting, (e) rake with -latitude weighting, (f) spacecurve. Each block consists of 20x24 grid -cells, and white blocks consist entirely of land cells. - -The `distribution\_type` options allow standard Cartesian distribution of -blocks, redistribution via a ‘rake’ algorithm for improved load -balancing across processors, and redistribution based on space-filling -curves. There are also three additional distribution types -(‘roundrobin,’ ‘sectrobin,’ ‘sectcart’) that improve land-block -elimination rates and also allow more flexibility in the number of -processors used. The rake and space-filling curve algorithms are -primarily helpful when using squarish processor domains where some -processors (located near the equator) would otherwise have little work -to do. Processor domains need not be rectangular, however. - -`distribution\_wght` chooses how the work-per-block estimates are -weighted. The ‘block’ option is the default in POP, which uses a lot of -array syntax requiring calculations over entire blocks (whether or not -land is present), and is provided here for direct-communication -compatibility with POP. The ‘latitude’ option weights the blocks based -on latitude and the number of ocean grid cells they contain. - -The rake distribution type is initialized as a standard, Cartesian -distribution. Using the work-per-block estimates, blocks are “raked" -onto neighboring processors as needed to improve load balancing -characteristics among processors, first in the x direction and then in -y. - -Space-filling curves reduce a multi-dimensional space (2D, in our case) -to one dimension. The curve is composed of a string of blocks that is -snipped into sections, again based on the work per processor, and each -piece is placed on a processor for optimal load balancing. This option -requires that the block size be chosen such that the number of blocks in -the x direction equals the number of blocks in the y direction, and that -number must be factorable as :math:`2^n 3^m 5^p` where :math:`n, m, p` -are integers. For example, a 16x16 array of blocks, each containing -20x24 grid cells, fills the gx1 grid (:math:`n=4, m=p=0`). If either of -these conditions is not met, a Cartesian distribution is used instead. - -While the Cartesian distribution groups sets of blocks by processor, the -‘roundrobin’ distribution loops through the blocks and processors -together, putting one block on each processor until the blocks are gone. -This provides good load balancing but poor communication characteristics -due to the number of neighbors and the amount of data needed to -communicate. The ‘sectrobin’ and ‘sectcart’ algorithms loop similarly, -but put groups of blocks on each processor to improve the communication -characteristics. In the ‘sectcart’ case, the domain is divided into two -(east-west) halves and the loops are done over each, sequentially. -:ref:`fig-distribscorecard` provides an overview of the pros and cons -for the distribution types. - -.. _fig-distribscorecard: - -.. figure:: ./figures/scorecard.png - :scale: 20% - - Figure 10 - -:ref:`fig-distribscorecard` : Scorecard for block distribution choices in -CICE, courtesy T. Craig. For more information, see -http://www.cesm.ucar.edu/events/ws.2012/Presentations/SEWG2/craig.pdf - -The `maskhalo` options in the namelist improve performance by removing -unnecessary halo communications where there is no ice. There is some -overhead in setting up the halo masks, which is done during the -timestepping procedure as the ice area changes, but this option -usually improves timings even for relatively small processor counts. -T. Craig has found that performance improved by more than 20% for -combinations of updated decompositions and masked haloes, in CESM’s -version of CICE. A practical guide for choosing a CICE grid -decomposition, based on experience in CESM, is available: -http://oceans11.lanl.gov/drupal/CICE/DecompositionGuide - -Throughout the code, (i, j) loops have been combined into a single loop, -often over just ocean cells or those containing sea ice. This was done -to reduce unnecessary operations and to improve vector performance. - -:ref:`fig-timings` illustrates the computational expense of various -options, relative to the total time (excluding initialization) of a -7-layer configuration using BL99 thermodynamics, EVP dynamics, and the -‘ccsm3’ shortwave parameterization on the gx1 grid, run for one year -from a no-ice initial condition. The block distribution consisted of -20 \ :math:`\times` 192 blocks spread over 32 processors (‘slenderX2’) -with no threads and -O2 optimization. Timings varied by about -:math:`\pm3`\ % in identically configured runs due to machine load. -Extra time required for tracers has two components, that needed to carry -the tracer itself (advection, category conversions) and that needed for -the calculations associated with the particular tracer. The age tracers -(FY and iage) require very little extra calculation, so their timings -represent essentially the time needed just to carry an extra tracer. The -topo melt pond scheme is slightly faster than the others because it -calculates pond area and volume once per grid cell, while the others -calculate it for each thickness category. - -.. _fig-timings: - -.. figure:: ./figures/histograms.png - :scale: 20% - - Figure 11 - -:ref:`fig-timings` : Change in ‘TimeLoop’ timings from the 7-layer -configuration using BL99 thermodynamics and EVP dynamics. Timings -were made on a nondedicated machine, with variations of about -:math:`\pm3`\ % in identically configured runs (light grey). Darker -grey indicates the time needed for extra required options; The -Delta-Eddington radiation scheme is required for all melt pond -schemes and the aerosol tracers, and the level-ice pond -parameterization additionally requires the level-ice tracers. - -------------- -Adding things -------------- - -.. _addtimer: - -~~~~~~ -Timers -~~~~~~ - -Timing any section of code, or multiple sections, consists of defining -the timer and then wrapping the code with start and stop commands for -that timer. Printing of the timer output is done simultaneously for all -timers. To add a timer, first declare it (`timer\_[tmr]`) at the top of -**ice\_timers.F90** (we recommend doing this in both the **mpi/** and -**serial/** directories), then add a call to *get\_ice\_timer* in the -subroutine *init\_ice\_timers*. In the module containing the code to be -timed, `call ice\_timer\_start`(`timer\_[tmr]`) at the beginning of the -section to be timed, and a similar call to `ice\_timer\_stop` at the end. -A use `ice\_timers` statement may need to be added to the subroutine being -modified. Be careful not to have one command outside of a loop and the -other command inside. Timers can be run for individual blocks, if -desired, by including the block ID in the timer calls. - -.. _addhist: - -~~~~~~~~~~~~~~ -History fields -~~~~~~~~~~~~~~ - -To add a variable to be printed in the history output, search for -‘example’ in **ice\_history\_shared.F90**: - -#. add a frequency flag for the new field - -#. add the flag to the namelist (here and also in **ice\_in**) - -#. add an index number - -and in **ice\_history.F90**: - -#. broadcast the flag - -#. add a call to `define\_hist\_field` - -#. add a call to `accum\_hist\_field` - -The example is for a standard, two-dimensional (horizontal) field; for -other array sizes, choose another history variable with a similar shape -as an example. Some history variables, especially tracers, are grouped -in other files according to their purpose (bgc, melt ponds, etc.). - -To add an output frequency for an existing variable, see -section :ref:`history`. - -.. _addtrcr: - -~~~~~~~ -Tracers -~~~~~~~ - -Each optional tracer has its own module, **ice\_[tracer].F90**, which -also contains as much of the additional tracer code as possible, and for -backward compatibility of binary restart files, each new tracer has its -own binary restart file. We recommend that the logical namelist variable -`tr\_[tracer]` be used for all calls involving the new tracer outside of -**ice\_[tracer].F90**, in case other users do not want to use that -tracer. - -A number of optional tracers are available in the code, including ice -age, first-year ice area, melt pond area and volume, brine height, -aerosols, and level ice area and volume (from which ridged ice -quantities are derived). Salinity, enthalpies, age, aerosols, level-ice -volume, brine height and most melt pond quantities are volume-weighted -tracers, while first-year area, pond area, level-ice area and all of the -biogeochemistry tracers in this release are area-weighted tracers. In -the absence of sources and sinks, the total mass of a volume-weighted -tracer such as aerosol (kg) is conserved under transport in horizontal -and thickness space (the mass in a given grid cell will change), whereas -the aerosol concentration (kg/m) is unchanged following the motion, and -in particular, the concentration is unchanged when there is surface or -basal melting. The proper units for a volume-weighted mass tracer in the -tracer array are kg/m. - -In several places in the code, tracer computations must be performed on -the conserved “tracer volume" rather than the tracer itself; for -example, the conserved quantity is :math:`h_{pnd}a_{pnd}a_{lvl}a_{i}`, -not :math:`h_{pnd}`. Conserved quantities are thus computed according to -the tracer dependencies, and code must be included to account for new -dependencies (e.g., :math:`a_{lvl}` and :math:`a_{pnd}` in -**ice\_itd.F90** and **ice\_mechred.F90**). - -To add a tracer, follow these steps using one of the existing tracers as -a pattern. - -#. **ice\_domain\_size.F90**: increase `max\_ntrcr` (can also add option - to **comp\_ice** and **bld/Macros.\***) - -#. **ice\_state.F90**: declare `nt\_[tracer]` and `tr\_[tracer]` - -#. **ice\_[tracer].F90**: create initialization, physics, restart - routines - -#. **ice\_fileunits.F90**: add new dump and restart file units - -#. **ice\_init.F90**: (some of this may be done in **ice\_[tracer].F90** - instead) - - - add new module and `tr\_[tracer]` to list of used modules and - variables - - - add logical namelist variable `tr\_[tracer]` - - - initialize namelist variable - - - broadcast namelist variable - - - print namelist variable to diagnostic output file - - - increment number of tracers in use based on namelist input (`ntrcr`) - - - define tracer types (`trcr\_depend` = 0 for ice area tracers, 1 for - ice volume, 2 for snow volume, 2+nt\_[tracer] for dependence on - other tracers) - -#. **ice\_itd.F90**, **ice\_mechred.F90**: Account for new dependencies - if needed. - -#. **CICE\_InitMod.F90**: initialize tracer (includes reading restart - file) - -#. **CICE\_RunMod.F90**, **ice\_step\_mod.F90**: - - - call routine to write tracer restart data - - - call physics routines in **ice\_[tracer].F90** (often called from - **ice\_step\_mod.F90**) - -#. **ice\_restart.F90**: define restart variables (for binary,  and PIO) - -#. **ice\_history\_[tracer].F90**: add history variables - (Section :ref:`addhist`) - -#. **ice\_in**: add namelist variables to *tracer\_nml* and - *icefields\_nml* - -#. If strict conservation is necessary, add diagnostics as noted for - topo ponds in Section :ref:`ponds`. - ---------------- -Troubleshooting ---------------- - -Check the FAQ: http://oceans11.lanl.gov/drupal/CICE/FAQ. - -.. _setup: - -~~~~~~~~~~~~~ -Initial setup -~~~~~~~~~~~~~ - -The script **comp\_ice** is configured so that the files **grid**, -**kmt**, **ice\_in**, **run\_ice**, **iced\_gx3\_v5.0** and -**ice.restart\_file** are NOT overwritten after the first setup. If you -wish to make changes to the original files in **input\_templates/** -rather than those in the run directory, either remove the files from the -run directory before executing **comp\_ice** or edit the script. - -The code may abort during the setup phase for any number of reasons, and -often the buffer containing the diagnostic output fails to print before -the executable exits. The quickest way to get the diagnostic information -is to run the code in an interactive shell with just the command `cice` -for serial runs or “`mpirun -np N cice`” for MPI runs, where N is the -appropriate number of processors (or a command appropriate for your -computer’s software). - -If the code fails to compile or run, or if the model configuration is -changed, try the following: - -- create **Macros.\***. **Makefile.\*** and **run\_ice.\*** files for - your particular platform, if they do not already exist (type ‘uname - -s’ at the prompt and compare the result with the file suffixes; we - rename `UNICOS/mp` as `UNICOS` for simplicity). - -- modify the `INCLUDE` directory path and other settings for your system - in the scripts, **Macros.\*** and **Makefile.\*** files. - -- alter directory paths, file names and the execution command as needed - in **run\_ice** and **ice\_in**. - -- ensure that `nprocs` in **ice\_in** is equal to `NTASK` in **comp\_ice**. - -- ensure that the block size `NXBLOCK`, `NYBLOCK` in **comp\_ice** is - compatible with the processor\_shape and other domain options in - **ice\_in** - -- if using the rake or space-filling curve algorithms for block - distribution (`distribution\_type` in **ice\_in**) the code will abort - if `MXBLCKS` is not large enough. The correct value is provided in the - diagnostic output. - -- if starting from a restart file, ensure that kcatbound is the same as - that used to create the file (`kcatbound` = 0 for the files included in - this code distribution). Other configuration parameters, such as - `NICELYR`, must also be consistent between runs. - -- for stand-alone runs, check that `-Dcoupled` is *not* set in the - **Macros.\*** file. - -- for coupled runs, check that `-Dcoupled` and other - coupled-model-specific (e.g., CESM, popcice or hadgem) preprocessing - options are set in the **Macros.\*** file. - -- edit the grid size and other parameters in **comp\_ice**. - -- remove the **compile/** directory completely and recompile. - -.. _restarttrouble: - -~~~~~~~~ -Restarts -~~~~~~~~ - -CICE version 5 introduces a new model configuration that makes -restarting from older simulations difficult. In particular, the number -of ice categories, the category boundaries, and the number of vertical -layers within each category must be the same in the restart file and in -the run restarting from that file. Moreover, significant differences in -the physics, such as the salinity profile, may cause the code to fail -upon restart. Therefore, new model configurations may need to be started -using `runtype` = ‘initial’. Binary restart files that were provided with -CICE v4.1 were made using the BL99 thermodynamics with 4 layers and 5 -thickness categories (`kcatbound` = 0) and therefore can not be used for -the default CICE v5 configuration (7 layers). In addition, CICE’s -default restart file format is now  instead of binary. - -Restarting a run using `runtype` = ‘continue’ requires restart data for -all tracers used in the new run. If tracer restart data is not -available, use `runtype` = ‘initial’, setting `ice\_ic` to the name of the -core restart file and setting to true the namelist restart flags for -each tracer that is available. The unavailable tracers will be -initialized to their default settings. - -On tripole grids, use `restart\_ext` = true when using either binary or -regular (non-PIO) netcdf. - -Provided that the same number of ice layers (default: 4) will be used -for the new runs, it is possible to convert v4.1 restart files to the -new file structure and then to  format. If the same physical -parameterizations are used, the code should be able to execute from -these files. However if different physics is used (for instance, mushy -thermo instead of BL99), the code may still fail. To convert a v4.1 -restart file: - -#. Edit the code **input\_templates/convert\_restarts.f90** for your - model configuration and path names. Compile and run this code to - create a binary restart file that can be read using v5. Copy the - resulting file to the **restart/** subdirectory in your working - directory. - -#. In your working directory, turn off all tracer restart flags in - **ice\_in** and set the following: - - - runtype = ‘initial’ - - - ice\_ic = ‘./restart/[your binary file name]’ - - - restart = .true. - - - use\_restart\_time = .true. - -#. In **CICE\_InitMod.F90**, comment out the call to - restartfile(ice\_ic) and uncomment the call to - restartfile\_v4(ice\_ic) immediately below it. This will read the - v4.1 binary file and write a v5  file containing the same - information. - -If restart files are taking a long time to be written serially (i.e., -not using PIO), see the next section. - -~~~~~~~~~~~~~~ -Slow execution -~~~~~~~~~~~~~~ - -On some architectures, underflows (:math:`10^{-300}` for example) are -not flushed to zero automatically. Usually a compiler flag is available -to do this, but if not, try uncommenting the block of code at the end of -subroutine *stress* in **ice\_dyn\_evp.F90** or **ice\_dyn\_eap.F90**. -You will take a hit for the extra computations, but it will not be as -bad as running with the underflows. - -In some configurations, multiple calls to scatter or gather global -variables may overfill MPI’s buffers, causing the code to slow down -(particularly when writing large output files such as restarts). To -remedy this problem, set `BARRIERS yes` in **comp\_ice**. This -synchronizes MPI messages, keeping the buffers in check. - -~~~~~~~~~~~~~~~ -Debugging hints -~~~~~~~~~~~~~~~ - -Several utilities are available that can be helpful when debugging the -code. Not all of these will work everywhere in the code, due to possible -conflicts in module dependencies. - -*debug\_ice* (**CICE.F90**) - A wrapper for *print\_state* that is easily called from numerous - points during the timestepping loop (see - **CICE\_RunMod.F90\_debug**, which can be substituted for - **CICE\_RunMod.F90**). - -*print\_state* (**ice\_diagnostics.F90**) - Print the ice state and forcing fields for a given grid cell. - -`dbug` = true (**ice\_in**) - Print numerous diagnostic quantities. - -`print\_global` (**ice\_in**) - If true, compute and print numerous global sums for energy and mass - balance analysis. This option can significantly degrade code - efficiency. - -`print\_points` (**ice\_in**) - If true, print numerous diagnostic quantities for two grid cells, - one near the north pole and one in the Weddell Sea. This utility - also provides the local grid indices and block and processor numbers - (`ip`, `jp`, `iblkp`, `mtask`) for these points, which can be used in - conjunction with `check\_step`, to call *print\_state*. These flags - are set in **ice\_diagnostics.F90**. This option can be fairly slow, - due to gathering data from processors. - -*global\_minval, global\_maxval, global\_sum* (**ice\_global\_reductions.F90**) - Compute and print the minimum and maximum values for an individual - real array, or its global sum. - -~~~~~~~~~~ -Known bugs -~~~~~~~~~~ - -#. Fluxes sent to the CESM coupler may have incorrect values in grid - cells that change from an ice-free state to having ice during the - given time step, or vice versa, due to scaling by the ice area. The - authors of the CESM flux coupler insist on the area scaling so that - the ice and land models are treated consistently in the coupler (but - note that the land area does not suddenly become zero in a grid cell, - as does the ice area). - -#. With the old CCSM radiative scheme (`shortwave` = ‘default’ or - ‘ccsm3’), a sizable fraction (more than 10%) of the total shortwave - radiation is absorbed at the surface but should be penetrating into - the ice interior instead. This is due to use of the aggregated, - effective albedo rather than the bare ice albedo when - `snowpatch` :math:`< 1`. - -#. The date-of-onset diagnostic variables, `melt\_onset` and `frz\_onset`, - are not included in the core restart file, and therefore may be - incorrect for the current year if the run is restarted after Jan 1. - Also, these variables were implemented with the Arctic in mind and - may be incorrect for the Antarctic. - -#. The single-processor *system\_clock* time may give erratic results on - some architectures. - -#. History files that contain time averaged data (`hist\_avg` = true in - **ice\_in**) will be incorrect if restarting from midway through an - averaging period. - -#. In stand-alone runs, restarts from the end of `ycycle` will not be - exact. - -#. Using the same frequency twice in `histfreq` will have unexpected - consequences and causes the code to abort. - -#. Latitude and longitude fields in the history output may be wrong when - using padding. - -~~~~~~~~~~~~~~~~~~~~~~~~~ -Interpretation of albedos -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The snow-and-ice albedo, `albsni`, and diagnostic albedos `albice`, `albsno`, -and `albpnd` are merged over categories but not scaled (divided) by the -total ice area. (This is a change from CICE v4.1 for `albsni`.) The latter -three history variables represent completely bare or completely snow- or -melt-pond-covered ice; that is, they do not take into account the snow -or melt pond fraction (`albsni` does, as does the code itself during -thermodyamic computations). This is to facilitate comparison with -typical values in measurements or other albedo parameterizations. The -melt pond albedo `albpnd` is only computed for the Delta-Eddington -shortwave case. - -With the Delta-Eddington parameterization, the albedo depends on the -cosine of the zenith angle (:math:`\cos\varphi`, `coszen`) and is zero if -the sun is below the horizon (:math:`\cos\varphi < 0`). Therefore -time-averaged albedo fields would be low if a diurnal solar cycle is -used, because zero values would be included in the average for half of -each 24-hour period. To rectify this, a separate counter is used for the -averaging that is incremented only when :math:`\cos\varphi > 0`. The -albedos will still be zero in the dark, polar winter hemisphere. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Proliferating subprocess parameterizations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -With the addition of several alternative parameterizations for sea ice -processes, a number of subprocesses now appear in multiple parts of the -code with differing descriptions. For instance, sea ice porosity and -permeability, along with associated flushing and flooding, are -calculated separately for mushy thermodynamics, topo and level-ice melt -ponds, and for the brine height tracer, each employing its own -equations. Likewise, the BL99 and mushy thermodynamics compute freeboard -and snow–ice formation differently, and the topo and level-ice melt pond -schemes both allow fresh ice to grow atop melt ponds, using slightly -different formulations for Stefan freezing. These various process -parameterizations will be compared and their subprocess descriptions -possibly unified in the future. - ------------- -Testing CICE ------------- - -Version 6, August 2017 -This documents how to use the testing features developed for the -CICE Consortium CICE sea ice model. - -.. _basic: - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Individual tests and test suites -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The CICE scripts support both setup of individual tests as well as test suites. Individual -tests are run from the command line like - - > cice.setup -t smoke -m wolf -g gx3 -p 8x2 -s diag1,run5day -testid myid - -where -m designates a specific machine. Test suites are multiple tests that are specified in -an input file and are started on the command line like - - > cice.setup -ts base_suite -m wolf -testid myid - -cice.setup with -t or -ts require a testid to uniquely name test directories. The format -of the case directory name for a test will always be -${machine}_${test}_${grid}_${pes}_${soptions}.${testid} - -To build and run a test, the process is the same as a case, - cd into the test directory, - - run cice.build - - run cice.submit - -The test results will be generated in a local file called "test_output". - -When running a test suite, the cice.setup command line automatically generates all the tests -under a directory names ${test_suite}.${testid}. It then automatically builds and submits all -tests. When the tests are complete, run the results.csh script to see the results from all the -tests. - -Tests are defined under configuration/scripts/tests. The tests currently supported are: - smoke - Runs the model for default length. The length and options can - be set with the -s commmand line option. The test passes if the - model completes successfully. - restart - Runs the model for 10 days, writing a restart file at day 5 and - again at day 10. Runs the model a second time starting from the - day 5 restart and writing a restart at day 10 of the model run. - The test passes if both the 10 day and 5 day restart run complete and - if the restart files at day 10 from both runs are bit-for-bit identical. - -Please run './cice.setup -h' for additional details. - -.. _additional: - -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Additional testing options -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are several additional options on the cice.setup command line for testing that -provide the ability to regression test and compare tests to each other. - - -bd defines a baseline directory where tests can be stored for regression testing - - -bg defines a version name that where the current tests can be saved for regression testing - - -bc defines a version name that the current tests should be compared to for regression testing - - -td provides a way to compare tests with each other - -To use -bg, - > cice.setup -ts base_suite -m wolf -testid v1 -bg version1 -bd $SCRATCH/CICE_BASELINES - will copy all the results from the test suite to $SCRATCH/CICE_BASELINES/version1. - -To use -bc, - > cice.setup -ts base_suite -m wolf -testid v2 -bc version1 -bd $SCRATCH/CICE_BASELINES - will compare all the results from this test suite to results saved before in $SCRATCH/CICE_BASELINES/version1. - --bc and -bg can be combined, - >cice.setup -ts base_suite -m wolf -testid v2 -bg version2 -bc version1 -bd $SCRATCH/CICE_BASELINES - will save the current results to $SCRATCH/CICE_BASELINES/version2 and compare the current results to - results save before in $SCRATCH/CICE_BASELINES/version1. - --bg, -bc, and -bd are used for regression testing. There is a default -bd on each machine. - --td allows a user to compare one test result to another. For instance, - > cice.setup -t smoke -m wolf -g gx3 -p 8x2 -s run5day -testid t01 - > cice.setup -t smoke -m wolf -g gx3 -p 4x2 -s run5day -testid t01 -td smoke_gx3_8x2_run5day - -An additional check will be done for the second test (because of the -td argument), and it will compare -the output from the first test "smoke_gx3_8x2_run5day" to the output from it's test "smoke_gx3_4x2_run5day" -and generate a result for that. It's important that the first test complete before the second test is -done. Also, the -td option works only if the testid and the machine are the same for the baseline -run and the current run. - -.. _format: - -~~~~~~~~~~~~~~~~~ -Test suite format -~~~~~~~~~~~~~~~~~ - -The format for the test suite file is relatively simple. It is a text file with white space delimited -columns like, - -.. _tab-test: - -.. csv-table:: Table 7 - :header: "#Test", "Grid", "PEs", "Sets", "BFB-compare" - :widths: 7, 7, 7, 15, 15 - - "smoke", "gx3", "8x2", "diag1,run5day", "" - "smoke", "gx3", "8x2", "diag24,run1year,medium", "" - "smoke", "gx3", "4x1", "debug,diag1,run5day", "" - "smoke", "gx3", "8x2", "debug,diag1,run5day", "" - "smoke", "gx3", "4x2", "diag1,run5day", "smoke_gx3_8x2_diag1_run5day" - "smoke", "gx3", "4x1", "diag1,run5day,thread", "smoke_gx3_8x2_diag1_run5day" - "smoke", "gx3", "4x1", "diag1,run5day", "smoke_gx3_4x1_diag1_run5day_thread" - "restart", "gx3", "8x1", "", "" - "restart", "gx3", "4x2", "debug", "" - - -The first column is the test name, the second the grid, the third the pe count, the fourth column is -the -s options and the fifth column is the -td argument. The fourth and fifth columns are optional. -The argument to -ts defines which filename to choose and that argument can contain a path. cice.setup -will also look for the filename in configuration/scripts/tests where some preset test suites are defined. - -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Example Tests (Quickstart) -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -********************************************** -To generate a baseline dataset for a test case -********************************************** - -./cice.setup -t smoke -m wolf -bg cicev6.0.0 -testid t00 - -cd wolf_smoke_gx3_4x1.t00 - -./cice.build - -./cice.submit - -# After job finishes, check output - -cat test_output - -**************************************************** -To run a test case and compare to a baseline dataset -**************************************************** - -./cice.setup -t smoke -m wolf -bc cicev6.0.0 -testid t01 - -cd wolf_smoke_gx3_4x1.t01 - -./cice.build - -./cice.submit - -# After job finishes, check output - -cat test_output - -********************************************* -To run a test suite to generate baseline data -********************************************* - -./cice.setup -m wolf -ts base_suite -testid t02 -bg cicev6.0.0bs - -cd base_suite.t02 - -# Once all jobs finish, concatenate all output - -./results.csh # All tests results will be stored in results.log - -# To plot a timeseries of "total ice extent", "total ice area", and "total ice volume" - -./timeseries.csh - -ls \*.png - -*********************************************** -To run a test suite to compare to baseline data -*********************************************** - -./cice.setup -m wolf -ts base_suite -testid t03 -bc cicev6.0.0bs - -cd base_suite.t03 - -# Once all jobs finish, concatenate all output - -./results.csh # All tests results will be stored in results.log - -# To plot a timeseries of "total ice extent", "total ice area", and "total ice volume" - -./timeseries.csh - -ls \*.png - -************************** -To compare to another test -************************** -`First:` - -./cice.setup -m wolf -t smoke -testid t01 -p 8x2 - -cd wolf_smoke_gx3_8x2.t01 - -./cice.build - -./cice.submit - -# After job finishes, check output - -cat test_output - -`Then, do the comparison:` - -./cice.setup -m wolf -t smoke -testid t01 -td smoke_gx3_8x2 -s thread -p 4x1 - -cd wolf_smoke_gx3_4x1_thread.t01 - -./cice.build - -./cice.submit - -# After job finishes, check output - -cat test_output - -****************** -Additional Details -****************** - -- In general, the baseline generation, baseline compare, and test diff are independent. -- Use the '-bd' flag to specify the location where you want the baseline dataset - to be written. Without specifying '-bd', the baseline dataset will be written - to the default baseline directory found in the env. file (ICE_MACHINE_BASELINE). -- If '-bd' is not passed, the scripts will look for baseline datasets in the default - baseline directory found in the env. file (ICE_MACHINE_BASELINE). - If the '-bd' option is passed, the scripts will look for baseline datasets in the - location passed to the -bd argument. -- To generate a baseline dataset for a specific version (for regression testing), - use '-bg '. The scripts will then place the baseline dataset - in $ICE_MACHINE_BASELINE// -- The '-testid' flag allows users to specify a testing id that will be added to the - end of the case directory. For example, "./cice.setup -m wolf -t smoke -testid t12 -p 4x1" - creates the directory wolf_smoke_gx3_4x1.t12. This flag is REQUIRED if using -t or -ts. - -.. _compliance: - -~~~~~~~~~~~~~~~~~~~~ -Code Compliance Test -~~~~~~~~~~~~~~~~~~~~ - -A core tenet of CICE dycore and Icepack innovations is that they must not change -the physics and biogeochemistry of existing model configurations, notwithstanding -obsolete model components. Therefore, alterations to existing CICE Consortium code -must only fix demonstrable numerical or scientific inaccuracies or bugs, or be -necessary to introduce new science into the code. New physics and biogeochemistry -introduced into the model must not change model answers when switched off, and in -that case CICEcore and Icepack must reproduce answers bit-for-bit as compared to -previous simulations with the same namelist configurations. This bit-for-bit -requirement is common in Earth System Modeling projects, but often cannot be achieved -in practice because model additions may require changes to existing code. In this -circumstance, bit-for-bit reproducibility using one compiler may not be unachievable -on a different computing platform with a different compiler. Therefore, tools for -scientific testing of CICE code changes have been developed to accompany bit-for-bit -testing. These tools exploit the statistical properties of simulated sea ice thickness -to confirm or deny the null hypothesis, which is that new additions to the CICE dycore -and Icepack have not significantly altered simulated ice volume using previous model -configurations. Here we describe the CICE testing tools, which are applies to output -from five-year gx-1 simulations that use the standard CICE atmospheric forcing. -A scientific justification of the testing is provided in -:cite:`Hunke2018`. - -.. _paired: - -******************************* -Two-Stage Paired Thickness Test -******************************* - -The first quality check aims to confirm the null hypotheses -:math:`H_0\!:\!\mu_d{=}0` at every model grid point, given the mean -thickness difference :math:`\mu_d` between paired CICE simulations -‘:math:`a`’ and ‘:math:`b`’ that should be identical. :math:`\mu_d` is -approximated as -:math:`\bar{h}_{d}=\tfrac{1}{n}\sum_{i=1}^n (h_{ai}{-}h_{bi})` for -:math:`n` paired samples of ice thickness :math:`h_{ai}` and -:math:`h_{bi}` in each grid cell of the gx-1 mesh. Following -:cite:`Wilks2006`, the associated :math:`t`-statistic -expects a zero mean, and is therefore - -.. math:: - t=\frac{\bar{h}_{d}}{\sigma_d/\sqrt{n_{eff}}} - :label: t-distribution - -given variance -:math:`\sigma_d^{\;2}=\frac{1}{n-1}\sum_{i=1}^{n}(h_{di}-\bar{h}_d)^2` -of :math:`h_{di}{=}(h_{ai}{-}h_{bi})` and effective sample size - -.. math:: - n_{eff}{=}n\frac{({1-r_1})}{({1+r_1})} - :label: neff - -for lag-1 autocorrelation: - -.. math:: - r_1=\frac{\sum\limits_{i=1}^{n-1}\big[(h_{di}-\bar{h}_{d1:n-1})(h_{di+1}-\bar{h}_{d2:n})\big]}{\sqrt{\sum\limits_{i=1}^{n-1} (h_{di}-\bar{h}_{d1:n-1})^2 \sum\limits_{i=2}^{n} (h_{di}-\bar{h}_{d2:n})^2 }}. - :label: r1 - -Here, :math:`\bar{h}_{d1:n-1}` is the mean of all samples except the -last, and :math:`\bar{h}_{d2:n}` is the mean of samples except the -first, and both differ from the overall mean :math:`\bar{h}_d` in -equations (:eq:`t-distribution`). That is: - -.. math:: - \bar{h}_{d1:n-1}=\frac{1}{n{-}1} \sum \limits_{i=1}^{n-1} h_{di},\quad - \bar{h}_{d2:n}=\frac{1}{n{-}1} \sum \limits_{i=2}^{n} h_{di},\quad - \bar{h}_d=\frac{1}{n} \sum \limits_{i=1}^{n} {h}_{di} - :label: short-means - -Following :cite:`Zwiers1995`, the effective sample size is -limited to :math:`n_{eff}\in[2,n]`. This definition of :math:`n_{eff}` -assumes ice thickness evolves as an AR(1) process -:cite:`VonStorch1999`, which can be justified by analyzing -the spectral density of daily samples of ice thickness from 5-year -records in CICE Consortium member models :cite:`Hunke2018`. -The AR(1) approximation is inadmissible for paired velocity samples, -because ice drift possesses periodicity from inertia and tides -:cite:`Hibler2006,Lepparanta2012,Roberts2015`. Conversely, -tests of paired ice concentration samples may be less sensitive to ice -drift than ice thickness. In short, ice thickness is the best variable -for CICE Consortium quality control (QC), and for the test of the mean -in particular. - -Care is required in analyzing mean sea ice thickness changes using -(:eq:`t-distribution`) with -:math:`N{=}n_{eff}{-}1` degrees of freedom. -:cite:`Zwiers1995` demonstrate that the :math:`t`-test in -(:eq:`t-distribution`) becomes conservative when -:math:`n_{eff} < 30`, meaning that :math:`H_0` may be erroneously -confirmed for highly auto-correlated series. Strong autocorrelation -frequently occurs in modeled sea ice thickness, and :math:`r_1>0.99` is -possible in parts of the gx-1 domain for the five-year QC simulations. -In the event that :math:`H_0` is confirmed but :math:`2\leq n_{eff}<30`, -the :math:`t`-test progresses to the ‘Table Lookup Test’ of -:cite:`Zwiers1995`, to check that the first-stage test -using (:eq:`t-distribution`) was not -conservative. The Table Lookup Test chooses critical :math:`t` values -:math:`|t|`_. -There are 2 options for posting CICE results to CDash: 1) The automated -script, 2) The manual method. - -***************** -Automatic Script -***************** - -To automatically run the CICE tests, and post the results to the CICE Cdash dashboard, -users need to copy and run the ``run.suite`` script: - -.. code-block:: bash - - cp configuration/scripts/run.suite . - ./run.suite -m -testid -bc -bg - -The run.suite script does the following: - -- Creates a fresh clone of the CICE-Consortium repository -- ``cd`` to cloned repo -- run ``cice.setup`` to generate the base_suite directories. The output - is piped to ``log.suite`` -- Running ``cice.setup`` submits each individual job to the queue. -- ``run.suite`` monitors the queue manager to determine when all jobs have - finished (pings the queue manager once every 5 minutes). -- Once all jobs complete, cd to base_suite directory and run ``./results.csh`` -- Run ``./run_ctest.csh`` in order to post the test results to the CDash dashboard - -***************** -Manual Method -***************** - -To manually run the CICE tests and post the results to the CICE CDash dashboard, -users essentially just need to perform all steps available in run.suite, detailed below: - -- Pass the ``-report`` flag to cice.setup when running the ``base_suite`` test suite. - The ``-report`` flag copies the required CTest / CDash scripts to the suite - directory. -- ``cice.setup`` compiles the CICE code, and submits all of the jobs to the - queue manager. -- After every job has been submitted and completed, ``cd`` to the suite directory. -- Parse the results, by running ``./results.csh``. -- Run the CTest / CDash script ``./run_ctest.csh``. - -If the ``run_ctest.csh`` script is unable to post the testing results to the CDash -server, a message will be printed to the screen detailing instructions on how to attempt -to post the results from another server. If ``run_ctest.csh`` fails to submit the results, -it will generate a tarball ``cice_ctest.tgz`` that contains the necessary files for -submission. Copy this file to another server (CMake version 2.8+ required), extract the -archive, and run ``./run_ctest.csh -submit``. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -End-To-End Testing Procedure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of a step-by-step procedure for testing a code change that results -in non-bit-for-bit results: - -.. code-block:: bash - - # Create a baseline dataset (only necessary if no baseline exists on the system) - ./cice.setup -m onyx -ts base_suite -testid base0 -bg cicev6.0.0 -a - - # Check out the updated code, or clone from a pull request - - # Run the test with the new code - ./cice.setup -m onyx -ts base_suite -testid test0 -bc cicev6.0.0 -a - - # Check the results - cd base_suite.test0 - ./results.csh - - #### If the BFB tests fail, perform the compliance testing #### - # Create a QC baseline - ./cice.setup -m onyx -t smoke -g gx1 -p 44x1 -testid qc_base -s qc,medium -a - cd onyx_smoke_gx1_44x1_medium_qc.qc_base - ./cice.build - ./cice.submit - - # Check out the updated code or clone from a pull request - - # Create the t-test testing data - ./cice.setup -m onyx -t smoke -g gx1 -p 44x1 -testid qc_test -s qc,medium -a - cd onyx_smoke_gx1_44x1_medium_qc.qc_test - ./cice.build - ./cice.submit - - # Wait for runs to finish - - # Perform the QC test - cp configuration/scripts/tests/QC/cice.t-test.py - ./cice.t-test.py /p/work/turner/CICE_RUNS/onyx_smoke_gx1_44x1_medium_qc.qc_base \ - /p/work/turner/CICE_RUNS/onyx_smoke_gx1_44x1_medium_qc.qc_test - - # Example output: - INFO:__main__:Number of files: 1825 - INFO:__main__:Two-Stage Test Passed - INFO:__main__:Quadratic Skill Test Passed for Northern Hemisphere - INFO:__main__:Quadratic Skill Test Passed for Southern Hemisphere - INFO:__main__: - INFO:__main__:Quality Control Test PASSED - -.. _tabnamelist: - -------------------------- -Table of namelist options -------------------------- - -.. _tab-namelist: - -.. csv-table:: Table 8 - :header: "variable", "options/format", "description", "recommended value" - :widths: 15, 15, 30, 15 - - "*setup_nml*", " ", " ", " " - "", "", "*Time, Diagnostics*", "" - "``days_per_year``", "``360`` or ``365``", "number of days in a model year", "365" - "``use_leap_years``", "true/false", "if true, include leap days", "" - "``year_init``", "yyyy", "the initial year, if not using restart", "" - "``istep0``", "integer", "initial time step number", "0" - "``dt``", "seconds", "thermodynamics time step length", "3600." - "``npt``", "integer", "total number of time steps to take", "" - "``ndtd``", "integer", "number of dynamics/advection/ridging/steps per thermo timestep", "1" - "", "", "*Initialization/Restarting*", "" - "``runtype``", "``initial``", "start from ``ice_ic``", "" - "", "``continue``", "restart using ``pointer_file``", "" - "``ice_ic``", "``default``", "latitude and sst dependent", "default" - "", "``none``", "no ice", "" - "", "path/file", "restart file name", "" - "``restart``", "true/false", "initialize using restart file", "``.true.``" - "``use_restart_time``", "true/false", "set initial date using restart file", "``.true.``" - "``restart_format``", "nc", "read/write  restart files (use with PIO)", "" - "", "bin", "read/write binary restart files", "" - "``lcdf64``", "true/false", "if true, use 64-bit  format", "" - "``restart_dir``", "path/", "path to restart directory", "" - "``restart_ext``", "true/false", "read/write halo cells in restart files", "" - "``restart_file``", "filename prefix", "output file for restart dump", "‘iced’" - "``pointer_file``", "pointer filename", "contains restart filename", "" - "``dumpfreq``", "``y``", "write restart every ``dumpfreq_n`` years", "y" - "", "``m``", "write restart every ``dumpfreq_n`` months", "" - "", "``d``", "write restart every ``dumpfreq_n`` days", "" - "``dumpfreq_n``", "integer", "frequency restart data is written", "1" - "``dump_last``", "true/false", "if true, write restart on last time step of simulation", "" - "", "", "*Model Output*", "" - "``bfbflag``", "true/false", "for bit-for-bit diagnostic output", "" - "``diagfreq``", "integer", "frequency of diagnostic output in ``dt``", "24" - "", "*e.g.*, 10", "once every 10 time steps", "" - "``diag_type``", "``stdout``", "write diagnostic output to stdout", "" - "", "``file``", "write diagnostic output to file", "" - "``diag_file``", "filename", "diagnostic output file (script may reset)", "" - "``print_global``", "true/false", "print diagnostic data, global sums", "``.false.``" - "``print_points``", "true/false", "print diagnostic data for two grid points", "``.false.``" - "``latpnt``", "real", "latitude of (2) diagnostic points", "" - "``lonpnt``", "real", "longitude of (2) diagnostic points", "" - "``dbug``", "true/false", "if true, write extra diagnostics", "``.false.``" - "``histfreq``", "string array", "defines output frequencies", "" - "", "``y``", "write history every ``histfreq_n`` years", "" - "", "``m``", "write history every ``histfreq_n`` months", "" - "", "``d``", "write history every ``histfreq_n`` days", "" - "", "``h``", "write history every ``histfreq_n`` hours", "" - "", "``1``", "write history every time step", "" - "", "``x``", "unused frequency stream (not written)", "" - "``histfreq_n``", "integer array", "frequency history output is written", "" - "", "0", "do not write to history", "" - "``hist_avg``", "true", "write time-averaged data", "``.true.``" - "", "false", "write snapshots of data", "" - "``history_dir``", "path/", "path to history output directory", "" - "``history_file``", "filename prefix", "output file for history", "‘iceh’" - "``write_ic``", "true/false", "write initial condition", "" - "``incond_dir``", "path/", "path to initial condition directory", "" - "``incond_file``", "filename prefix", "output file for initial condition", "‘iceh’" - "``runid``", "string", "label for run (currently CESM only)", "" - "", "", "", "" - "*grid_nml*", "", "", "" - "", "", "*Grid*", "" - "``grid_format``", "``nc``", "read  grid and kmt files", "‘bin’" - "", "``bin``", "read direct access, binary file", "" - "``grid_type``", "``rectangular``", "defined in *rectgrid*", "" - "", "``displaced_pole``", "read from file in *popgrid*", "" - "", "``tripole``", "read from file in *popgrid*", "" - "", "``regional``", "read from file in *popgrid*", "" - "``grid_file``", "filename", "name of grid file to be read", "‘grid’" - "``kmt_file``", "filename", "name of land mask file to be read", "‘kmt’" - "``gridcpl_file``", "filename", "input file for coupling grid info", "" - "``kcatbound``", "``0``", "original category boundary formula", "0" - "", "``1``", "new formula with round numbers", "" - "", "``2``", "WMO standard categories", "" - "", "``-1``", "one category", "" - "", "", "", "" - "*domain_nml*", "", "", "" - "", "", "*Domain*", "" - "``nprocs``", "integer", "number of processors to use", "" - "``processor_shape``", "``slenderX1``", "1 processor in the y direction (tall, thin)", "" - "", "``slenderX2``", "2 processors in the y direction (thin)", "" - "", "``square-ice``", "more processors in x than y, :math:`\sim` square", "" - "", "``square-pop``", "more processors in y than x, :math:`\sim` square", "" - "``distribution_type``", "``cartesian``", "distribute blocks in 2D Cartesian array", "" - "", "``roundrobin``", "1 block per proc until blocks are used", "" - "", "``sectcart``", "blocks distributed to domain quadrants", "" - "", "``sectrobin``", "several blocks per proc until used", "" - "", "``rake``", "redistribute blocks among neighbors", "" - "", "``spacecurve``", "distribute blocks via space-filling curves", "" - "``distribution_weight``", "``block``", "full block size sets ``work_per_block``", "" - "", "``latitude``", "latitude/ocean sets ``work_per_block``", "" - "``ew_boundary_type``", "``cyclic``", "periodic boundary conditions in x-direction", "" - "", "``open``", "Dirichlet boundary conditions in x", "" - "``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "" - "", "``open``", "Dirichlet boundary conditions in y", "" - "", "``tripole``", "U-fold tripole boundary conditions in y", "" - "", "``tripoleT``", "T-fold tripole boundary conditions in y", "" - "``maskhalo_dyn``", "true/false", "mask unused halo cells for dynamics", "" - "``maskhalo_remap``", "true/false", "mask unused halo cells for transport", "" - "``maskhalo_bound``", "true/false", "mask unused halo cells for boundary updates", "" - "", "", "", "" - "*tracer_nml*", "", "", "" - "", "", "*Tracers*", "" - "``tr_iage``", "true/false", "ice age", "" - "``restart_age``", "true/false", "restart tracer values from file", "" - "``tr_FY``", "true/false", "first-year ice area", "" - "``restart_FY``", "true/false", "restart tracer values from file", "" - "``tr_lvl``", "true/false", "level ice area and volume", "" - "``restart_lvl``", "true/false", "restart tracer values from file", "" - "``tr_pond_cesm``", "true/false", "CESM melt ponds", "" - "``restart_pond_cesm``", "true/false", "restart tracer values from file", "" - "``tr_pond_topo``", "true/false", "topo melt ponds", "" - "``restart_pond_topo``", "true/false", "restart tracer values from file", "" - "``tr_pond_lvl``", "true/false", "level-ice melt ponds", "" - "``restart_pond_lvl``", "true/false", "restart tracer values from file", "" - "``tr_aero``", "true/false", "aerosols", "" - "``restart_aero``", "true/false", "restart tracer values from file", "" - "*thermo_nml*", "", "", "" - "", "", "*Thermodynamics*", "" - "``kitd``", "``0``", "delta function ITD approximation", "1" - "", "``1``", "linear remapping ITD approximation", "" - "``ktherm``", "``0``", "zero-layer thermodynamic model", "" - "", "``1``", "Bitz and Lipscomb thermodynamic model", "" - "", "``2``", "mushy-layer thermodynamic model", "" - "``conduct``", "``MU71``", "conductivity :cite:`MU71`", "" - "", "``bubbly``", "conductivity :cite:`PETB07`", "" - "``a_rapid_mode``", "real", "brine channel diameter", "0.5x10 :math:`^{-3}` m" - "``Rac_rapid_mode``", "real", "critical Rayleigh number", "10" - "``aspect_rapid_mode``", "real", "brine convection aspect ratio", "1" - "``dSdt_slow_mode``", "real", "drainage strength parameter", "-1.5x10 :math:`^{-7}` m/s/K" - "``phi_c_slow_mode``", ":math:`0<\phi_c < 1`", "critical liquid fraction", "0.05" - "``phi_i_mushy``", ":math:`0<\phi_i < 1`", "solid fraction at lower boundary", "0.85" - "", "", "", "" - "*dynamics_nml*", "", "", "" - "", "", "*Dynamics*", "" - "``kdyn``", "``0``", "dynamics OFF", "1" - "", "``1``", "EVP dynamics", "" - "", "``2``", "EAP dynamics", "" - "``revised_evp``", "true/false", "use revised EVP formulation", "" - "``ndte``", "integer", "number of EVP subcycles", "120" - "``advection``", "``remap``", "linear remapping advection", "‘remap’" - "", "``upwind``", "donor cell advection", "" - "``kstrength``", "``0``", "ice strength formulation :cite:`Hibler79`", "1" - "", "``1``", "ice strength formulation :cite:`Rothrock75`", "" - "``krdg_partic``", "``0``", "old ridging participation function", "1" - "", "``1``", "new ridging participation function", "" - "``krdg_redist``", "``0``", "old ridging redistribution function", "1" - "", "``1``", "new ridging redistribution function", "" - "``mu_rdg``", "real", "e-folding scale of ridged ice", "" - "``Cf``", "real", "ratio of ridging work to PE change in ridging", "17." - "", "", "", "" - "*shortwave_nml*", "", "", "" - "", "", "*Shortwave*", "" - "``shortwave``", "``default``", "NCAR CCSM3 distribution method", "" - "", "``dEdd``", "Delta-Eddington method", "" - "``albedo_type``", "``default``", "NCAR CCSM3 albedos", "‘default’" - "", "``constant``", "four constant albedos", "" - "``albicev``", ":math:`0<\alpha <1`", "visible ice albedo for thicker ice", "" - "``albicei``", ":math:`0<\alpha <1`", "near infrared ice albedo for thicker ice", "" - "``albsnowv``", ":math:`0<\alpha <1`", "visible, cold snow albedo", "" - "``albsnowi``", ":math:`0<\alpha <1`", "near infrared, cold snow albedo", "" - "``ahmax``", "real", "albedo is constant above this thickness", "0.3 m" - "``R_ice``", "real", "tuning parameter for sea ice albedo from Delta-Eddington shortwave", "" - "``R_pnd``", "real", "... for ponded sea ice albedo …", "" - "``R_snw``", "real", "... for snow (broadband albedo) …", "" - "``dT_mlt``", "real", ":math:`\Delta` temperature per :math:`\Delta` snow grain radius", "" - "``rsnw_mlt``", "real", "maximum melting snow grain radius", "" - "``kalg``", "real", "absorption coefficient for algae", "" - "", "", "", "" - "*ponds_nml*", "", "", "" - "", "", "*Melt Ponds*", "" - "``hp1``", "real", "critical ice lid thickness for topo ponds", "0.01 m" - "``hs0``", "real", "snow depth of transition to bare sea ice", "0.03 m" - "``hs1``", "real", "snow depth of transition to pond ice", "0.03 m" - "``dpscale``", "real", "time scale for flushing in permeable ice", ":math:`1\times 10^{-3}`" - "``frzpnd``", "``hlid``", "Stefan refreezing with pond ice thickness", "‘hlid’" - "", "``cesm``", "CESM refreezing empirical formula", "" - "``rfracmin``", ":math:`0 \le r_{min} \le 1`", "minimum melt water added to ponds", "0.15" - "``rfracmax``", ":math:`0 \le r_{max} \le 1`", "maximum melt water added to ponds", "1.0" - "``pndaspect``", "real", "aspect ratio of pond changes (depth:area)", "0.8" - "", "", "", "" - "*zbgc_nml*", "", "", "" - "", "", "*Biogeochemistry*", "" - "``tr_brine``", "true/false", "brine height tracer", "" - "``tr_zaero``", "true/false", "vertical aerosol tracers", "" - "``modal_aero``", "true/false", "modal aersols", "" - "``restore_bgc``", "true/false", "restore bgc to data", "" - "``solve_zsal`", "true/false", "update salinity tracer profile", "" - "``bgc_data_dir``", "path/", "data directory for bgc", "" - "``skl_bgc``", "true/false", "biogeochemistry", "" - "``sil_data_type``", "``default``", "default forcing value for silicate", "" - "", "``clim``", "silicate forcing from ocean climatology :cite:`GLBA06`", "" - "``nit_data_type``", "``default``", "default forcing value for nitrate", "" - "", "``clim``", "nitrate forcing from ocean climatology :cite:`GLBA06`", "" - "", "``sss``", "nitrate forcing equals salinity", "" - "``fe_data_type``", "``default``", "default forcing value for iron", "" - "", "``clim``", "iron forcing from ocean climatology", "" - "``bgc_flux_type``", "``Jin2006``", "ice–ocean flux velocity of :cite:`JDWSTWLG06`", "" - "", "``constant``", "constant ice–ocean flux velocity", "" - "``restart_bgc``", "true/false", "restart tracer values from file", "" - "``tr_bgc_C_sk``", "true/false", "algal carbon tracer", "" - "``tr_bgc_chl_sk``", "true/false", "algal chlorophyll tracer", "" - "``tr_bgc_Am_sk``", "true/false", "ammonium tracer", "" - "``tr_bgc_Sil_sk``", "true/false", "silicate tracer", "" - "``tr_bgc_DMSPp_sk``", "true/false", "particulate DMSP tracer", "" - "``tr_bgc_DMSPd_sk``", "true/false", "dissolved DMSP tracer", "" - "``tr_bgc_DMS_sk``", "true/false", "DMS tracer", "" - "``phi_snow``", "real", "snow porosity for brine height tracer", "" - "", "", "", "" - "*forcing_nml*", "", "", "" - "", "", "*Forcing*", "" - "``formdrag``", "true/false", "calculate form drag", "" - "``atmbndy``", "``default``", "stability-based boundary layer", "‘default’" - "", "``constant``", "bulk transfer coefficients", "" - "``fyear_init``", "yyyy", "first year of atmospheric forcing data", "" - "``ycycle``", "integer", "number of years in forcing data cycle", "" - "``atm_data_format``", "``nc``", "read  atmo forcing files", "" - "", "``bin``", "read direct access, binary files", "" - "``atm_data_type``", "``default``", "constant values defined in the code", "" - "", "``LYq``", "AOMIP/Large-Yeager forcing data", "" - "", "``monthly``", "monthly forcing data", "" - "", "``ncar``", "NCAR bulk forcing data", "" - "", "``oned``", "column forcing data", "" - "``atm_data_dir``", "path/", "path to atmospheric forcing data directory", "" - "``calc_strair``", "true", "calculate wind stress and speed", "" - "", "false", "read wind stress and speed from files", "" - "``highfreq``", "true/false", "high-frequency atmo coupling", "" - "``natmiter``", "integer", "number of atmo boundary layer iterations", "" - "``calc_Tsfc``", "true/false", "calculate surface temperature", "``.true.``" - "``precip_units``", "``mks``", "liquid precipitation data units", "" - "", "``mm_per_month``", "", "" - "", "``mm_per_sec``", "(same as MKS units)", "" - "``tfrz_option``", "``minus1p8``", "constant ocean freezing temperature (:math:`-1.8^{\circ} C`)", "" - "", "``linear_salt``", "linear function of salinity (ktherm=1)", "" - "", "``mushy_layer``", "matches mushy-layer thermo (ktherm=2)", "" - "``ustar_min``", "real", "minimum value of ocean friction velocity", "0.0005 m/s" - "``fbot_xfer_type``", "``constant``", "constant ocean heat transfer coefficient", "" - "", "``Cdn_ocn``", "variable ocean heat transfer coefficient", "" - "``update_ocn_f``", "true", "include frazil water/salt fluxes in ocn fluxes", "" - "", "false", "do not include (when coupling with POP)", "" - "``l_mpond_fresh``", "true", "retain (topo) pond water until ponds drain", "" - "", "false", "release (topo) pond water immediately to ocean", "" - "``oceanmixed_ice``", "true/false", "active ocean mixed layer calculation", "``.true.`` (if uncoupled)" - "``ocn_data_format``", "``nc``", "read  ocean forcing files", "" - "", "``bin``", "read direct access, binary files", "" - "``sss_data_type``", "``default``", "constant values defined in the code", "" - "", "``clim``", "climatological data", "" - "", "``near``", "POP ocean forcing data", "" - "``sst_data_type``", "``default``", "constant values defined in the code", "" - "", "``clim``", "climatological data", "" - "", "``ncar``", "POP ocean forcing data", "" - "``ocn_data_dir``", "path/", "path to oceanic forcing data directory", "" - "``oceanmixed_file``", "filename", "data file containing ocean forcing data", "" - "``restore_sst``", "true/false", "restore sst to data", "" - "``trestore``", "integer", "sst restoring time scale (days)", "" - "``restore_ice``", "true/false", "restore ice state along lateral boundaries", "" - "", "", "", "" - "*icefields_tracer_nml*", "", "", "" - "", "", "*History Fields*", "" - "``f_``", "string", "frequency units for writing ```` to history", "" - "", "``y``", "write history every ``histfreq_n`` years", "" - "", "``m``", "write history every ``histfreq_n`` months", "" - "", "``d``", "write history every ``histfreq_n`` days", "" - "", "``h``", "write history every ``histfreq_n`` hours", "" - "", "``1``", "write history every time step", "" - "", "``x``", "do not write ```` to history", "" - "", "``md``", "*e.g.,* write both monthly and daily files", "" - "``f__ai``", "", "grid cell average of ```` (:math:`\times a_i`)", "" - diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index 08dc141b2..49cbd11ac 100644 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -1,10 +1,293 @@ :tocdepth: 3 -.. _running: +.. _running_cice: -Execution procedures +Running CICE ==================== +Quick-start instructions are provided in the :ref:`quickstart` section. + +.. _scripts: + +Scripts +------- + +The CICE scripts are written to allow quick setup of cases and tests. Once a case is +generated, users can manually modify the namelist and other files to custom configure +the case. Several settings are available via scripts as well. + +Overview +~~~~~~~~ + +Most of the scripts that configure, build and run CICE are contained in +the directory **configuration/scripts/**, except for **cice.setup**, which is +in the main directory. **cice.setup** is the main script that generates a case. + +Users may need to port the scripts to their local machine. +Specific instructions for porting are provided in :ref:`porting`. + +``cice.setup -h`` will provide the latest information about how to use the tool. +``cice.setup --help`` will provide an extended version of the help. +There are three usage modes, + +* ``--case`` or ``-c`` creates individual stand alone cases. +* ``--test`` creates individual tests. Tests are just cases that have some extra automation in order to carry out particular tests such as exact restart. +* ``--suite`` creates a test suite. Test suites are predefined sets of tests and ``--suite`` provides the ability to quickly setup, build, and run a full suite of tests. + +All modes will require use of ``--mach`` or ``-m`` to specify the machine and case and test modes +can use ``--set`` or ``-s`` to define specific options. ``--test`` and ``--suite`` will require ``--testid`` to be set +and both of the test modes can use ``--bdir``, ``--bgen``, ``--bcmp``, and ``--diff`` to generate (save) results and compare results with prior results. +Testing will be described in greater detail in the :ref:`testing` section. + +Again, ``cice.setup --help`` will show the latest usage information including +the available ``--set`` options, the current ported machines, and the test choices. + +To create a case, run **cice.setup**:: + + cice.setup -c mycase -m machine + cd mycase + +Once a case/test is created, several files are placed in the case directory + +- **env.[machine]** defines the environment +- **cice.settings** defines many variables associated with building and running the model +- **makdep.c** is a tool that will automatically generate the make dependencies +- **Macros.[machine]** defines the Makefile macros +- **Makefile** is the makefile used to build the model +- **cice.build** is a script that builds and compiles the model +- **ice\_in** is the namelist input file +- **setup\_run\_dirs.csh** is a script that will create the run directories. This will be called automatically from the **cice.run** script if the user does not invoke it. +- **cice.run** is a batch run script +- **cice.submit** is a simple script that submits the cice.run script + +All scripts and namelist are fully resolved in the case. Users can edit any +of the files in the case directory manually to change the model configuration, +build options, or batch settings. The file +dependency is indicated in the above list. For instance, if any of the files before +**cice.build** in the list are edited, **cice.build** should be rerun. + +The **casescripts/** directory holds scripts used to create the case and can +largely be ignored. Once a case is created, the **cice.build** script should be run +interactively and then the case should be submitted by executing the +**cice.submit** script interactively. The **cice.submit** script +simply submits the **cice.run script**. +You can also submit the **cice.run** script on the command line. + +Some hints: + +- To change the tracer numbers or block sizes required at build time, edit the **cice.settings** file. +- To change namelist, manually edit the **ice_in** file +- To change batch settings, manually edit the top of the **cice.run** or **cice.test** (if running a test) file +- To turn on the debug compiler flags, set ``ICE_BLDDEBUG`` in **cice.setttings** to true +- To change compiler options, manually edit the Macros file +- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true. To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false + +To build and run:: + + ./cice.build + ./cice.submit + +The build and run log files will be copied into the logs directory in the case directory. +Other model output will be in the run directory. The run directory is set in **cice.settings** +via the ``ICE_RUNDIR`` variable. To modify the case setup, changes should be made in the +case directory, NOT the run directory. + +.. _case_options: + +Command Line Options +~~~~~~~~~~~~~~~~~~~~ + +``cice.setup -h`` provides a summary of the command line options. There are three different modes, ``--case``, ``--test``, and ``--suite``. This section provides details about the relevant options for setting up cases with examples. +Testing will be described in greater detail in the :ref:`testing` section. + +``--help``, ``-h`` + prints ``cice.setup`` help information to the terminal and exits. + +``--version`` + prints the CICE version to the terminal and exits. + +``--setvers VERSION`` + updates the CICE version in your sandbox. The version should be some like n.m.p.string. + +``--case``, ``-c`` CASE + specifies the case name. This can be either a relative path of an absolute path. This cannot be used with --test or --suite. Either ``--case``, ``--test``, or ``--suite`` is required. + +``--mach``, ``-m`` MACHINE + specifies the machine name. This should be consistent with the name defined in the Macros and env files in **configurations/scripts/machines**. This is required in all modes. + +``--env``, ``-e`` ENVIRONMENT1,ENVIRONMENT2,ENVIRONMENT3 + specifies the environment or compiler associated with the machine. This should be consistent with the name defined in the Macros and env files in **configurations/scripts/machines**. Each machine can have multiple supported environments including support for different compilers or other system setups. When used with ``--suite`` or ``--test``, the ENVIRONMENT can be a set of comma deliminated values with no spaces and the tests will then be run for all of those environments. With ``--case``, only one ENVIRONMENT should be specified. (default is intel) + +``--pes``, ``-p`` MxN[[xBXxBY[xMB] + specifies the number of tasks and threads the case should be run on. This only works with ``--case``. The format is tasks x threads or "M"x"N" where M is tasks and N is threads and both are integers. BX, BY, and MB can also be set via this option where BX is the x-direction blocksize, BY is the y-direction blocksize, and MB is the max-blocks setting. If BX, BY, and MB are not set, they will be computed automatically based on the grid size and the task/thread count. More specifically, this option has three modes, --pes MxN, --pes MxNxBXxBY, and --pes MxNxBXxBYxMB. (default is 4x1) + +``--acct`` ACCOUNT + specifies a batch account number. This is optional. See :ref:`account` for more information. + +``--grid``, ``-g`` GRID + specifies the grid. This is a string and for the current CICE driver, gx1 and gx3 are supported. (default = gx3) + +``--set``, ``-s`` SET1,SET2,SET3 + specifies the optional settings for the case. The settings for ``--suite`` are defined in the suite file. Multiple settings can be specified by providing a comma deliminated set of values without spaces between settings. The available settings are in **configurations/scripts/options** and ``cice.setup --help`` will also list them. These settings files can change either the namelist values or overall case settings (such as the debug flag). + +For CICE, when setting up cases, the ``--case`` and ``--mach`` must be specified. +It's also recommended that ``--env`` be set explicitly as well. +``--pes`` and ``--grid`` can be very useful. +``--acct`` is not normally used. A more convenient method +is to use the **~/cice\_proj** file, see :ref:`account`. The ``--set`` option can be +extremely handy. The ``--set`` options are documented in :ref:`settings`. + +.. _settings: + +Preset Options +~~~~~~~~~~~~~~ + +There are several preset options. These are hardwired in +**configurations/scripts/options** and are specfied for a case or test by +the ``--set`` command line option. You can see the full list of settings +by doing ``cice.setup --help``. + +The default CICE namelist and CICE settings are specified in the +files **configuration/scripts/ice_in** and +**configuration/scripts/cice.settings** respectively. When picking a +preset setting (option), the set_env.setting and set_nml.setting will be used to +change the defaults. This is done as part of the ``cice.setup`` and the +modifications are resolved in the **cice.settings** and **ice_in** file placed in +the case directory. If multiple options are chosen and then conflict, then the last +option chosen takes precedent. Not all options are compatible with each other. + +Some of the options are + +``debug`` which turns on the compiler debug flags + +``short``, ``medium``, ``long`` which change the batch time limit + +``gx3`` and ``gx1`` are associate with grid specific settings + +``diag1`` which turns on diagnostics each timestep + +``leap`` which turns on the leap year + +``pondcesm``, ``pondlvl``, ``pondtopo`` which turn on the various pond schemes + +``run10day``, ``run1year``, etc which specifies a run length + +``dslenderX1``, ``droundrobin``, ``dspacecurve``, etc specify decomposition options + +``dynevp``, ``dyneap``, ``dynoff``, and ``dynrevp`` specify dynamics choices + +``therm0``, ``thermBL``, and ``thermmushy`` are thermodynamics settings + +``swccsm3`` which turns on the ccsm3 shortwave and albedo computation + +``bgc*`` which turns of various bgc configurations + +and there are others. These may change as needed. Use ``cice.setup --help`` to see the latest. To add a new option, just add the appropriate file in **configuration/scripts/options**. For more information, see :ref:`dev_options` + +Examples +~~~~~~~~~ + +The simplest case is just to setup a default configurations specifying the +case name, machine, and environment:: + + cice.setup --case mycase1 --mach spirit --env intel + +To add some optional settings, one might do:: + + cice.setup --case mycase2 --mach spirit --env intel --set debug,diag1,run1year,pondtopo + +Once the cases are created, users are free to modify the cice.settings and ice_in namelist to further modify their setup. + +.. _porting: + +Porting +------- + +To port, an **env.[machine]_[environment]** and **Macros.[machine]_[environment}** file have to be added to the +**configuration/scripts/machines/** directory and the +**configuration/scripts/cice.batch.csh** file needs to be modified. +In general, the machine is specified in ``cice.setup`` with ``--mach`` +and the environment (compiler) is specified with ``--env``. + +- cd to **configuration/scripts/machines/** + +- Copy an existing env and a Macros file to new names for your new machine + +- Edit your env and Macros files + +- cd .. to **configuration/scripts/** + +- Edit the **cice.batch.csh** script to add a section for your machine + with batch settings and job launch settings + +- Download and untar a forcing dataset to the location defined by + ``ICE_MACHINE_INPUTDATA`` in the env file + +In fact, this process almost certainly will require some iteration. The easiest way +to carry this out is to create an initial set of changes as described above, then +create a case and manually modify the **env.[machine]** file and **Macros.[machine]** +file until the case can build and run. Then copy the files from the case +directory back to **configuration/scripts/machines/** and update +the **configuration/scripts/cice.batch.csh** file, retest, +and then add and commit the updated machine files to the repository. + +.. _account: + +Machine Account Settings +~~~~~~~~~~~~~~~~~~~~~~~~ + +The machine account default is specified by the variable ``ICE_MACHINE_ACCT`` in +the **env.[machine]** file. The easiest way to change a user's default is to +create a file in your home directory called **.cice\_proj** and add your +preferred account name to the first line. +There is also an option (``--acct``) in **cice.setup** to define the account number. +The order of precedent is **cice.setup** command line option, +**.cice\_proj** setting, and then value in the **env.[machine]** file. + +.. _force: + +Forcing data +------------ + +The input data space is defined on a per machine basis by the ``ICE_MACHINE_INPUTDATA`` +variable in the **env.[machine]** file. That file space is often shared among multiple +users, and it can be desirable to consider using a common file space with group read +and write permissions such that a set of users can update the inputdata area as +new datasets are available. + +CICE input datasets are stored on an anonymous ftp server. More information about +how to download the input data can be found at https://github.com/CICE-Consortium/CICE/wiki. +Test forcing datasets are available for various grids at the ftp site. +These data files are designed only for testing the code, not for use in production runs +or as observational data. Please do not publish results based on these data sets. + + +Run Directories +--------------- + +The **cice.setup** script creates a case directory. However, the model +is actually built and run under the ``ICE_OBJDIR`` and ``ICE_RUNDIR`` directories +as defined in the **cice.settings** file. + +Build and run logs will be copied from the run directory into the case **logs/** +directory when complete. + + +Local modifications +------------------- + +Scripts and other case settings can be changed manually in the case directory and +used. Source code can be modified in the main sandbox. When changes are made, the code +should be rebuilt before being resubmitted. It is always recommended that users +modify the scripts and input settings in the case directory, NOT the run directory. +In general, files in the run directory are overwritten by versions in the case +directory when the model is built, submitted, and run. + + + +Old Notes +------------ + To compile and execute the code: in the source directory, #. Download the forcing data used for testing from the CICE-Consortium github page, @@ -150,18 +433,3 @@ filename can be assigned to ice\_ic in **ice\_in**. Consult Section :ref:`init` for more details. Restarts are exact for MPI or single processor runs. -~~~~~~~ -Scripts -~~~~~~~ - -~~~~~~~~~~~ -Directories -~~~~~~~~~~~ - -~~~~~~~~~~~~~~~~~~~ -Local modifications -~~~~~~~~~~~~~~~~~~~ - -~~~~~~~~~~~~ -Forcing data -~~~~~~~~~~~~ diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 83fecf391..775811967 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -1,266 +1,527 @@ :tocdepth: 3 +.. _testing: + Testing CICE -============ +================ + +This section documents primarily how to use the CICE scripts to carry +out CICE testing. Exactly what to test is a separate question and +depends on the kinds of code changes being made. Prior to merging +changes to the CICE Consortium master, changes will be reviewed and +developers will need to provide a summary of the tests carried out. -Version 6, August 2017 -This documents how to use the testing features developed for the -CICE Consortium CICE sea ice model. +There is a base suite of tests provided by default with CICE and this +may be a good starting point for testing. -.. _basic: +The testing scripts support several features + - Ability to test individual (via ``--test``)or multiple tests (via ``--suite``) + using an input file to define the suite + - Ability to use test suites defined in the package or test suites defined by the user + - Ability to store test results for regresssion testing (``--bgen``) + - Ability to compare results to prior baselines to verify bit-for-bit (``--bcmp``) + - Ability to define where baseline tests are stored (``--bdir``) + - Ability to compare tests against each other (``--diff``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Individual tests and test suites -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _indtests: + +Individual Tests +---------------- The CICE scripts support both setup of individual tests as well as test suites. Individual -tests are run from the command line like +tests are run from the command line:: - > cice.setup -t smoke -m wolf -g gx3 -p 8x2 -s diag1,run5day -testid myid + ./cice.setup --test smoke --mach conrad --env cray --set diag1,debug --testid myid -where -m designates a specific machine. Test suites are multiple tests that are specified in -an input file and are started on the command line like +Tests are just like cases but have some additional scripting around them. Individual +tests can be created and manually modified just like cases. +Many of the command line arguments for individual tests +are similar to :ref:`case_options` for ``--case``. +For individual tests, the following command line options can be set - > cice.setup -ts base_suite -m wolf -testid myid +``--test`` TESTNAME + specifies the test type. This is probably either smoke or restart but see `cice.setup --help` for the latest. This is required instead of ``--case``. -cice.setup with -t or -ts require a testid to uniquely name test directories. The format -of the case directory name for a test will always be -${machine}_${test}_${grid}_${pes}_${soptions}.${testid} +``--testid`` ID + specifies the testid. This is required for every use of ``--test`` and ``--suite``. This is a user defined string that will allow each test to have a unique case and run directory name. This is also required. -To build and run a test, the process is the same as a case, - cd into the test directory, - - run cice.build - - run cice.submit +``--mach`` MACHINE (see :ref:`case_options`) + +``--env`` ENVIRONMENT1 (see :ref:`case_options`) + +``--set`` SET1,SET2,SET3 (see :ref:`case_options`) + +``--acct`` ACCOUNT (see :ref:`case_options`) + +``--grid`` GRID (see :ref:`case_options`) + +``--pes`` MxNxBXxBYxMB (see :ref:`case_options`) + +There are several additional options that come with ``--test`` that are not available +with ``--case`` for regression and comparision testing, + +``--bdir`` DIR + specifies the top level location of the baseline results. This is used in conjuction with ``--bgen`` and ``--bcmp``. The default is set by ICE_MACHINE_BASELINE in the env.[machine]_[environment] file. + +``--bgen`` DIR + specifies the name of the directory under [bdir] where test results will be stored. When this flag is set, it automatically creates that directory and stores results from the test under that directory. If DIR is set to ``default``, then the scripts will automatically generate a directory name based on the CICE hash and the date and time. This can be useful for tracking the baselines by hash. + +``--bcmp`` DIR + specifies the name of the directory under [bdir] that the current tests will be compared to. When this flag is set, it automatically invokes regression testing and compares results from the current test to those prior results. If DIR is set to ``default``, then the script will automatically generate the last directory name in the [bdir] directory. This can be useful for automated regression testing. + +``--diff`` LONG_TESTNAME + invokes a comparison against another local test. This allows different tests to be compared to each other for bit-for-bit-ness. This is different than ``--bcmp``. ``--bcmp`` is regression testing, comparing identical test results between different model versions. ``--diff`` allows comparison of two different test cases against each other. For instance, different block sizes, decompositions, and other model features are expected to produced identical results and ``--diff`` supports that testing. The restrictions for use of ``--diff`` are that the test has to already be completed and the testid has to match. The LONG_TESTNAME string should be of format [test]_[grid]_[pes]_[sets]. The [machine], [env], and [testid] will be added to that string to complete the testname being compared. (See also :ref:`examplediff`) + +The format of the case directory name for a test will always be +``[machine]_[env]_[test]_[grid]_[pes]_[sets].[testid]`` +The [sets] will always be sorted alphabetically by the script so ``--set debug,diag1`` and +``--set diag1,debug`` produces the same testname and test with _debug_diag1 in that order. -The test results will be generated in a local file called "test_output". +To build and run a test, the process is the same as a case. cd to the +test directory, run the build script, and run the submit script:: -When running a test suite, the cice.setup command line automatically generates all the tests -under a directory names ${test_suite}.${testid}. It then automatically builds and submits all -tests. When the tests are complete, run the results.csh script to see the results from all the -tests. + cd [test_case] + ./cice.build + ./cice.submit -Tests are defined under configuration/scripts/tests. The tests currently supported are: - smoke - Runs the model for default length. The length and options can - be set with the -s commmand line option. The test passes if the +The test results will be generated in a local file called **test_output**. +To check those results:: + + cat test_output + +Tests are defined under **configuration/scripts/tests/**. Some tests currently supported are: + +- smoke - Runs the model for default length. The length and options can + be set with the ``--set`` command line option. The test passes if the model completes successfully. - restart - Runs the model for 10 days, writing a restart file at day 5 and - again at day 10. Runs the model a second time starting from the - day 5 restart and writing a restart at day 10 of the model run. - The test passes if both the 10 day and 5 day restart run complete and - if the restart files at day 10 from both runs are bit-for-bit identical. +- restart - Runs the model for 10 days, writing a restart file at the end of day 5 and + again at the end of the run. Runs the model a second time starting from the + day 5 restart and writes a restart at then end of day 10 of the model run. + The test passes if both runs complete and + if the restart files at the end of day 10 from both runs are bit-for-bit identical. +- decomp - Runs a set of different decompositions on a given configuration -Please run './cice.setup -h' for additional details. +Please run ``./cice.setup --help`` for the latest information. -.. _additional: -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Additional testing options -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Adding a new test +~~~~~~~~~~~~~~~~~~~~~~~~ -There are several additional options on the cice.setup command line for testing that -provide the ability to regression test and compare tests to each other. +See :ref:`dev_testing` - -bd defines a baseline directory where tests can be stored for regression testing - - -bg defines a version name that where the current tests can be saved for regression testing - - -bc defines a version name that the current tests should be compared to for regression testing - - -td provides a way to compare tests with each other -To use -bg, - > cice.setup -ts base_suite -m wolf -testid v1 -bg version1 -bd $SCRATCH/CICE_BASELINES - will copy all the results from the test suite to $SCRATCH/CICE_BASELINES/version1. +Example. Basic default single test +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Define the test, mach, env, and testid. +:: + + ./cice.setup --test smoke --mach wolf --env gnu --testid t00 + cd wolf_gnu_smoke_col_1x1.t00 + ./cice.build + ./cice.submit + ./cat test_output + + +Example. Simple test with some options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add ``--set`` +:: + + ./cice.setup --test smoke --mach wolf --env gnu --set diag1,debug --testid t00 + cd wolf_gnu_smoke_col_1x1_debug_diag1.t00 + ./cice.build + ./cice.submit + ./cat test_output + -To use -bc, - > cice.setup -ts base_suite -m wolf -testid v2 -bc version1 -bd $SCRATCH/CICE_BASELINES - will compare all the results from this test suite to results saved before in $SCRATCH/CICE_BASELINES/version1. +Example. Single test, generate a baseline dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --bc and -bg can be combined, - >cice.setup -ts base_suite -m wolf -testid v2 -bg version2 -bc version1 -bd $SCRATCH/CICE_BASELINES - will save the current results to $SCRATCH/CICE_BASELINES/version2 and compare the current results to - results save before in $SCRATCH/CICE_BASELINES/version1. +Add ``--bgen`` +:: --bg, -bc, and -bd are used for regression testing. There is a default -bd on each machine. + ./cice.setup --test smoke --mach wolf -env gnu --bgen cice.v01 --testid t00 --set diag1 + cd wolf_gnu_smoke_col_1x1_diag1.t00 + ./cice.build + ./cice.submit + ./cat test_output --td allows a user to compare one test result to another. For instance, - > cice.setup -t smoke -m wolf -g gx3 -p 8x2 -s run5day -testid t01 - > cice.setup -t smoke -m wolf -g gx3 -p 4x2 -s run5day -testid t01 -td smoke_gx3_8x2_run5day -An additional check will be done for the second test (because of the -td argument), and it will compare -the output from the first test "smoke_gx3_8x2_run5day" to the output from it's test "smoke_gx3_4x2_run5day" -and generate a result for that. It's important that the first test complete before the second test is -done. Also, the -td option works only if the testid and the machine are the same for the baseline -run and the current run. +Example. Single test, compare results to a prior baseline. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _format: +Add ``--bcmp``. For this to work, +the prior baseline must exist and have the exact same base testname +[machine]_[env]_[test]_[grid]_[pes]_[sets] +:: -~~~~~~~~~~~~~~~~~ -Test suite format -~~~~~~~~~~~~~~~~~ + ./cice.setup --test smoke --mach wolf -env gnu --bcmp cice.v01 --testid t01 --set diag1 + cd wolf_gnu_smoke_col_1x1_diag1.t01 + ./cice.build + ./cice.submit + ./cat test_output -The format for the test suite file is relatively simple. It is a text file with white space delimited -columns like, -.. _tab-test: +Example. Simple test, generate a baseline dataset and compare to a prior baseline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. csv-table:: Table 7 - :header: "#Test", "Grid", "PEs", "Sets", "BFB-compare" - :widths: 7, 7, 7, 15, 15 +Use ``--bgen`` and ``--bcmp``. The prior baseline must exist already. +:: - "smoke", "gx3", "8x2", "diag1,run5day", "" - "smoke", "gx3", "8x2", "diag24,run1year,medium", "" - "smoke", "gx3", "4x1", "debug,diag1,run5day", "" - "smoke", "gx3", "8x2", "debug,diag1,run5day", "" - "smoke", "gx3", "4x2", "diag1,run5day", "smoke_gx3_8x2_diag1_run5day" - "smoke", "gx3", "4x1", "diag1,run5day,thread", "smoke_gx3_8x2_diag1_run5day" - "smoke", "gx3", "4x1", "diag1,run5day", "smoke_gx3_4x1_diag1_run5day_thread" - "restart", "gx3", "8x1", "", "" - "restart", "gx3", "4x2", "debug", "" + ./cice.setup --test smoke --mach wolf -env gnu --bgen cice.v02 --bcmp cice.v01 --testid t02 --set diag1 + cd wolf_gnu_smoke_col_1x1_diag1.t02 + ./cice.build + ./cice.submit + ./cat test_output +.. _examplediff: -The first column is the test name, the second the grid, the third the pe count, the fourth column is -the -s options and the fifth column is the -td argument. The fourth and fifth columns are optional. -The argument to -ts defines which filename to choose and that argument can contain a path. cice.setup -will also look for the filename in configuration/scripts/tests where some preset test suites are defined. +Example. Simple test, comparison against another test +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -~~~~~~~~~~~~~~~~~~~~~~~~~~ -Example Tests (Quickstart) -~~~~~~~~~~~~~~~~~~~~~~~~~~ +``--diff`` provides a way to compare tests with each other. +For this to work, the tests have to be run in a specific order and +the testids need to match. The test +is always compared relative to the current case directory. -********************************************** -To generate a baseline dataset for a test case -********************************************** +To run the first test, +:: -./cice.setup -t smoke -m wolf -bg cicev6.0.0 -testid t00 + ./cice.setup --test smoke --mach wolf -env gnu --testid tx01 --set debug + cd wolf_gnu_smoke_col_1x1_debug.tx01 + ./cice.build + ./cice.submit + ./cat test_output -cd wolf_smoke_gx3_4x1.t00 +Then to run the second test and compare to the results from the first test +:: -./cice.build + ./cice.setup --test smoke --mach wolf -env gnu --testid tx01 --diff smoke_col_1x1_debug + cd wolf_gnu_smoke_col_1x1.tx01 + ./cice.build + ./cice.submit + ./cat test_output -./cice.submit +The scripts will add a [machine]_[environment] to the beginning of the diff +argument and the same testid to the end of the diff argument. Then the runs +will be compared for bit-for-bit and a result will be produced in test_output. -# After job finishes, check output -cat test_output +.. _testsuites: -**************************************************** -To run a test case and compare to a baseline dataset -**************************************************** +Test suites +------------ -./cice.setup -t smoke -m wolf -bc cicev6.0.0 -testid t01 +Test suites support running multiple tests specified via +an input file. When invoking the test suite option (``--suite``) with **cice.setup**, +all tests will be created, built, and submitted automatically under +a local directory called testsuite.[testid] as part of involing the suite.:: -cd wolf_smoke_gx3_4x1.t01 + ./cice.setup --suite base_suite --mach wolf --env gnu --testid myid -./cice.build +Like an individual test, the ``--testid`` option must be specified and can be any +string. Once the tests are complete, results can be checked by running the +results.csh script in the [suite_name].[testid]:: -./cice.submit + cd testsuite.[testid] + ./results.csh -# After job finishes, check output +Multiple suites are supported on the command line as comma separated arguments:: -cat test_output + ./cice.setup --suite base_suite,decomp_suite --mach wolf --env gnu --testid myid -********************************************* -To run a test suite to generate baseline data -********************************************* +If a user adds ``--set`` to the suite, all tests in that suite will add that option:: -./cice.setup -m wolf -ts base_suite -testid t02 -bg cicev6.0.0bs + ./cice.setup --suite base_suite,decomp_suite --mach wolf --env gnu --testid myid -s debug -cd base_suite.t02 +The option settings defined in the suite have precendent over the command line +values if there are conflicts. -# Once all jobs finish, concatenate all output +The predefined test suites are defined under **configuration/scripts/tests** and +the files defining the suites +have a suffix of .ts in that directory. The format for the test suite file +is relatively simple. +It is a text file with white space delimited +columns that define a handful of values in a specific order. +The first column is the test name, the second the grid, the third the pe count, +the fourth column is +the ``--set`` options and the fifth column is the ``--diff`` argument. +The fourth and fifth columns are optional. +Lines that begin with # or are blank are ignored. For example, +:: -./results.csh # All tests results will be stored in results.log + #Test Grid PEs Sets Diff + smoke col 1x1 diag1 + smoke col 1x1 diag1,run1year smoke_col_1x1_diag1 + smoke col 1x1 debug,run1year + restart col 1x1 debug + restart col 1x1 diag1 + restart col 1x1 pondcesm + restart col 1x1 pondlvl + restart col 1x1 pondtopo -# To plot a timeseries of "total ice extent", "total ice area", and "total ice volume" +The argument to ``--suite`` defines the test suite (.ts) filename and that argument +can contain a path. +**cice.setup** +will look for the filename in the local directory, in **configuration/scripts/tests/**, +or in the path defined by the ``--suite`` option. -./timeseries.csh +Because many of the command line options are specified in the input file, ONLY the +following options are valid for suites, -ls \*.png +``--suite`` filename + required, input filename with list of suites -*********************************************** -To run a test suite to compare to baseline data -*********************************************** +``--mach`` MACHINE + required -./cice.setup -m wolf -ts base_suite -testid t03 -bc cicev6.0.0bs +``--env`` ENVIRONMENT1,ENVIRONMENT2 + strongly recommended -cd base_suite.t03 +``--set`` SET1,SET2 + optional -# Once all jobs finish, concatenate all output +``--acct`` ACCOUNT + optional -./results.csh # All tests results will be stored in results.log +``--testid`` ID + required -# To plot a timeseries of "total ice extent", "total ice area", and "total ice volume" +``--bdir`` DIR + optional, top level baselines directory and defined by default by ICE_MACHINE_BASELINE in **env.[machine]_[environment]**. -./timeseries.csh +``--bgen`` DIR + recommended, test output is copied to this directory under [bdir] -ls \*.png +``--bcmp`` DIR + recommended, test output are compared to prior results in this directory under [bdir] -************************** -To compare to another test -************************** -`First:` +``--report`` + This is only used by ``--suite`` and when set, invokes a script that sends the test results to the results page when all tests are complete. Please see :ref:`testreporting` for more information. -./cice.setup -m wolf -t smoke -testid t01 -p 8x2 +Please see :ref:`case_options` and :ref:`indtests` for more details about how these options are used. -cd wolf_smoke_gx3_8x2.t01 -./cice.build +Example. Basic test suite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -./cice.submit +Specify suite, mach, env, testid. +:: -# After job finishes, check output + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a + cd base_suite.v01a + #wait for runs to complete + ./results.csh -cat test_output -`Then, do the comparison:` +Example. Basic test suite on multiple environments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -./cice.setup -m wolf -t smoke -testid t01 -td smoke_gx3_8x2 -s thread -p 4x1 +Specify multiple envs. +:: -cd wolf_smoke_gx3_4x1_thread.t01 + ./cice.setup --suite base_suite --mach conrad --env cray,pgi,intel,gnu --testid v01a + cd base_suite.v01a + #wait for runs to complete + ./results.csh -./cice.build +Each env can be run as a separate invokation of `cice.setup` but if that +approach is taken, it is recommended that different testids be used. -./cice.submit -# After job finishes, check output +Example. Basic test suite with generate option defined +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -cat test_output +Add ``--set`` +:: -****************** -Additional Details -****************** + ./cice.setup --suite base_suite --mach conrad --env gnu --testid v01b --set diag1 + cd base_suite.v01b + #wait for runs to complete + ./results.csh + +If there are conflicts between the ``--set`` options in the suite and on the command line, +the suite will take precedent. + + +Example. Multiple test suites from a single command line +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add comma delimited list of suites +:: + + ./cice.setup --suite base_suite,decomp_suite --mach conrad --env gnu --testid v01c + cd base_suite.v01c + #wait for runs to complete + ./results.csh + +If there are redundant tests in multiple suites, the scripts will understand that and only +create one test. + + +Example. Basic test suite, store baselines in user defined name +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add ``--bgen`` +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a + cd base_suite.v01a + #wait for runs to complete + ./results.csh + +This will store the results in the default [bdir] directory under the subdirectory cice.v01a. + +Example. Basic test suite, store baselines in user defined top level directory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add ``--bgen`` and ``--bdir`` +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a --bdir /tmp/user/CICE_BASELINES + cd base_suite.v01a + #wait for runs to complete + ./results.csh + +This will store the results in /tmp/user/CICE_BASELINES/cice.v01a. + + +Example. Basic test suite, store baselines in auto-generated directory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add ``--bgen default`` +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen default + cd base_suite.v01a + #wait for runs to complete + ./results.csh + +This will store the results in the default [bdir] directory under a directory name generated by the script that includes the hash and date. + + +Example. Basic test suite, compare to prior baselines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add ``--bcmp`` +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v02a --bcmp cice.v01a + cd base_suite.v02a + #wait for runs to complete + ./results.csh + +This will compare to results saved in the baseline [bdir] directory under +the subdirectory cice.v01a. You can use other regression options as well +(``--bdir`` and ``--bgen``) + + +Example. Basic test suite, use of default string in regression testing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +default is a special argument to ``--bgen`` and ``--bcmp``. When used, the +scripts will automate generation of the directories. In the case of ``--bgen``, +a unique directory name consisting of the hash and a date will be created. +In the case of ``--bcmp``, the latest directory in [bdir] will automatically +be used. This provides a number of useful features + + - the ``--bgen`` directory will be named after the hash automatically + - the ``--bcmp`` will always find the most recent set of baselines + - the ``--bcmp`` reporting will include information about the comparison directory + name which will include hash information + - automation can be invoked easily, especially if ``--bdir`` is used to create separate + baseline directories as needed. + +Imagine the case where the default settings are used and ``--bdir`` is used to +create a unique location. You could easily carry out regular builds automatically via, +:: + + set mydate = `date -u "+%Y%m%d"` + git clone https://github.com/myfork/cice cice.$mydate --recursive + cd cice.$mydate + ./cice.setup --suite base_suite --mach conrad --env cray,gnu,intel,pgi --testid $mydate --bcmp default --bgen default --bdir /tmp/work/user/CICE_BASELINES_MASTER + +When this is invoked, a new set of baselines will be generated and compared to the prior +results each time without having to change the arguments. + + +Example. Create and test a custom suite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create your own input text file consisting of 5 columns of data, + - Test + - Grid + - pes + - sets (optional) + - diff test (optional) + +such as +:: + + > cat mysuite + smoke col 1x1 diag1,debug + restart col 1x1 + restart col 1x1 diag1,debug restart_col_1x1 + restart col 1x1 mynewoption,diag1,debug + +then use that input file, mysuite +:: + + ./cice.setup --suite mysuite --mach conrad --env cray --testid v01a --bgen default + cd mysuite.v01a + #wait for runs to complete + ./results.csh + +You can use all the standard regression testing options (``--bgen``, ``--bcmp``, +``--bdir``). Make sure any "diff" testing that goes on is on tests that +are created earlier in the test list, as early as possible. Unfortunately, +there is still no absolute guarantee the tests will be completed in the correct +sequence. + + +.. _testreporting: + +Test Reporting +--------------- + +The CICE testing scripts have the capability to post test results +to the official `wiki page `_. +You may need write permission on the wiki. If you are interested in using the +wiki, please contact the consortium. + +To post results, once a test suite is complete, run ``results.csh`` and +``report_results.csh`` from the suite directory, +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a + cd base_suite.v01a + #wait for runs to complete + ./results.csh + ./report_results.csh + +The reporting can also be automated by adding ``--report`` to ``cice.setup`` +:: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --report + +With ``--report``, the suite will create all the tests, build and submit them, +wait for all runs to be complete, and run the results and report_results scripts. -- In general, the baseline generation, baseline compare, and test diff are independent. -- Use the '-bd' flag to specify the location where you want the baseline dataset - to be written. Without specifying '-bd', the baseline dataset will be written - to the default baseline directory found in the env. file (ICE_MACHINE_BASELINE). -- If '-bd' is not passed, the scripts will look for baseline datasets in the default - baseline directory found in the env. file (ICE_MACHINE_BASELINE). - If the '-bd' option is passed, the scripts will look for baseline datasets in the - location passed to the -bd argument. -- To generate a baseline dataset for a specific version (for regression testing), - use '-bg '. The scripts will then place the baseline dataset - in $ICE_MACHINE_BASELINE// -- The '-testid' flag allows users to specify a testing id that will be added to the - end of the case directory. For example, "./cice.setup -m wolf -t smoke -testid t12 -p 4x1" - creates the directory wolf_smoke_gx3_4x1.t12. This flag is REQUIRED if using -t or -ts. .. _compliance: -~~~~~~~~~~~~~~~~~~~~ -Code Compliance Test -~~~~~~~~~~~~~~~~~~~~ +Code Compliance Test (non bit-for-bit validation) +---------------------------------------------------- -A core tenet of CICE dycore and Icepack innovations is that they must not change +A core tenet of CICE dycore and CICE innovations is that they must not change the physics and biogeochemistry of existing model configurations, notwithstanding obsolete model components. Therefore, alterations to existing CICE Consortium code must only fix demonstrable numerical or scientific inaccuracies or bugs, or be necessary to introduce new science into the code. New physics and biogeochemistry introduced into the model must not change model answers when switched off, and in -that case CICEcore and Icepack must reproduce answers bit-for-bit as compared to +that case CICEcore and CICE must reproduce answers bit-for-bit as compared to previous simulations with the same namelist configurations. This bit-for-bit requirement is common in Earth System Modeling projects, but often cannot be achieved in practice because model additions may require changes to existing code. In this @@ -269,7 +530,7 @@ on a different computing platform with a different compiler. Therefore, tools f scientific testing of CICE code changes have been developed to accompany bit-for-bit testing. These tools exploit the statistical properties of simulated sea ice thickness to confirm or deny the null hypothesis, which is that new additions to the CICE dycore -and Icepack have not significantly altered simulated ice volume using previous model +and CICE have not significantly altered simulated ice volume using previous model configurations. Here we describe the CICE testing tools, which are applies to output from five-year gx-1 simulations that use the standard CICE atmospheric forcing. A scientific justification of the testing is provided in @@ -277,9 +538,9 @@ A scientific justification of the testing is provided in .. _paired: -******************************* + Two-Stage Paired Thickness Test -******************************* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first quality check aims to confirm the null hypotheses :math:`H_0\!:\!\mu_d{=}0` at every model grid point, given the mean @@ -377,9 +638,9 @@ autocorrelation :math:`r_1`. .. _quadratic: -******************************* + Quadratic Skill Compliance Test -******************************* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In addition to the two-stage test of mean sea ice thickness, we also check that paired simulations are highly correlated and have similar @@ -462,11 +723,12 @@ hemispheres, and must exceed a critical value nominally set to test and the Two-Stage test described in the previous section are provided in :cite:`Hunke2018`. -*************************** -Practical Testing Procedure -*************************** -The CICE code compliance test is performed by running a python script (cice.t-test.py). +Code Compliance Testing Procedure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The CICE code compliance test is performed by running a python script +(**configurations/scripts/tests/QC/cice.t-test.py**). In order to run the script, the following requirements must be met: * Python v2.7 or later @@ -476,7 +738,7 @@ In order to run the script, the following requirements must be met: * basemap Python package (optional) In order to generate the files necessary for the compliance test, test cases should be -created with the ``qc`` option (i.e., ``-s qc``) when running cice.setup. This +created with the ``qc`` option (i.e., ``--set qc``) when running cice.setup. This option results in daily, non-averaged history files being written for a 5 year simulation. To install the necessary Python packages, the ``pip`` Python utility can be used. @@ -487,7 +749,9 @@ To install the necessary Python packages, the ``pip`` Python utility can be used pip install --user numpy pip install --user matplotlib -To run the compliance test: +To run the compliance test, setup a baseline run with the original baseline model and then +a perturbation run based on recent model changes. Use ``--sets qc`` in both runs in addition +to other settings needed. Then use the QC script to compare history output, .. code-block:: bash @@ -510,65 +774,9 @@ Implementation notes: 1) Provide a pass/fail on each of the confidence intervals, 2) Facilitate output of a bitmap for each test so that locations of failures can be identified. -~~~~~~~~~~~~~~~~~~~~ -CICE Test Reporting -~~~~~~~~~~~~~~~~~~~~ - -The CICE testing scripts have the capability of posting the test results -to an online dashboard, located `on CDash `_. -There are 2 options for posting CICE results to CDash: 1) The automated -script, 2) The manual method. - -***************** -Automatic Script -***************** -To automatically run the CICE tests, and post the results to the CICE Cdash dashboard, -users need to copy and run the ``run.suite`` script: - -.. code-block:: bash - - cp configuration/scripts/run.suite . - ./run.suite -m -testid -bc -bg - -The run.suite script does the following: - -- Creates a fresh clone of the CICE-Consortium repository -- ``cd`` to cloned repo -- run ``cice.setup`` to generate the base_suite directories. The output - is piped to ``log.suite`` -- Running ``cice.setup`` submits each individual job to the queue. -- ``run.suite`` monitors the queue manager to determine when all jobs have - finished (pings the queue manager once every 5 minutes). -- Once all jobs complete, cd to base_suite directory and run ``./results.csh`` -- Run ``./run_ctest.csh`` in order to post the test results to the CDash dashboard - -***************** -Manual Method -***************** - -To manually run the CICE tests and post the results to the CICE CDash dashboard, -users essentially just need to perform all steps available in run.suite, detailed below: - -- Pass the ``-report`` flag to cice.setup when running the ``base_suite`` test suite. - The ``-report`` flag copies the required CTest / CDash scripts to the suite - directory. -- ``cice.setup`` compiles the CICE code, and submits all of the jobs to the - queue manager. -- After every job has been submitted and completed, ``cd`` to the suite directory. -- Parse the results, by running ``./results.csh``. -- Run the CTest / CDash script ``./run_ctest.csh``. - -If the ``run_ctest.csh`` script is unable to post the testing results to the CDash -server, a message will be printed to the screen detailing instructions on how to attempt -to post the results from another server. If ``run_ctest.csh`` fails to submit the results, -it will generate a tarball ``cice_ctest.tgz`` that contains the necessary files for -submission. Copy this file to another server (CMake version 2.8+ required), extract the -archive, and run ``./run_ctest.csh -submit``. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ End-To-End Testing Procedure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Below is an example of a step-by-step procedure for testing a code change that results in non-bit-for-bit results: @@ -617,283 +825,58 @@ in non-bit-for-bit results: INFO:__main__: INFO:__main__:Quality Control Test PASSED -.. _tabnamelist: - -------------------------- -Table of namelist options -------------------------- - -.. _tab-namelist: - -.. csv-table:: Table 8 - :header: "variable", "options/format", "description", "recommended value" - :widths: 15, 15, 30, 15 - - "*setup_nml*", " ", " ", " " - "", "", "*Time, Diagnostics*", "" - "``days_per_year``", "``360`` or ``365``", "number of days in a model year", "365" - "``use_leap_years``", "true/false", "if true, include leap days", "" - "``year_init``", "yyyy", "the initial year, if not using restart", "" - "``istep0``", "integer", "initial time step number", "0" - "``dt``", "seconds", "thermodynamics time step length", "3600." - "``npt``", "integer", "total number of time steps to take", "" - "``ndtd``", "integer", "number of dynamics/advection/ridging/steps per thermo timestep", "1" - "", "", "*Initialization/Restarting*", "" - "``runtype``", "``initial``", "start from ``ice_ic``", "" - "", "``continue``", "restart using ``pointer_file``", "" - "``ice_ic``", "``default``", "latitude and sst dependent", "default" - "", "``none``", "no ice", "" - "", "path/file", "restart file name", "" - "``restart``", "true/false", "initialize using restart file", "``.true.``" - "``use_restart_time``", "true/false", "set initial date using restart file", "``.true.``" - "``restart_format``", "nc", "read/write  restart files (use with PIO)", "" - "", "bin", "read/write binary restart files", "" - "``lcdf64``", "true/false", "if true, use 64-bit  format", "" - "``restart_dir``", "path/", "path to restart directory", "" - "``restart_ext``", "true/false", "read/write halo cells in restart files", "" - "``restart_file``", "filename prefix", "output file for restart dump", "‘iced’" - "``pointer_file``", "pointer filename", "contains restart filename", "" - "``dumpfreq``", "``y``", "write restart every ``dumpfreq_n`` years", "y" - "", "``m``", "write restart every ``dumpfreq_n`` months", "" - "", "``d``", "write restart every ``dumpfreq_n`` days", "" - "``dumpfreq_n``", "integer", "frequency restart data is written", "1" - "``dump_last``", "true/false", "if true, write restart on last time step of simulation", "" - "", "", "*Model Output*", "" - "``bfbflag``", "true/false", "for bit-for-bit diagnostic output", "" - "``diagfreq``", "integer", "frequency of diagnostic output in ``dt``", "24" - "", "*e.g.*, 10", "once every 10 time steps", "" - "``diag_type``", "``stdout``", "write diagnostic output to stdout", "" - "", "``file``", "write diagnostic output to file", "" - "``diag_file``", "filename", "diagnostic output file (script may reset)", "" - "``print_global``", "true/false", "print diagnostic data, global sums", "``.false.``" - "``print_points``", "true/false", "print diagnostic data for two grid points", "``.false.``" - "``latpnt``", "real", "latitude of (2) diagnostic points", "" - "``lonpnt``", "real", "longitude of (2) diagnostic points", "" - "``dbug``", "true/false", "if true, write extra diagnostics", "``.false.``" - "``histfreq``", "string array", "defines output frequencies", "" - "", "``y``", "write history every ``histfreq_n`` years", "" - "", "``m``", "write history every ``histfreq_n`` months", "" - "", "``d``", "write history every ``histfreq_n`` days", "" - "", "``h``", "write history every ``histfreq_n`` hours", "" - "", "``1``", "write history every time step", "" - "", "``x``", "unused frequency stream (not written)", "" - "``histfreq_n``", "integer array", "frequency history output is written", "" - "", "0", "do not write to history", "" - "``hist_avg``", "true", "write time-averaged data", "``.true.``" - "", "false", "write snapshots of data", "" - "``history_dir``", "path/", "path to history output directory", "" - "``history_file``", "filename prefix", "output file for history", "‘iceh’" - "``write_ic``", "true/false", "write initial condition", "" - "``incond_dir``", "path/", "path to initial condition directory", "" - "``incond_file``", "filename prefix", "output file for initial condition", "‘iceh’" - "``runid``", "string", "label for run (currently CESM only)", "" - "", "", "", "" - "*grid_nml*", "", "", "" - "", "", "*Grid*", "" - "``grid_format``", "``nc``", "read  grid and kmt files", "‘bin’" - "", "``bin``", "read direct access, binary file", "" - "``grid_type``", "``rectangular``", "defined in *rectgrid*", "" - "", "``displaced_pole``", "read from file in *popgrid*", "" - "", "``tripole``", "read from file in *popgrid*", "" - "", "``regional``", "read from file in *popgrid*", "" - "``grid_file``", "filename", "name of grid file to be read", "‘grid’" - "``kmt_file``", "filename", "name of land mask file to be read", "‘kmt’" - "``gridcpl_file``", "filename", "input file for coupling grid info", "" - "``kcatbound``", "``0``", "original category boundary formula", "0" - "", "``1``", "new formula with round numbers", "" - "", "``2``", "WMO standard categories", "" - "", "``-1``", "one category", "" - "", "", "", "" - "*domain_nml*", "", "", "" - "", "", "*Domain*", "" - "``nprocs``", "integer", "number of processors to use", "" - "``processor_shape``", "``slenderX1``", "1 processor in the y direction (tall, thin)", "" - "", "``slenderX2``", "2 processors in the y direction (thin)", "" - "", "``square-ice``", "more processors in x than y, :math:`\sim` square", "" - "", "``square-pop``", "more processors in y than x, :math:`\sim` square", "" - "``distribution_type``", "``cartesian``", "distribute blocks in 2D Cartesian array", "" - "", "``roundrobin``", "1 block per proc until blocks are used", "" - "", "``sectcart``", "blocks distributed to domain quadrants", "" - "", "``sectrobin``", "several blocks per proc until used", "" - "", "``rake``", "redistribute blocks among neighbors", "" - "", "``spacecurve``", "distribute blocks via space-filling curves", "" - "``distribution_weight``", "``block``", "full block size sets ``work_per_block``", "" - "", "``latitude``", "latitude/ocean sets ``work_per_block``", "" - "``ew_boundary_type``", "``cyclic``", "periodic boundary conditions in x-direction", "" - "", "``open``", "Dirichlet boundary conditions in x", "" - "``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "" - "", "``open``", "Dirichlet boundary conditions in y", "" - "", "``tripole``", "U-fold tripole boundary conditions in y", "" - "", "``tripoleT``", "T-fold tripole boundary conditions in y", "" - "``maskhalo_dyn``", "true/false", "mask unused halo cells for dynamics", "" - "``maskhalo_remap``", "true/false", "mask unused halo cells for transport", "" - "``maskhalo_bound``", "true/false", "mask unused halo cells for boundary updates", "" - "", "", "", "" - "*tracer_nml*", "", "", "" - "", "", "*Tracers*", "" - "``tr_iage``", "true/false", "ice age", "" - "``restart_age``", "true/false", "restart tracer values from file", "" - "``tr_FY``", "true/false", "first-year ice area", "" - "``restart_FY``", "true/false", "restart tracer values from file", "" - "``tr_lvl``", "true/false", "level ice area and volume", "" - "``restart_lvl``", "true/false", "restart tracer values from file", "" - "``tr_pond_cesm``", "true/false", "CESM melt ponds", "" - "``restart_pond_cesm``", "true/false", "restart tracer values from file", "" - "``tr_pond_topo``", "true/false", "topo melt ponds", "" - "``restart_pond_topo``", "true/false", "restart tracer values from file", "" - "``tr_pond_lvl``", "true/false", "level-ice melt ponds", "" - "``restart_pond_lvl``", "true/false", "restart tracer values from file", "" - "``tr_aero``", "true/false", "aerosols", "" - "``restart_aero``", "true/false", "restart tracer values from file", "" - "*thermo_nml*", "", "", "" - "", "", "*Thermodynamics*", "" - "``kitd``", "``0``", "delta function ITD approximation", "1" - "", "``1``", "linear remapping ITD approximation", "" - "``ktherm``", "``0``", "zero-layer thermodynamic model", "" - "", "``1``", "Bitz and Lipscomb thermodynamic model", "" - "", "``2``", "mushy-layer thermodynamic model", "" - "``conduct``", "``MU71``", "conductivity :cite:`MU71`", "" - "", "``bubbly``", "conductivity :cite:`PETB07`", "" - "``a_rapid_mode``", "real", "brine channel diameter", "0.5x10 :math:`^{-3}` m" - "``Rac_rapid_mode``", "real", "critical Rayleigh number", "10" - "``aspect_rapid_mode``", "real", "brine convection aspect ratio", "1" - "``dSdt_slow_mode``", "real", "drainage strength parameter", "-1.5x10 :math:`^{-7}` m/s/K" - "``phi_c_slow_mode``", ":math:`0<\phi_c < 1`", "critical liquid fraction", "0.05" - "``phi_i_mushy``", ":math:`0<\phi_i < 1`", "solid fraction at lower boundary", "0.85" - "", "", "", "" - "*dynamics_nml*", "", "", "" - "", "", "*Dynamics*", "" - "``kdyn``", "``0``", "dynamics OFF", "1" - "", "``1``", "EVP dynamics", "" - "", "``2``", "EAP dynamics", "" - "``revised_evp``", "true/false", "use revised EVP formulation", "" - "``ndte``", "integer", "number of EVP subcycles", "120" - "``advection``", "``remap``", "linear remapping advection", "‘remap’" - "", "``upwind``", "donor cell advection", "" - "``kstrength``", "``0``", "ice strength formulation :cite:`Hibler79`", "1" - "", "``1``", "ice strength formulation :cite:`Rothrock75`", "" - "``krdg_partic``", "``0``", "old ridging participation function", "1" - "", "``1``", "new ridging participation function", "" - "``krdg_redist``", "``0``", "old ridging redistribution function", "1" - "", "``1``", "new ridging redistribution function", "" - "``mu_rdg``", "real", "e-folding scale of ridged ice", "" - "``Cf``", "real", "ratio of ridging work to PE change in ridging", "17." - "", "", "", "" - "*shortwave_nml*", "", "", "" - "", "", "*Shortwave*", "" - "``shortwave``", "``default``", "NCAR CCSM3 distribution method", "" - "", "``dEdd``", "Delta-Eddington method", "" - "``albedo_type``", "``default``", "NCAR CCSM3 albedos", "‘default’" - "", "``constant``", "four constant albedos", "" - "``albicev``", ":math:`0<\alpha <1`", "visible ice albedo for thicker ice", "" - "``albicei``", ":math:`0<\alpha <1`", "near infrared ice albedo for thicker ice", "" - "``albsnowv``", ":math:`0<\alpha <1`", "visible, cold snow albedo", "" - "``albsnowi``", ":math:`0<\alpha <1`", "near infrared, cold snow albedo", "" - "``ahmax``", "real", "albedo is constant above this thickness", "0.3 m" - "``R_ice``", "real", "tuning parameter for sea ice albedo from Delta-Eddington shortwave", "" - "``R_pnd``", "real", "... for ponded sea ice albedo …", "" - "``R_snw``", "real", "... for snow (broadband albedo) …", "" - "``dT_mlt``", "real", ":math:`\Delta` temperature per :math:`\Delta` snow grain radius", "" - "``rsnw_mlt``", "real", "maximum melting snow grain radius", "" - "``kalg``", "real", "absorption coefficient for algae", "" - "", "", "", "" - "*ponds_nml*", "", "", "" - "", "", "*Melt Ponds*", "" - "``hp1``", "real", "critical ice lid thickness for topo ponds", "0.01 m" - "``hs0``", "real", "snow depth of transition to bare sea ice", "0.03 m" - "``hs1``", "real", "snow depth of transition to pond ice", "0.03 m" - "``dpscale``", "real", "time scale for flushing in permeable ice", ":math:`1\times 10^{-3}`" - "``frzpnd``", "``hlid``", "Stefan refreezing with pond ice thickness", "‘hlid’" - "", "``cesm``", "CESM refreezing empirical formula", "" - "``rfracmin``", ":math:`0 \le r_{min} \le 1`", "minimum melt water added to ponds", "0.15" - "``rfracmax``", ":math:`0 \le r_{max} \le 1`", "maximum melt water added to ponds", "1.0" - "``pndaspect``", "real", "aspect ratio of pond changes (depth:area)", "0.8" - "", "", "", "" - "*zbgc_nml*", "", "", "" - "", "", "*Biogeochemistry*", "" - "``tr_brine``", "true/false", "brine height tracer", "" - "``tr_zaero``", "true/false", "vertical aerosol tracers", "" - "``modal_aero``", "true/false", "modal aersols", "" - "``restore_bgc``", "true/false", "restore bgc to data", "" - "``solve_zsal`", "true/false", "update salinity tracer profile", "" - "``bgc_data_dir``", "path/", "data directory for bgc", "" - "``skl_bgc``", "true/false", "biogeochemistry", "" - "``sil_data_type``", "``default``", "default forcing value for silicate", "" - "", "``clim``", "silicate forcing from ocean climatology :cite:`GLBA06`", "" - "``nit_data_type``", "``default``", "default forcing value for nitrate", "" - "", "``clim``", "nitrate forcing from ocean climatology :cite:`GLBA06`", "" - "", "``sss``", "nitrate forcing equals salinity", "" - "``fe_data_type``", "``default``", "default forcing value for iron", "" - "", "``clim``", "iron forcing from ocean climatology", "" - "``bgc_flux_type``", "``Jin2006``", "ice–ocean flux velocity of :cite:`JDWSTWLG06`", "" - "", "``constant``", "constant ice–ocean flux velocity", "" - "``restart_bgc``", "true/false", "restart tracer values from file", "" - "``tr_bgc_C_sk``", "true/false", "algal carbon tracer", "" - "``tr_bgc_chl_sk``", "true/false", "algal chlorophyll tracer", "" - "``tr_bgc_Am_sk``", "true/false", "ammonium tracer", "" - "``tr_bgc_Sil_sk``", "true/false", "silicate tracer", "" - "``tr_bgc_DMSPp_sk``", "true/false", "particulate DMSP tracer", "" - "``tr_bgc_DMSPd_sk``", "true/false", "dissolved DMSP tracer", "" - "``tr_bgc_DMS_sk``", "true/false", "DMS tracer", "" - "``phi_snow``", "real", "snow porosity for brine height tracer", "" - "", "", "", "" - "*forcing_nml*", "", "", "" - "", "", "*Forcing*", "" - "``formdrag``", "true/false", "calculate form drag", "" - "``atmbndy``", "``default``", "stability-based boundary layer", "‘default’" - "", "``constant``", "bulk transfer coefficients", "" - "``fyear_init``", "yyyy", "first year of atmospheric forcing data", "" - "``ycycle``", "integer", "number of years in forcing data cycle", "" - "``atm_data_format``", "``nc``", "read  atmo forcing files", "" - "", "``bin``", "read direct access, binary files", "" - "``atm_data_type``", "``default``", "constant values defined in the code", "" - "", "``LYq``", "AOMIP/Large-Yeager forcing data", "" - "", "``monthly``", "monthly forcing data", "" - "", "``ncar``", "NCAR bulk forcing data", "" - "", "``oned``", "column forcing data", "" - "``atm_data_dir``", "path/", "path to atmospheric forcing data directory", "" - "``calc_strair``", "true", "calculate wind stress and speed", "" - "", "false", "read wind stress and speed from files", "" - "``highfreq``", "true/false", "high-frequency atmo coupling", "" - "``natmiter``", "integer", "number of atmo boundary layer iterations", "" - "``calc_Tsfc``", "true/false", "calculate surface temperature", "``.true.``" - "``precip_units``", "``mks``", "liquid precipitation data units", "" - "", "``mm_per_month``", "", "" - "", "``mm_per_sec``", "(same as MKS units)", "" - "``tfrz_option``", "``minus1p8``", "constant ocean freezing temperature (:math:`-1.8^{\circ} C`)", "" - "", "``linear_salt``", "linear function of salinity (ktherm=1)", "" - "", "``mushy_layer``", "matches mushy-layer thermo (ktherm=2)", "" - "``ustar_min``", "real", "minimum value of ocean friction velocity", "0.0005 m/s" - "``fbot_xfer_type``", "``constant``", "constant ocean heat transfer coefficient", "" - "", "``Cdn_ocn``", "variable ocean heat transfer coefficient", "" - "``update_ocn_f``", "true", "include frazil water/salt fluxes in ocn fluxes", "" - "", "false", "do not include (when coupling with POP)", "" - "``l_mpond_fresh``", "true", "retain (topo) pond water until ponds drain", "" - "", "false", "release (topo) pond water immediately to ocean", "" - "``oceanmixed_ice``", "true/false", "active ocean mixed layer calculation", "``.true.`` (if uncoupled)" - "``ocn_data_format``", "``nc``", "read  ocean forcing files", "" - "", "``bin``", "read direct access, binary files", "" - "``sss_data_type``", "``default``", "constant values defined in the code", "" - "", "``clim``", "climatological data", "" - "", "``near``", "POP ocean forcing data", "" - "``sst_data_type``", "``default``", "constant values defined in the code", "" - "", "``clim``", "climatological data", "" - "", "``ncar``", "POP ocean forcing data", "" - "``ocn_data_dir``", "path/", "path to oceanic forcing data directory", "" - "``oceanmixed_file``", "filename", "data file containing ocean forcing data", "" - "``restore_sst``", "true/false", "restore sst to data", "" - "``trestore``", "integer", "sst restoring time scale (days)", "" - "``restore_ice``", "true/false", "restore ice state along lateral boundaries", "" - "", "", "", "" - "*icefields_tracer_nml*", "", "", "" - "", "", "*History Fields*", "" - "``f_``", "string", "frequency units for writing ```` to history", "" - "", "``y``", "write history every ``histfreq_n`` years", "" - "", "``m``", "write history every ``histfreq_n`` months", "" - "", "``d``", "write history every ``histfreq_n`` days", "" - "", "``h``", "write history every ``histfreq_n`` hours", "" - "", "``1``", "write history every time step", "" - "", "``x``", "do not write ```` to history", "" - "", "``md``", "*e.g.,* write both monthly and daily files", "" - "``f__ai``", "", "grid cell average of ```` (:math:`\times a_i`)", "" + +.. _testplotting: + +Test Plotting +---------------- + +The CICE scripts include a script (``timeseries.csh``) that will generate a timeseries +figure from the diagnostic output file. +When running a test suite, the ``timeseries.csh`` script is automatically copied to the suite directory. +If the ``timeseries.csh`` script is to be used on a test / case that is not a part of a test suite, +users will need to run the ``timeseries.csh`` script from the tests directory +(``./configuration/scripts/tests/timeseries.csh``), or copy it to a local directory and run it +locally (``cp configuration/scripts/tests/timeseries.csh .`` followed by +``./timeseries.csh /path/to/ice_diag.full_ITD``. The plotting script can be run +on any of the output files - icefree, slab, full_ITD, land). To generate the figure, +run the ``timeseries.csh`` script and pass the full path to the ice_diag file as an argument. + +For example: + +Run the test suite. :: + +$ ./cice.setup -m conrad -e intel --suite base_suite -acct --testid t00 + +Wait for suite to finish then go to the directory. :: + +$ cd base_suite.t00 + +Run the timeseries script on the desired case. :: + +$ ./timeseries.csh /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ice_diag.full_ITD + +The output figures are placed in the directory where the ice_diag file is located. + +This plotting script can be used to plot the following variables: + + - area fraction + - average ice thickness (m) + - average snow depth (m) + - air temperature (C) + - shortwave radiation (:math:`W/m^2`) + - longwave radiation (:math:`W/m^2`) + - snowfall + - average salinity (ppt) + - surface temperature (C) + - outward longwave flux (:math:`W/m^2`) + - sensible heat flux (:math:`W/m^2`) + - latent heat flux (:math:`W/m^2`) + - top melt (m) + - bottom melt (m) + - lateral melt (m) + - new ice (m) + - congelation (m) + - snow-ice (m) + - initial energy change (:math:`W/m^2`) diff --git a/doc/source/user_guide/ug_troubleshooting.rst b/doc/source/user_guide/ug_troubleshooting.rst index c71d64c55..3ab5d7b40 100644 --- a/doc/source/user_guide/ug_troubleshooting.rst +++ b/doc/source/user_guide/ug_troubleshooting.rst @@ -1,77 +1,70 @@ :tocdepth: 3 +.. _troubleshooting: + Troubleshooting =============== -Check the FAQ: http://oceans11.lanl.gov/drupal/CICE/FAQ. +Check the FAQ: https://github.com/CICE-Consortium/Icepack/wiki .. _setup: -~~~~~~~~~~~~~ Initial setup -~~~~~~~~~~~~~ +------------- -The script **comp\_ice** is configured so that the files **grid**, -**kmt**, **ice\_in**, **run\_ice**, **iced\_gx3\_v5.0** and -**ice.restart\_file** are NOT overwritten after the first setup. If you -wish to make changes to the original files in **input\_templates/** -rather than those in the run directory, either remove the files from the -run directory before executing **comp\_ice** or edit the script. +If there are problems, you can manually edit +the env, Macros, and **cice.run** files in the case directory until things are +working properly. Then you can copy the env and Macros files back to +**configuration/scripts/machines**. -The code may abort during the setup phase for any number of reasons, and -often the buffer containing the diagnostic output fails to print before -the executable exits. The quickest way to get the diagnostic information -is to run the code in an interactive shell with just the command `cice` -for serial runs or “`mpirun -np N cice`” for MPI runs, where N is the -appropriate number of processors (or a command appropriate for your -computer’s software). +Changes made directly in the run directory, e.g. to the namelist file, will be overwritten +if scripts in the case directory are run again later. -If the code fails to compile or run, or if the model configuration is -changed, try the following: +If changes are needed in the **cice.run.setup.csh** script, it must be manually modified. -- create **Macros.\***. **Makefile.\*** and **run\_ice.\*** files for - your particular platform, if they do not already exist (type ‘uname - -s’ at the prompt and compare the result with the file suffixes; we - rename `UNICOS/mp` as `UNICOS` for simplicity). +Ensure that the block size ``ICE_BLCKX``, ``ICE_BLCKY``, and ``ICE_MXBLCKS`` in **cice.settings** is +compatible with the processor\_shape and other domain options in **ice\_in** -- modify the `INCLUDE` directory path and other settings for your system - in the scripts, **Macros.\*** and **Makefile.\*** files. +If using the rake or space-filling curve algorithms for block +distribution (`distribution\_type` in **ice\_in**) the code will abort +if `MXBLCKS` is not large enough. The correct value is provided in the +diagnostic output. Also, the spacecurve setting can only be used with certain +block sizes that results in number of blocks in the x and y directions being +only multiples of 2, 3, or 5. -- alter directory paths, file names and the execution command as needed - in **run\_ice** and **ice\_in**. +If starting from a restart file, ensure that kcatbound is the same as +that used to create the file (`kcatbound` = 0 for the files included in +this code distribution). Other configuration parameters, such as +`NICELYR`, must also be consistent between runs. -- ensure that `nprocs` in **ice\_in** is equal to `NTASK` in **comp\_ice**. +For stand-alone runs, check that `-Dcoupled` is *not* set in the +**Macros.\*** file. -- ensure that the block size `NXBLOCK`, `NYBLOCK` in **comp\_ice** is - compatible with the processor\_shape and other domain options in - **ice\_in** +For coupled runs, check that `-Dcoupled` and other +coupled-model-specific (e.g., CESM, popcice or hadgem) preprocessing +options are set in the **Macros.\*** file. -- if using the rake or space-filling curve algorithms for block - distribution (`distribution\_type` in **ice\_in**) the code will abort - if `MXBLCKS` is not large enough. The correct value is provided in the - diagnostic output. +Set ``ICE_CLEANBUILD`` to true to clean before rebuilding. -- if starting from a restart file, ensure that kcatbound is the same as - that used to create the file (`kcatbound` = 0 for the files included in - this code distribution). Other configuration parameters, such as - `NICELYR`, must also be consistent between runs. -- for stand-alone runs, check that `-Dcoupled` is *not* set in the - **Macros.\*** file. +.. _restarttrouble: -- for coupled runs, check that `-Dcoupled` and other - coupled-model-specific (e.g., CESM, popcice or hadgem) preprocessing - options are set in the **Macros.\*** file. +Restarts +-------------- -- edit the grid size and other parameters in **comp\_ice**. +Manual restart tests require the path to the restart file be included in ``ice_in`` in the +namelist file. -- remove the **compile/** directory completely and recompile. +Ensure that ``kcatbound`` is the same as that used to create the restart file. +Other configuration parameters, such as ``NICELYR``, must also be consistent between runs. -.. _restarttrouble: - -~~~~~~~~ -Restarts -~~~~~~~~ +.. + this is commented out now + Underflows + ----------- + - Tests using a debug flag that traps underflows will fail unless a "flush-to-zero" flag + is set in the Macros file. This is due to very small exponential values in the delta-Eddington + radiation scheme. CICE version 5 introduces a new model configuration that makes restarting from older simulations difficult. In particular, the number @@ -104,13 +97,13 @@ these files. However if different physics is used (for instance, mushy thermo instead of BL99), the code may still fail. To convert a v4.1 restart file: -#. Edit the code **input\_templates/convert\_restarts.f90** for your +- Edit the code **input\_templates/convert\_restarts.f90** for your model configuration and path names. Compile and run this code to create a binary restart file that can be read using v5. Copy the resulting file to the **restart/** subdirectory in your working directory. -#. In your working directory, turn off all tracer restart flags in +- In your working directory, turn off all tracer restart flags in **ice\_in** and set the following: - runtype = ‘initial’ @@ -121,18 +114,18 @@ restart file: - use\_restart\_time = .true. -#. In **CICE\_InitMod.F90**, comment out the call to - restartfile(ice\_ic) and uncomment the call to - restartfile\_v4(ice\_ic) immediately below it. This will read the - v4.1 binary file and write a v5  file containing the same - information. +- In **CICE\_InitMod.F90**, comment out the call to + restartfile(ice\_ic) and uncomment the call to + restartfile\_v4(ice\_ic) immediately below it. This will read the + v4.1 binary file and write a v5  file containing the same + information. If restart files are taking a long time to be written serially (i.e., not using PIO), see the next section. -~~~~~~~~~~~~~~ + Slow execution -~~~~~~~~~~~~~~ +-------------------- On some architectures, underflows (:math:`10^{-300}` for example) are not flushed to zero automatically. Usually a compiler flag is available @@ -147,9 +140,9 @@ variables may overfill MPI’s buffers, causing the code to slow down remedy this problem, set `BARRIERS yes` in **comp\_ice**. This synchronizes MPI messages, keeping the buffers in check. -~~~~~~~~~~~~~~~ + Debugging hints -~~~~~~~~~~~~~~~ +----------------------- Several utilities are available that can be helpful when debugging the code. Not all of these will work everywhere in the code, due to possible @@ -185,11 +178,11 @@ conflicts in module dependencies. Compute and print the minimum and maximum values for an individual real array, or its global sum. -~~~~~~~~~~ + Known bugs -~~~~~~~~~~ +-------------- -#. Fluxes sent to the CESM coupler may have incorrect values in grid +- Fluxes sent to the CESM coupler may have incorrect values in grid cells that change from an ice-free state to having ice during the given time step, or vice versa, due to scaling by the ice area. The authors of the CESM flux coupler insist on the area scaling so that @@ -197,38 +190,38 @@ Known bugs note that the land area does not suddenly become zero in a grid cell, as does the ice area). -#. With the old CCSM radiative scheme (`shortwave` = ‘default’ or +- With the old CCSM radiative scheme (`shortwave` = ‘default’ or ‘ccsm3’), a sizable fraction (more than 10%) of the total shortwave radiation is absorbed at the surface but should be penetrating into the ice interior instead. This is due to use of the aggregated, effective albedo rather than the bare ice albedo when `snowpatch` :math:`< 1`. -#. The date-of-onset diagnostic variables, `melt\_onset` and `frz\_onset`, +- The date-of-onset diagnostic variables, `melt\_onset` and `frz\_onset`, are not included in the core restart file, and therefore may be incorrect for the current year if the run is restarted after Jan 1. Also, these variables were implemented with the Arctic in mind and may be incorrect for the Antarctic. -#. The single-processor *system\_clock* time may give erratic results on +- The single-processor *system\_clock* time may give erratic results on some architectures. -#. History files that contain time averaged data (`hist\_avg` = true in +- History files that contain time averaged data (`hist\_avg` = true in **ice\_in**) will be incorrect if restarting from midway through an averaging period. -#. In stand-alone runs, restarts from the end of `ycycle` will not be +- In stand-alone runs, restarts from the end of `ycycle` will not be exact. -#. Using the same frequency twice in `histfreq` will have unexpected +- Using the same frequency twice in `histfreq` will have unexpected consequences and causes the code to abort. -#. Latitude and longitude fields in the history output may be wrong when +- Latitude and longitude fields in the history output may be wrong when using padding. -~~~~~~~~~~~~~~~~~~~~~~~~~ + Interpretation of albedos -~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------- The snow-and-ice albedo, `albsni`, and diagnostic albedos `albice`, `albsno`, and `albpnd` are merged over categories but not scaled (divided) by the @@ -250,9 +243,9 @@ each 24-hour period. To rectify this, a separate counter is used for the averaging that is incremented only when :math:`\cos\varphi > 0`. The albedos will still be zero in the dark, polar winter hemisphere. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Proliferating subprocess parameterizations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------- With the addition of several alternative parameterizations for sea ice processes, a number of subprocesses now appear in multiple parts of the @@ -265,4 +258,6 @@ and snow–ice formation differently, and the topo and level-ice melt pond schemes both allow fresh ice to grow atop melt ponds, using slightly different formulations for Stefan freezing. These various process parameterizations will be compared and their subprocess descriptions -possibly unified in the future. \ No newline at end of file +possibly unified in the future. + +