diff --git a/Config/config_postprocess.xml b/Config/config_postprocess.xml index f7a3fb39..d738d206 100644 --- a/Config/config_postprocess.xml +++ b/Config/config_postprocess.xml @@ -30,7 +30,7 @@ @@ -54,7 +54,7 @@ @@ -62,7 +62,7 @@ @@ -70,7 +70,7 @@ @@ -78,7 +78,7 @@ @@ -86,7 +86,7 @@ @@ -94,7 +94,7 @@ @@ -102,7 +102,7 @@ @@ -110,7 +110,7 @@ diff --git a/Machines/machine_postprocess.xml b/Machines/machine_postprocess.xml index 350f56f2..7b6c6bd3 100644 --- a/Machines/machine_postprocess.xml +++ b/Machines/machine_postprocess.xml @@ -336,14 +336,14 @@ - 3 - 3 + 3 + 3 3 /scratch/training/CESM_post/obs_root/amwg/amwg_data - 3 - 3 + 4 + 4 /scratch/training/CESM_post/obs_root/pcwg/ice/data @@ -353,8 +353,8 @@ /glade/p/cesm/lmwg/diag/lnd_diag_data - 3 - 3 + 4 + 4 /scratch/training/CESM_post/obs_root diff --git a/Templates/batch_ada.tmpl b/Templates/batch_ada.tmpl index e08df1f0..3ab6be77 100644 --- a/Templates/batch_ada.tmpl +++ b/Templates/batch_ada.tmpl @@ -9,7 +9,6 @@ #BSUB -R "span[ptile={{ ppn }}]" #BSUB -J {{ processName }} #BSUB -W {{ wallclock }} -#BSUB -P {{ project }} #BSUB -L /bin/bash #Uses the bash login shell to initialize the job's execution environment. #BSUB -R "rusage[mem={{memory}}]" #Request 25000MB per process (CPU) for the job diff --git a/Templates/postprocess.tmpl b/Templates/postprocess.tmpl index df01fe31..804c75e7 100644 --- a/Templates/postprocess.tmpl +++ b/Templates/postprocess.tmpl @@ -56,8 +56,8 @@ log_filename={{ caseRoot }}/logs/{{ processName }}.log.$today {{ imb_env_vars }} {{ mpirun }} {{ imb_run }} {{ imb_options }} >> ${log_filename} 2>&1 {% elif standalone %} -{{ mpirun }} ./{{ postProcessCmd }} {{ debug }} {{ backtrace }} --caseroot {{ caseRoot }} --standalone {{ cmdl_opt }} >> ${log_filename} 2>&1 +{{ mpirun }} {{ postProcessCmd }} {{ debug }} {{ backtrace }} --caseroot {{ caseRoot }} --standalone {{ cmdl_opt }} >> ${log_filename} 2>&1 {% else %} -{{ mpirun }} ./{{ postProcessCmd }} {{ debug }} {{ backtrace }} --caseroot {{ caseRoot }} >> ${log_filename} 2>&1 +{{ mpirun }} {{ postProcessCmd }} {{ debug }} {{ backtrace }} --caseroot {{ caseRoot }} >> ${log_filename} 2>&1 {% endif %} diff --git a/atm_diag/regridclimo.ncl b/atm_diag/regridclimo.ncl index 94836738..22f87ef5 100644 --- a/atm_diag/regridclimo.ncl +++ b/atm_diag/regridclimo.ncl @@ -156,9 +156,9 @@ end if print("Regridding: ") print("SRC:"+srcFileName) print("DST:"+dstFileName) - if(isfilepresent(dstFileName)) then - system("rm "+dstFileName) - end if +; if(isfilepresent(dstFileName)) then + system("rm -f "+dstFileName) +; end if src_file=addfile(srcFileName,"r") dst_file=addfile(dstFileName,"c") @@ -223,7 +223,7 @@ end if var_in_exclude = (/"lat", "lon", "area", "date_written", "time_written"/) ;--- Specify a list of variables on the source SE file that should be directly copied - var_in_copy = (/"time_bnds"/) + var_in_copy = (/"time_bnds","cosp_ht_bnds","cosp_sr_bnds","cosp_prs_bnds","cosp_tau_bnds"/) ;--- Specify a list of variables to be regridded var_out = "All_Variables" ; to be regridded @@ -293,11 +293,34 @@ end if ib = ind(dNames.eq."ilev") klev = dSizes(ia) klevi = dSizes(ib) - ; standard CAM dimensions - dimNames = (/"time", "lat", "lon", "lev", "ilev", "nbnd", "chars"/) - dimSizes = (/ -1 , nlat , mlon , klev, klevi , 2 , 8 /) - dimUnlim = (/ True , False, False, False, False , False , False /) - filedimdef(dst_file,dimNames,dimSizes,dimUnlim) +; cosp dimensions + if (any("cosp_scol".eq.dNames)) + iscol = ind(dNames.eq."cosp_scol") + nscol = dSizes(iscol) + itau = ind(dNames.eq."cosp_tau") + ntau = dSizes(itau) + iprs = ind(dNames.eq."cosp_prs") + nprs = dSizes(iprs) + isr = ind(dNames.eq."cosp_sr") + nsr = dSizes(isr) + isza = ind(dNames.eq."cosp_sza") + nsza = dSizes(isza) + iht = ind(dNames.eq."cosp_ht") + nht = dSizes(iht) +; CAM dimensions plus cosp + dimNames = (/"time", "lat", "lon", "lev", "ilev", "nbnd", "chars","cosp_scol","cosp_prs","cosp_sr","cosp_tau","cosp_sza","cosp_ht"/) + dimSizes = (/ -1 , nlat , mlon , klev, klevi , 2 , 8 ,nscol ,nprs ,nsr ,ntau ,nsza ,nht /) + dimUnlim = (/ True , False, False, False, False , False , False ,False ,False ,False ,False ,False ,False /) + filedimdef(dst_file,dimNames,dimSizes,dimUnlim) + else +; standard CAM dimensions + dimNames = (/"time", "lat", "lon", "lev", "ilev", "nbnd", "chars"/) + dimSizes = (/ -1 , nlat , mlon , klev, klevi , 2 , 8 /) + dimUnlim = (/ True , False, False, False, False , False , False /) + filedimdef(dst_file,dimNames,dimSizes,dimUnlim) + end if + + ;--- The following are explicitly added because they are 'special' @@ -332,6 +355,13 @@ end if vdims = getfilevardims(src_file, var_in(nv)) filevardef(dst_file, var_in(nv), getfilevartypes(src_file,var_in(nv)) \ , (/ "time", vdims(1), "lat", "lon" /) ) + delete(vdims) + end if + if (rank_in .eq.4.and.var_in(nv).eq."FISCCP1_COSP") then + vdims = getfilevardims(src_file, var_in(nv)) + filevardef(dst_file, var_in(nv), getfilevartypes(src_file,var_in(nv)) \ + , (/ "time", "cosp_prs", "cosp_tau","lat", "lon" /) ) + delete(vdims) end if dumAtts = new( 1, getfilevartypes(src_file,var_in(nv))) @@ -374,13 +404,13 @@ end if any(var_in(nv).eq.var_in_copy) ) then rank_in = dimsizes(getfilevardimsizes(src_file, var_in(nv))) - if (rank_in .eq.1 .or. any(var_in(nv).eq.var_in_copy) ) then dst_file->$var_in(nv)$ = (/ src_file->$var_in(nv)$ /) end if if (rank_in .ge.2 .and. .not.any(var_in(nv).eq.var_in_copy)) then dst_file->$var_in(nv)$ = (/ ESMF_regrid_with_weights(src_file->$var_in(nv)$,wgtFileName,Opt_RGRD) /) end if + delete(rank_in) end if end if end do ; nv diff --git a/create_python_env b/create_python_env index d5561cf9..06b19f36 100755 --- a/create_python_env +++ b/create_python_env @@ -143,8 +143,10 @@ if [ -d $env ]; then info="$progname - ${pp_dir}/cesm-env2 virtual environment already exists. Skipping conda create" print_result "$info" else - export CONDA_ENVS_PATH=${pp_dir} - export CONDA_PKGS_DIRS=${pp_dir}/conda_pkgs + #export CONDA_ENVS_PATH=${pp_dir} + export CONDA_ENVS_PATH=${env} + #export CONDA_PKGS_DIRS=${pp_dir}/conda_pkgs + export CONDA_PKGS_DIRS=${env}/conda_pkgs conda create python=2.7 -n cesm-env2 #virtualenv --system-site-packages -p python2 cesm-env2 fi diff --git a/diagnostics/diagnostics/atm/Config/config_diags_atm.xml b/diagnostics/diagnostics/atm/Config/config_diags_atm.xml index 812ad271..82441f56 100644 --- a/diagnostics/diagnostics/atm/Config/config_diags_atm.xml +++ b/diagnostics/diagnostics/atm/Config/config_diags_atm.xml @@ -228,7 +228,7 @@ @@ -236,7 +236,7 @@ diff --git a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml index b56bdd0e..23619e8b 100644 --- a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml +++ b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml @@ -346,7 +346,7 @@ Applies to both model and control cases." @@ -354,7 +354,7 @@ Applies to both model and control cases." diff --git a/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml b/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml index f32c8fe7..2e0b7dfc 100644 --- a/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml +++ b/diagnostics/diagnostics/ocn/Config/config_diags_ocn.xml @@ -92,7 +92,7 @@ @@ -216,7 +216,7 @@ Applies to both model and control cases." @@ -304,7 +304,7 @@ Applies to both model and control cases." @@ -312,7 +312,7 @@ Applies to both model and control cases." @@ -413,7 +413,7 @@ Applies to both model and control cases." @@ -421,7 +421,7 @@ Applies to both model and control cases." @@ -637,7 +637,7 @@ Applies to both model and control cases." @@ -645,7 +645,7 @@ Applies to both model and control cases." @@ -894,7 +894,7 @@ Applies to both model and control cases." @@ -908,21 +908,21 @@ Applies to both model and control cases." @@ -1018,7 +1018,7 @@ Applies to both model and control cases." @@ -1032,21 +1032,21 @@ Applies to both model and control cases." diff --git a/diagnostics/diagnostics/ocn/Plots/ocn_diags_plot_bc.py b/diagnostics/diagnostics/ocn/Plots/ocn_diags_plot_bc.py index e7fd98e1..5a0e228d 100755 --- a/diagnostics/diagnostics/ocn/Plots/ocn_diags_plot_bc.py +++ b/diagnostics/diagnostics/ocn/Plots/ocn_diags_plot_bc.py @@ -75,13 +75,14 @@ def _convert_plots(self, workdir, imgFormat, files): psFiles = list() psFiles = sorted(files) + # check if the convert command exists rc = cesmEnvLib.which('convert') if rc is not None and imgFormat.lower() in ['png','gif']: for psFile in psFiles: sourceFile = '{0}/{1}.ps'.format(workdir, psFile) -## print('...... convert source file {0}'.format(sourceFile)) + print('...... convert source file {0}'.format(sourceFile)) # check if the sourceFile exists rc, err_msg = cesmEnvLib.checkFile(sourceFile,'read') @@ -94,13 +95,15 @@ def _convert_plots(self, workdir, imgFormat, files): # convert the image from ps to imgFormat try: + print('before subprocess') pipe = subprocess.check_call( ['convert', '-trim', '-bordercolor', 'white', '-border', '5x5', '-density', '95', '{0}'.format(sourceFile),'{0}'.format(imgFile)]) -## print('...... created {0} size = {1}'.format(imgFile, os.path.getsize(imgFile))) + print('...... created {0} size = {1}'.format(imgFile, os.path.getsize(imgFile))) except subprocess.CalledProcessError as e: print('...... failed to create {0}'.format(imgFile)) print('WARNING: convert_plots call to convert failed with error:') print(' {0}'.format(e.output)) else: + print('continuing') continue else: print('WARNING: convert_plots unable to find convert command in path.') diff --git a/diagnostics/diagnostics/ocn/Plots/poplog_timeseries.py b/diagnostics/diagnostics/ocn/Plots/poplog_timeseries.py index 67fd2447..2cba129f 100755 --- a/diagnostics/diagnostics/ocn/Plots/poplog_timeseries.py +++ b/diagnostics/diagnostics/ocn/Plots/poplog_timeseries.py @@ -85,6 +85,11 @@ def generate_plots(self, env): # chdir into the working directory os.chdir(env['WORKDIR']) + # update POP transport plots if high-res + if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') : + self._expectedPlots_transportDiags = [('Drake_Passage','diagts_transport.drake'), ('Mozambique_Channel','diagts_transport.mozam'), ('Bering_Strait','diagts_transport.bering'),('Indonesian_Throughflow','diagts_transport.itf'),('Windward_Passage','diagts_transport.windward'),('Florida_Strait','diagts_transport.florida')] + self._expectedInFiles = ['diagts_3d.asc', 'diagts_cfc.asc', 'diagts_ecosys.asc', 'diagts_fwflux.asc', 'diagts_hflux.asc', 'diagts_info.asc', 'diagts_precfactor.asc','diagts_nino.asc', 'transports.bering.asc', 'transports.drake.asc', 'transports.florida.asc','transports.itf.asc', 'transports.mozambique.asc', 'transports.windward.asc'] + for nclPlotFile in self._ncl: # copy the NCL command to the workdir shutil.copy2('{0}/{1}'.format(env['NCLPATH'],nclPlotFile), '{0}/{1}'.format(env['WORKDIR'], nclPlotFile)) diff --git a/diagnostics/diagnostics/ocn/model_timeseries.py b/diagnostics/diagnostics/ocn/model_timeseries.py index a718a259..06574e5e 100755 --- a/diagnostics/diagnostics/ocn/model_timeseries.py +++ b/diagnostics/diagnostics/ocn/model_timeseries.py @@ -252,7 +252,10 @@ def check_prerequisites(self, env): dtFilesString = ' '.join(dtFiles) # define the awk script to parse the dt log files - dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH']) + if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') : + dtFilesAwkPath = '{0}/process_pop2_dtfiles_hires.awk'.format(env['TOOLPATH']) + else: + dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH']) dtFilesAwkCmd = '{0} {1}'.format(dtFilesAwkPath, dtFilesString).split(' ') print('model_timeseries: dtFilesAwkCmd = {0}'.format(dtFilesAwkCmd)) diff --git a/ocn_diag/ncl_lib/field_2d_diff.ncl b/ocn_diag/ncl_lib/field_2d_diff.ncl index 83815979..6de7a899 100644 --- a/ocn_diag/ncl_lib/field_2d_diff.ncl +++ b/ocn_diag/ncl_lib/field_2d_diff.ncl @@ -4,13 +4,16 @@ load "$NCLPATH/get_environment_diff.ncl" load "$NCLPATH/contour_plot.ncl" begin - field_name = [/"SSH", "HMXL", "HBLT"/] + field_name = [/"SSH", "HMXL"/] missing = 1.0e30 fileid_1 = addfile(file_netcdf,"r") fileid_2 = addfile(cntrl_netcdf,"r") + if (isfilevar(fileid_1,"HBLT")) then + ListPush(field_name,"HBLT") + end if if (isfilevar(fileid_1,"DIA_DEPTH")) then ListPush(field_name,"DIA_DEPTH") end if diff --git a/ocn_diag/ncl_lib/field_2d_za_diff.ncl b/ocn_diag/ncl_lib/field_2d_za_diff.ncl index a93613c8..869bfe2d 100644 --- a/ocn_diag/ncl_lib/field_2d_za_diff.ncl +++ b/ocn_diag/ncl_lib/field_2d_za_diff.ncl @@ -20,10 +20,15 @@ begin region_index = (/ global, atlantic, pacific, indian, southern /) n_reg = dimsizes(region_index) - field_name = [/ "SSH", "HBLT", "HMXL" /] + field_name = [/ "SSH","HMXL" /] fileid_1 = addfile(file_netcdf_za,"r") fileid_2 = addfile(cntrl_netcdf_za,"r") + + + if (isfilevar(fileid_1,"HBLT")) then + ListPush(field_name,"HBLT") + end if if (isfilevar(fileid_1,"DIA_DEPTH")) then ListPush(field_name,"DIA_DEPTH") end if diff --git a/ocn_diag/ncl_lib/moc_netcdf.ncl b/ocn_diag/ncl_lib/moc_netcdf.ncl index ab5fde9c..4e66eaa8 100644 --- a/ocn_diag/ncl_lib/moc_netcdf.ncl +++ b/ocn_diag/ncl_lib/moc_netcdf.ncl @@ -17,7 +17,7 @@ begin depth = depth / 1.0e5 ; in km moc_components = tostring(fileid->moc_components) - transport_components = tostring(fileid->transport_components) + ;transport_components = tostring(fileid->transport_components) transport_regions = tostring(fileid->transport_regions) MOC = fileid->MOC MOC&$MOC!3$ = depth