diff --git a/components.yaml b/components.yaml index 14f76997..6fb642db 100644 --- a/components.yaml +++ b/components.yaml @@ -28,13 +28,13 @@ NCEP_Shared: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: rt1_4_10_geosit + tag: rt1_4_10_geosit5 develop: main MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git - tag: v2.8.0.7 + tag: v2.8.0.9 develop: develop FMS: @@ -46,7 +46,7 @@ FMS: GEOSana_GridComp: local: ./src/Components/@GEOSana_GridComp remote: ../GEOSana_GridComp.git - tag: v1.5.4.2 + tag: v1.5.4.4 develop: develop GEOSgcm_GridComp: @@ -101,7 +101,7 @@ geos-chem: GOCART: local: ./src/Components/@GEOSgcm_GridComp/GEOSagcm_GridComp/GEOSphysics_GridComp/@GEOSchem_GridComp/@GOCART remote: ../GOCART.git - tag: rt1.0.1_geosit + tag: rt1.0.1_geosit3 sparse: ./config/GOCART.sparse develop: develop @@ -121,7 +121,7 @@ mom6: GEOSgcm_App: local: ./src/Applications/@GEOSgcm_App remote: ../GEOSgcm_App.git - tag: rt1.5.6_geosit_hist2 + tag: rt1.5.6_geosit_hist5 develop: develop UMD_Etc: diff --git a/src/Applications/GAAS_App/ana.rc.tmpl b/src/Applications/GAAS_App/ana.rc.tmpl index ff285eb7..1cab053b 100644 --- a/src/Applications/GAAS_App/ana.rc.tmpl +++ b/src/Applications/GAAS_App/ana.rc.tmpl @@ -17,8 +17,9 @@ ODS_files:: # Active data (Note: deep blue in MYD04 is passive) -#___AVHRR___${FVWORK}/${EXPID}.patmosx_v05r02_L2a.asc.%y4%m2%d2_%h200z.ods -#___AVHRR___${FVWORK}/${EXPID}.patmosx_v05r02_L2a.des.%y4%m2%d2_%h200z.ods +##___AVHRR___${FVWORK}/${EXPID}.patmosx_v05r02_L2a.asc.%y4%m2%d2_%h200z.ods +##___AVHRR___${FVWORK}/${EXPID}.patmosx_v05r02_L2a.des.%y4%m2%d2_%h200z.ods +#___AVHRR___${FVWORK}/patmosx.%y4%m2%d2_%h2z.ods #___TERRA_NRT___${FVWORK}/${EXPID}.MOD04_L2a.land.%y4%m2%d2_%h200z.ods #___TERRA_NRT___${FVWORK}/${EXPID}.MOD04_L2a.ocean.%y4%m2%d2_%h200z.ods @@ -38,7 +39,7 @@ ODS_files:: #___MISR___${FVWORK}/misr_F12_0022.bright_tc8.obs.%y4%m2%d2.ods -#___AERONET___${FVWORK}/aeronet.obs.%y4%m2%d2.ods +#___AERONET___${FVWORK}/aeronet.obs.%y4%m2%d2_%h2z.ods # Passive data #/nobackup/3/PARASOL/Level2/ODS/Y%y4/M%m2/PARASOL_L2.aero_tc8.obs.%y4%m2%d2.ods diff --git a/src/Applications/GEOSdas_App/GEOSdas.csm b/src/Applications/GEOSdas_App/GEOSdas.csm index 038dc452..a377d4a1 100755 --- a/src/Applications/GEOSdas_App/GEOSdas.csm +++ b/src/Applications/GEOSdas_App/GEOSdas.csm @@ -158,7 +158,6 @@ if ( !($?STRICT) ) setenv STRICT 0 if ( !($?TIMEINC) ) setenv TIMEINC 360 if ( !($?USE_ASCAT) ) setenv USE_ASCAT 0 - if ( !($?USE_MODIS_STAGE) ) setenv USE_MODIS_STAGE 0 if ( !($?VAROFFSET) ) setenv VAROFFSET 180 if ( !($?VTRACK) ) setenv VTRACK 1 if ( !($?VTRKFRQA ) ) setenv VTRKFRQA 0 @@ -166,6 +165,12 @@ if ( !($?VTXRELOC ) ) setenv VTXRELOC 1 if ( !($?WCONSTRAINT) ) setenv WCONSTRAINT 0 + if ( $BATCH_SUBCMD == "sbatch" ) then + setenv BLOCKFLAG "-W" + else + setenv BLOCKFLAG "-W block=true" + endif + if ( !($?doPLOTS) ) setenv doPLOTS 0 if ( !($?doFSENS) ) then if ( -e stage4fsens.arc ) then @@ -613,15 +618,6 @@ exit 1 cp -f AGCM.BOOTSTRAP.rc.tmpl AGCM.rc.tmpl endif -# Turn off aerosol data sets for local acquire and replay -# ------------------------------------------------------- - if ( $LOCAL_ACQUIRE ) then - set aerosol_acquire = 0 - setenv USE_MODIS_STAGE 1 - else - set aerosol_acquire = 1 - endif - # If 4DVAR then link to adjoint nml file # -------------------------------------- if (-e fvcorepert_layout_4dvar.rc) then @@ -1557,11 +1553,7 @@ exit 1 fname2 " acquire -v -rc blend.acq -s $spool -d . -strict $nymd $nhms 060000 1 " fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname endif @@ -1699,11 +1691,7 @@ exit 1 endif fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname endif @@ -1897,11 +1885,7 @@ exit 1 fname2 " acquire -v -rc aod4fcst.acq -s $spool -d . -strict $initref[1] $initref[2] 030000 2 " fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname endif @@ -2479,11 +2463,7 @@ exit 1 fname2 "" fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname sleep 2 @ cyc_sec = $TIMEINC * 60 set next_cycle = (`tick $rpl_nymdb $rpl_nhmsb $cyc_sec`) @@ -2587,11 +2567,7 @@ exit 1 fname2 "acquire -v -rc obsys.acq -s $spool -d $FVWORK -ssh $strict $bnymd $bnhms $inhms 4" fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname sleep 2 set bnymd=`tick $bnymd` end @@ -2637,7 +2613,7 @@ exit 1 # Acquire data for obsys on cmd line # ---------------------------------- - if ( (`uname -n` !~ borg*) || ( $LOCAL_ACQUIRE ) ) then + if ( `uname -n` !~ borg* ) then # acquire tcvitals; ignore status return #--------------------------------------- @@ -2751,17 +2727,13 @@ exit 1 set obsflag = 2 set acqflags = "-d $spool -s $spool -e 999" set acqflags1 = "$acqflags" - set blockflag = "" + set qblock = "" else if ( !($?strict) ) set strict = "" set obsflag = 1 set acqflags = "-d $FVWORK -s $spool -strict" set acqflags1 = "-d $FVWORK -s $spool $strict" - if ( $BATCH_SUBCMD == "sbatch" ) then - set blockflag = "-W" - else - set blockflag = "-W block=true" - endif + set qblock = $BLOCKFLAG endif # Check for available AOD obs classes @@ -2815,7 +2787,7 @@ exit 1 if ( $AOD_OBSCLASS == "" ) setenv AOD_OBSCLASS none if ( $AOD_OBSCLASS == 0 ) setenv AOD_OBSCLASS none - if ($GAAS_ANA && $aerosol_acquire && ("$AOD_OBSCLASS" != "none")) then + if ($GAAS_ANA && ("$AOD_OBSCLASS" != "none")) then @ mstep = $nstep * 2 fname2 acquire_obsys -v $acqflags -ssh -drc $AERO_OBSDBRC \\ fname2 " "$bnymd $bnhms 030000 $mstep \\ @@ -2847,7 +2819,12 @@ exit 1 # submit job script #------------------ - $BATCH_SUBCMD $blockflag -o $acqlog $fname # acquire observations; ignore status return + if ( $LOCAL_ACQUIRE ) then + chmod 744 $fname + $fname |& tee $acqlog + else + $BATCH_SUBCMD $qblock -o $acqlog $fname # acquire observations; ignore status return + endif sleep 2 if ($obsflag == 1 && ( -e obsys.acq )) then cat obsys.acq >>! obsys.acq.all @@ -4136,11 +4113,7 @@ endif "Central AGCM Failed " if ( -e agcm_central.j ) then - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W agcm_central.j - else - qsub -W block=true agcm_central.j - endif + $BATCH_SUBCMD $BLOCKFLAG agcm_central.j else echo " ${MYNAME}: AGCM Failed to generate PBS jobs for Central, Aborting ... " exit(1) @@ -4278,22 +4251,6 @@ endif /bin/cp $FVHOME/run/gaas/avhrr_l2a.pcf $AODWORK /bin/cp $FVHOME/run/gaas/modis_l2a.pcf $AODWORK - # Reset value of MODIS_L2A_L2_DIR to $MODIS_STAGE_DIR? - # --------------------------------------------------- - if ($GAAS_ANA && $USE_MODIS_STAGE) then - if ( ! $?MODIS_STAGE_DIR ) then - echo "Error. MODIS_STAGE_DIR not defined" - echo "Aborting ... " - Call AbnormalExit_( 5 ) - endif - if ( ! -d $MODIS_STAGE_DIR ) then - echo "Error. MODIS_STAGE_DIR directory not found: $MODIS_STAGE_DIR" - echo "Aborting ... " - Call AbnormalExit_( 5 ) - endif - vED -i -vv MODIS_L2A_L2_DIR=$MODIS_STAGE_DIR $AODWORK/modis_l2a.pcf - endif - # Prepare ana.rc from template # ---------------------------- set ana_rc_tmpl = $FVHOME/run/gaas/ana.rc.tmpl @@ -4321,7 +4278,7 @@ endif sed -i "s/#___AVHRR___//" $ana_rc endif - if ($aod_obs == patmosx_asc) sed -i "s/#___AVHRR___//" $ana_rc + if ($aod_obs == patmosx_asc || $aod_obs == patmosx_ods) sed -i "s/#___AVHRR___//" $ana_rc if ($aod_obs == aeronet_obs) sed -i "s/#___AERONET___//" $ana_rc if ($aod_obs == misr_F12_bright) sed -i "s/#___MISR___//" $ana_rc if ($aod_obs == mod04_land_nnr) sed -i "s/#___TERRA___//" $ana_rc @@ -4340,12 +4297,6 @@ endif endif end -# if ($USE_MODIS_STAGE) then -# setenv MODIS_L2_HDF 1 -# sed -i "s/#___AQUA_NRT___//" $ana_rc -# sed -i "s/#___TERRA_NRT___//" $ana_rc -# endif - echo "cat $ana_rc" cat $ana_rc @@ -4441,13 +4392,8 @@ endif # submit job and save job ID #--------------------------- - if ( $BATCH_SUBCMD == "sbatch" ) then - set qblock = "-W" - else - set qblock = "-W block=true" - endif if ( $AODBLOCKJOB ) then - set jobIDline = (`$PBS_BIN/$BATCH_SUBCMD $qblock -o $gaasLOG $jobf`) + set jobIDline = (`$PBS_BIN/$BATCH_SUBCMD $BLOCKFLAG -o $gaasLOG $jobf`) else set jobIDline = (`$PBS_BIN/$BATCH_SUBCMD -o $gaasLOG $jobf`) endif @@ -4843,11 +4789,7 @@ endif fname2 " acquire_obsys -v -d $FVWORK $strict $anadate[1] $anadate[2] 060000 1 $tcvitals_class" fname2 "exit" - if ( $BATCH_SUBCMD == "sbatch" ) then - sbatch -W -o $acqlog $fname - else - qsub -W block=true -o $acqlog $fname - endif + $BATCH_SUBCMD $BLOCKFLAG -o $acqlog $fname endif ls -lrt *vtx.prs* *vtx.mix* diff --git a/src/Applications/GEOSdas_App/fvsetup b/src/Applications/GEOSdas_App/fvsetup index 869c51b0..83f8c91d 100755 --- a/src/Applications/GEOSdas_App/fvsetup +++ b/src/Applications/GEOSdas_App/fvsetup @@ -386,7 +386,7 @@ my ($acqloc); my ($fcstimes,$fcswait_hrs,$asnwait_hrs); my ($landbcs); my ($coupled, $ores, $mometc); -my ($o_servers); +my ($o_servers,$bckend_wrts); my ($sysfile, $nodeflg); my (@rmTilde); @@ -3062,7 +3062,7 @@ sub get_nodeflg { elsif ($ans == 3) { $nodeflg = "ivy"; $ncpus_per_node = 20 } elsif ($ans == 4) { $nodeflg = "hasw"; $ncpus_per_node = 24 } elsif ($ans == 5) { $nodeflg = "sky"; $ncpus_per_node = 36 } - elsif ($ans == 6) { $nodeflg = "cas"; $ncpus_per_node = 46 } + elsif ($ans == 6) { $nodeflg = "cas"; $ncpus_per_node = 45 } # currently not applicable at nccs #--------------------------------- @@ -3110,7 +3110,7 @@ sub get_times { # # ---- if ($res eq "C48") { $nx = 4; $ny = 24 } # (b) 96 elsif ($res eq "C90") { $nx = 4; $ny = 24 } # (c) 96 - elsif ($res eq "C180") { $nx = 8; $ny = 30 } # (d) 240 + elsif ($res eq "C180") { $nx = 15; $ny = 36 } # (d) 360 elsif ($res eq "C360") { $nx = 8; $ny = 48 } # (e) 384 else { $nx = 28; $ny = 48 } # (f+) 1344 } @@ -3143,7 +3143,6 @@ sub get_times { $asynbkg_min = 60; # time frequency of background fields (min) } else { $asynbkg_min = 180; # time frequency of background fields (min) - if ( $r21c) { $asynbkg_min = 60}; } } else { $splite = 0; $splitexe = 0; @@ -3444,7 +3443,6 @@ sub get_setgsi { $anaexec = "GSIsa.x"; $hybrid = ".false."; $hyb_ens = -1; - if ($r21c) { $hyb_ens = 4} if ($merra2) { $nosfcana = 0 } # will apply similarity to produce xana.sfc else { $nosfcana = 1 } # will not generation xana.sfc @@ -4735,6 +4733,7 @@ EOF $ny_pert = 6 * $nx_pert; $use_shmem = 0; $o_servers = 0; + $bckend_wrts = 0; $ios_nds = 1; $cldmicro = "1MOMENT"; if ( "$res" eq "c" && "$vres" eq "55" ) { @@ -5007,6 +5006,10 @@ EOF $ana_im_ens = $ana_im; $ana_jm_ens = $ana_jm; } elsif ( "$res" eq "C180" ) { # Cubed-sphere + if ($geosit) { + $o_servers = 3; + $bckend_wrts = 24; + } $cubed = 1; # $ios_nds = 2; $specres = "254"; @@ -5076,7 +5079,7 @@ EOF } elsif ( "$res" eq "C720" ) { # Cubed-sphere $cubed = 1; $o_servers = 8; -# $ios_nds = 3; + $bckend_wrts = 8; $specres = "254"; $jcap = "254"; $agcm_grid_type = "Cubed-Sphere"; @@ -7668,6 +7671,7 @@ print SCRIPT <<"EOF"; setenv NCPUS_GPERT $ncpus_gpert # Number of CPUs to run gcmPERT setenv NCPUS_AOD $ncpus_aod # Number of CPUs to run PSAS-AOD setenv O_SERVERS $o_servers # Number of IO servers + setenv O_WRITERS_PER_NODE $bckend_wrts # Number of backend writers #setenv GAAS_RUN_SLURM 1 # launch AOD analysis as separate batch job setenv AODBLOCKJOB 1 EOF @@ -7773,8 +7777,6 @@ print SCRIPT <<"EOF"; setenv LDHOME4ENS $ldashome4ens # land analysis home dir for atm_ens setenv IGNORE_0 1 # 1 = ignore 0 length obs files in acquire setenv ACFTBIAS $acftbias - setenv USE_MODIS_STAGE 0 # 1 = use MODIS data from MODIS_STAGE_DIR; 0 = don't - setenv MODIS_STAGE_DIR /discover/nobackup/projects/gmao/input/dao_ops/ops/flk/modis # L2a non-archive data dir setenv FCSTIMES "$fcstimes" setenv FCSTATS "21" setenv FHOURS $fhours21 @@ -7784,9 +7786,6 @@ print SCRIPT <<"EOF"; setenv GSI_DIAG2TXT 1 # setenv DO_0HR_IMP 1 - if ( -e \$FVHOME/run/replay.acq ) then - setenv USE_MODIS_STAGE 1 # for now, replay.acq controls only meteorology (AOD is not yet replayed to) - endif EOF if ( $merra2 ) { @@ -8060,7 +8059,7 @@ print SCRIPT <<"EOF"; setenv RUN_OPT_BLEND "esma_mpirun -np \$NCPUS \$HDF2RSX" if (\$G5GCM ) then if ( \$O_SERVERS > 0 ) then - setenv RUN_OPT_BEGIN "mpirun -np \$NCPUSX \$GCMX --npes_model \$NCUPS --nodes_output_server \$O_SERVERS --oserver_type multigroup --npes_backend_pernode \$O_SERVERS" + setenv RUN_OPT_BEGIN "mpirun -np \$NCPUSX \$GCMX --npes_model \$NCPUS --nodes_output_server \$O_SERVERS --oserver_type multigroup --npes_backend_pernode \$O_WRITERS_PER_NODE" else setenv RUN_OPT_BEGIN "esma_mpirun -np \$NCPUS \$GCMX" endif @@ -8851,6 +8850,7 @@ print SCRIPT <<"EOF"; setenv NCPUS_GSI $ncpus_gsi # Numbers of CPUs to run GSI setenv NCPUS_GPERT $ncpus_gpert # Numbers of CPUs to run GSI setenv O_SERVERS $o_servers # Number of IO servers + setenv O_WRITERS_PER_NODE $bckend_wrts # Backend writers setenv N_CPU \$NCPUS setenv EXPID $expid # experiment ID setenv CASE \$EXPID # experiment ID (for LSM's sake) @@ -9251,7 +9251,7 @@ print SCRIPT <<"EOF"; setenv RUN_OPT_BLEND "esma_mpirun -np \$NCPUS \$HDF2RSX" if (\$G5GCM ) then if ( \$O_SERVERS > 0 ) then - setenv RUN_OPT_BEGIN "mpirun -np \$NCPUSX \$GCMX --npes_model \$NCUPS --nodes_output_server \$O_SERVERS --oserver_type multigroup --npes_backend_pernode \$O_SERVERS" + setenv RUN_OPT_BEGIN "mpirun -np \$NCPUSX \$GCMX --npes_model \$NCPUS --nodes_output_server \$O_SERVERS --oserver_type multigroup --npes_backend_pernode \$O_WRITERS_PER_NODE" else setenv RUN_OPT_BEGIN "esma_mpirun -np \$NCPUS \$GCMX" endif @@ -10115,8 +10115,20 @@ sub copy_resources { } } } + # Update gocart files for pre-2000 case - # To Be Done + if ( $r21c ) { $casedir = "R21C" }; + if ( $geosit ) { $casedir = "GEOSIT"}; + if ( $r21c || $geosit ){ + if ( $nymdb < 20000331 && $hhb < 23 ) { + if ( -d "$fvetc/$casedir" ){ + my @files = glob("$fvetc/$casedir/19600101-20000331/" . "/*"); + foreach my $fn ( @files ) { + cp("$fn","$fvhome/run/gocart"); + } + } + } + } # # Edit psas.rc diff --git a/src/Applications/GEOSdas_App/testsuites/geos_it.input b/src/Applications/GEOSdas_App/testsuites/geos_it.input index 8664196a..1b00f2fc 100644 --- a/src/Applications/GEOSdas_App/testsuites/geos_it.input +++ b/src/Applications/GEOSdas_App/testsuites/geos_it.input @@ -98,11 +98,11 @@ Length of FORECAST run segments (in hours)? [123] Number of one-day DAS segments per PBS job? [1] > -Number of PEs in the zonal direction (NX)? [8] -> +Number of PEs in the zonal direction (NX)? [15] +> 15 -Number of PEs in the meridional direction (NY)? [30] -> 48 +Number of PEs in the meridional direction (NY)? [36] +> 36 Job nickname? [g5das] > git @@ -195,7 +195,7 @@ Do Aerosol Analysis (y/n)? [y] > AOD OBSERVING CLASSES [or type 'none']? -> mod04_061_his,myd04_061_his +> mod04_061_his,myd04_061_his,aeronet_obs Enable GAAS feedback to model (y/n)? [y] >