Update aerosol DA to use new COM structure (#1551)

This PR updates the j-jobs and python classes for aerosol DA to use the new COM directory structure. This PR also includes removing of the chem history staging for the ICSDIR in setup_expt.py. The aerosol fields are treated as FV3 tracers, so they are either available (warm start) with the fv_tracer files or start at 0 (cold start) and must be spun up from emissions. Fixes #1516
NOAA-EMC · May 1, 2023 · 6c48e94 · 6c48e94
1 parent ec2dd3a
commit 6c48e94
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 43 deletions.
diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE
@@ -8,20 +8,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan
 ##############################################
 # Set variables used in the script
 ##############################################
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
 
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
-mkdir -p "${COMOUT}"
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_CHEM_ANALYSIS
 
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+
+mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE
@@ -7,20 +7,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlinit" -c "base aeroanl aeroanl
 ##############################################
 # Set variables used in the script
 ##############################################
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
 
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
-mkdir -p "${COMOUT}"
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_CHEM_ANALYSIS
 
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+
+mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_RUN
@@ -8,20 +8,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr
 ##############################################
 # Set variables used in the script
 ##############################################
-GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-GDUMP="gdas"
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/chem}
-mkdir -p "${COMOUT}"
-
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/chem"
-
 ###############################################################
 # Run relevant script
 

diff --git a/parm/config/config.com b/parm/config/config.com
@@ -87,5 +87,6 @@ declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history'
 declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart'
 
 declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history'
+declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem'
 
 declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart'
diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py
@@ -46,7 +46,6 @@ def __init__(self, config):
                 'npz_anl': self.config['LEVS'] - 1,
                 'AERO_WINDOW_BEGIN': _window_begin,
                 'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H",
-                'comin_ges_atm': self.config.COMIN_GES.replace('chem', 'atmos'),  # 'chem' is COMPONENT, aerosol fields are in 'atmos' tracers
                 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
@@ -143,7 +142,7 @@ def finalize(self: Analysis) -> None:
         """
         # ---- tar up diags
         # path of output tar statfile
-        aerostat = os.path.join(self.task_config['COMOUTaero'], f"{self.task_config['APREFIX']}aerostat")
+        aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat")
 
         # get list of diag files to put in tarball
         diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4'))
@@ -161,9 +160,9 @@ def finalize(self: Analysis) -> None:
 
         # copy full YAML from executable to ROTDIR
         src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
-        dest = os.path.join(self.task_config['COMOUTaero'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
+        dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml")
         yaml_copy = {
-            'mkdir': [self.task_config['COMOUTaero']],
+            'mkdir': [self.task_config.COM_CHEM_ANALYSIS],
             'copy': [[src, dest]]
         }
         FileHandler(yaml_copy).sync()
@@ -175,8 +174,8 @@ def finalize(self: Analysis) -> None:
         bkglist = []
         for itile in range(1, self.task_config.ntiles + 1):
             tracer = template.format(tilenum=itile)
-            src = os.path.join(self.task_config.comin_ges_atm, 'RESTART', tracer)
-            dest = os.path.join(self.task_config.COMOUTaero, f'aeroges.{tracer}')
+            src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer)
+            dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}')
             bkglist.append([src, dest])
         FileHandler({'copy': bkglist}).sync()
 
@@ -191,7 +190,7 @@ def finalize(self: Analysis) -> None:
         for itile in range(1, self.task_config.ntiles + 1):
             tracer = template.format(tilenum=itile)
             src = os.path.join(self.task_config.DATA, 'anl', tracer)
-            dest = os.path.join(self.task_config.COMOUTaero, tracer)
+            dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer)
             inclist.append([src, dest])
         FileHandler({'copy': inclist}).sync()
 
@@ -207,7 +206,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None:
         # only need the fv_tracer files
         template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc'
         inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + template)
-        bkg_template = os.path.join(self.task_config.comin_ges_atm, 'RESTART', template)
+        bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, template)
         # get list of increment vars
         incvars_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'parm_gdas', 'aeroanl_inc_vars.yaml')
         incvars = YAMLFile(path=incvars_list_path)['incvars']
@@ -233,7 +232,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]:
         # NOTE for now this is FV3 RESTART files and just assumed to be fh006
 
         # get FV3 RESTART files, this will be a lot simpler when using history files
-        rst_dir = os.path.join(task_config.comin_ges_atm, 'RESTART')  # for now, option later?
+        rst_dir = task_config.COM_ATMOS_RESTART_PREV
         run_dir = os.path.join(task_config['DATA'], 'bkg')
 
         # Start accumulating list of background files to copy

diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py
@@ -58,14 +58,11 @@ def fill_COMROT_cycled(host, inputs):
 
     comrot = os.path.join(inputs.comrot, inputs.pslot)
 
-    do_ocean = do_ice = do_med = do_aerosols = False
+    do_ocean = do_ice = do_med = False
 
     if inputs.app in ['S2S', 'S2SW']:
         do_ocean = do_ice = do_med = True
 
-    if inputs.app in ['ATMA']:
-        do_aerosols = True
-
     if inputs.icsdir is None:
         warnings.warn("User did not provide '--icsdir' to stage initial conditions")
         return
@@ -182,17 +179,6 @@ def link_files_from_src_to_dst(src_dir, dst_dir):
         makedirs_if_missing(dst_dir)
         link_files_from_src_to_dst(src_dir, dst_dir)
 
-    # Link aerosol files
-    if do_aerosols:
-        if inputs.start in ['warm']:
-            detdir = f'{inputs.cdump}.{rdatestr[:8]}/{rdatestr[8:]}'
-        elif inputs.start in ['cold']:
-            detdir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}'
-        dst_dir = os.path.join(comrot, detdir, chem_dir)
-        src_dir = os.path.join(inputs.icsdir, detdir, chem_dir)
-        makedirs_if_missing(dst_dir)
-        link_files_from_src_to_dst(src_dir, dst_dir)
-
     # Link bias correction and radiance diagnostics files
     detdir = f'{inputs.cdump}.{idatestr[:8]}/{idatestr[8:]}'
     src_dir = os.path.join(inputs.icsdir, detdir, src_atm_anl_dir)