From 72cfc5b1b00bb03d725c76f1dca1879067a55a7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 2 Aug 2024 13:53:28 +0000 Subject: [PATCH 01/44] Initial commit --- .../exglobal_atm_analysis_fv3_increment.py | 12 +- scripts/exglobal_atm_analysis_initialize.py | 2 +- scripts/exglobal_atm_analysis_variational.py | 2 +- .../exglobal_atmens_analysis_fv3_increment.py | 12 +- .../exglobal_atmens_analysis_initialize.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 6 +- ush/python/pygfs/task/atm_analysis.py | 101 +++---- ush/python/pygfs/task/atmens_analysis.py | 112 +++---- ush/python/pygfs/task/jedi.py | 281 ++++++++++++++++++ 9 files changed, 374 insertions(+), 156 deletions(-) create mode 100644 ush/python/pygfs/task/jedi.py diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 66f6796343..ffdd1a18c1 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the init_fv3_increment and fv3_increment methods +# and runs the initialize_fv3inc and execute methods # which convert the JEDI increment into an FV3 increment import os @@ -17,7 +17,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) - AtmAnl.init_fv3_increment() - AtmAnl.fv3_increment() + # Instantiate the atm analysis object + AtmAnlFV3Inc = AtmAnalysis(config) + + # Initialize and execute + AtmAnlFV3Inc.initialize_fv3inc() + AtmAnlFV3Inc.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 1793b24b0b..3f908372cd 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -21,4 +21,4 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) - AtmAnl.initialize() + AtmAnl.initialize_var() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 07bc208331..e99552d1a5 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -19,4 +19,4 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) - AtmAnl.variational() + AtmAnl.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index c50b00548f..2245288d3f 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_fv3_increment.py # This script creates an AtmEnsAnalysis object -# and runs the init_fv3_increment and fv3_increment methods +# and runs the initialize_fv3inc and execute methods # which convert the JEDI increment into an FV3 increment import os @@ -17,7 +17,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_fv3_increment() - AtmEnsAnl.fv3_increment() + # Instantiate the atmens analysis object + AtmEnsAnlFV3Inc = AtmEnsAnalysis(config) + + # Initialize and execute + AtmEnsAnlFV3Inc.initialize_fv3inc() + AtmEnsAnlFV3Inc.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 1d578b44f2..590664b951 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -21,4 +21,4 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize() + AtmEnsAnl.initialize_var() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 30394537cd..6d0112d69d 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the letkf method -# which executes the global atm local ensemble analysis +# and runs the execute method which executes +# the global atm local ensemble analysis import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -19,4 +19,4 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.letkf() + AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda'])) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c..7268e1e126 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -5,24 +5,21 @@ import gzip import tarfile from logging import getLogger -from typing import Dict, List, Any from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, - chdir, - parse_j2yaml, save_as_yaml, - logit, - Executable, - WorkflowException) -from pygfs.task.analysis import Analysis + Task, + parse_j2yaml, + logit) +from pygfs.task.jedi import JEDI logger = getLogger(__name__.split('.')[-1]) -class AtmAnalysis(Analysis): +class AtmAnalysis(Task): """ - Class for global atm analysis tasks + Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") def __init__(self, config): @@ -31,7 +28,6 @@ def __init__(self, config): _res = int(self.task_config.CASE[1:]) _res_anl = int(self.task_config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -48,7 +44,6 @@ def __init__(self, config): 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications @@ -58,21 +53,39 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create JEDI object + self.jedi = JEDI(self.task_config) + @logit(logger) - def initialize(self: Analysis) -> None: + def initialize(self) -> None: """Initialize a global atm analysis This method will initialize a global atm analysis using JEDI. This includes: + - staging observation files + - staging bias correction files - staging CRTM fix files - staging FV3-JEDI fix files - staging B error files - staging model backgrounds - - generating a YAML file for the JEDI executable - creating output directories """ super().initialize() + # get JEDI variational configuration and save to YAML file + self.jedi.get_config(self.task_config) + + # link JEDI variational executable + self.jedi.link_exe(self.task_config) + + # stage observations + obs_dict = self.jedi.get_obs_dict() + FileHandler(obs_dict).sync() + + # stage bias corrections + bias_dict = self.jedi.get_bias_dict() + FileHandler(bias_dict).sync() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) @@ -102,11 +115,6 @@ def initialize(self: Analysis) -> None: bkg_staging_dict = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() - # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -116,54 +124,13 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def variational(self: Analysis) -> None: - - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('variational') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass + def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + super().execute() + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) + @logit(logger) - def init_fv3_increment(self: Analysis) -> None: - # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, - f"{self.task_config.JCB_ALGO}.yaml") - save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) - - # Link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() - - @logit(logger) - def fv3_increment(self: Analysis) -> None: - # Run executable - exec_cmd = Executable(self.task_config.APRUN_ATMANLFV3INC) - exec_cmd.add_default_arg(self.task_config.jedi_exe) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - @logit(logger) - def finalize(self: Analysis) -> None: + def finalize(self) -> None: """Finalize a global atm analysis This method will finalize a global atm analysis using JEDI. @@ -171,9 +138,9 @@ def finalize(self: Analysis) -> None: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - copy the updated bias correction files to ROTDIR - - write UFS model readable atm incrment file - """ + super().finalize() + # ---- tar up diags # path of output tar statfile atmstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat") @@ -198,7 +165,7 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") + src = self.task_config.jedi_yaml dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index bd5112050e..1114f757a4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -11,19 +11,20 @@ FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, to_YMD, chdir, + Task, parse_j2yaml, save_as_yaml, logit, Executable, WorkflowException, Template, TemplateConstants) -from pygfs.task.analysis import Analysis +from pygfs.task.jedi import JEDI logger = getLogger(__name__.split('.')[-1]) -class AtmEnsAnalysis(Analysis): +class AtmEnsAnalysis(Task): """ - Class for global atmens analysis tasks + Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") def __init__(self, config): @@ -31,7 +32,6 @@ def __init__(self, config): _res = int(self.task_config.CASE_ENS[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -45,7 +45,6 @@ def __init__(self, config): 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications @@ -55,21 +54,25 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create JEDI object + self.jedi = JEDI(self.task_config) + @logit(logger) - def initialize(self: Analysis) -> None: + def initialize_letkf(self) -> None: """Initialize a global atmens analysis This method will initialize a global atmens analysis using JEDI. This includes: + - staging observation files + - staging bias correction files - staging CRTM fix files - staging FV3-JEDI fix files - staging model backgrounds - - generating a YAML file for the JEDI executable - creating output directories Parameters ---------- - Analysis: parent class for GDAS task + None Returns ---------- @@ -77,6 +80,20 @@ def initialize(self: Analysis) -> None: """ super().initialize() + # get JEDI ensemble DA config dictionary and save to YAML file + self.jedi.get_config(self.task_config) + + # link JEDI ensemble DA executable + self.jedi.link_exe(self.task_config) + + # stage observations + obs_dict = self.jedi.get_obs_dict() + FileHandler(obs_dict).sync() + + # stage bias corrections + bias_dict = self.jedi.get_bias_dict() + FileHandler(bias_dict).sync() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) @@ -92,11 +109,6 @@ def initialize(self: Analysis) -> None: bkg_staging_dict = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() - # generate ensemble da YAML file - logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -106,76 +118,27 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def letkf(self: Analysis) -> None: - """Execute a global atmens analysis + def initialize_fv3inc(self): + # get JEDI-to-FV3 increment converter config and save to YAML file + self.jedi.get_config(self.task_config) - This method will execute a global atmens analysis using JEDI. - This includes: - - changing to the run directory - - running the global atmens analysis executable - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLLETKF) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('localensembleda') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def init_fv3_increment(self: Analysis) -> None: - # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, - f"{self.task_config.JCB_ALGO}.yaml") - save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) - - # Link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() + # link JEDI-to-FV3 increment converter executable + self.jedi.link_exe(self.task_config) @logit(logger) - def fv3_increment(self: Analysis) -> None: - # Run executable - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLFV3INC) - exec_cmd.add_default_arg(self.task_config.jedi_exe) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + super().execute() + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) + @logit(logger) - def finalize(self: Analysis) -> None: + def finalize(self) -> None: """Finalize a global atmens analysis This method will finalize a global atmens analysis using JEDI. This includes: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - - write UFS model readable atm incrment file Parameters ---------- @@ -185,6 +148,8 @@ def finalize(self: Analysis) -> None: ---------- None """ + super().finalize() + # ---- tar up diags # path of output tar statfile atmensstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX}atmensstat") @@ -209,7 +174,7 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") + src = self.task_config.jedi_yaml dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { @@ -231,6 +196,7 @@ def finalize(self: Analysis) -> None: logger.info("Copy UFS model readable atm increment file") cdate = to_fv3time(self.task_config.current_cycle) cdate_inc = cdate.replace('.', '_') + # loop over ensemble members for imem in range(1, self.task_config.NMEM_ENS + 1): memchar = f"mem{imem:03d}" diff --git a/ush/python/pygfs/task/jedi.py b/ush/python/pygfs/task/jedi.py new file mode 100644 index 0000000000..f7d501def3 --- /dev/null +++ b/ush/python/pygfs/task/jedi.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from pprint import pformat +from typing import List, Dict, Any, Optional +from jcb import render +from wxflow import (AttrDict, + chdir, rm_p, + parse_j2yaml, save_as_yaml, + logit, + Task, + Executable, + WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class JEDI: + + def __init__(self, task_config: AttrDict[str, Any]) -> None: + + _exe_name = os.path.basename(task_config.JEDIEXE) + + self.exe = os.path.join(task_config.DATA, _exe_name) + self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + self.config = {} + self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + + @logit(logger) + def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> None: + """Compile a JEDI configuration dictionary from a template file and save to a YAML file + + Parameters + ---------- + task_config : AttrDict + Dictionary of all configuration variables associated with a GDAS task. + algorithm (optional) : str + Name of the algorithm used to generate the JEDI configuration dictionary. + It will override the algorithm set in the task_config.JCB_<>_YAML file. + + Returns + ---------- + None + """ + + logger.info(f"Generate JEDI YAML config: {self.yaml}") + + if 'JCB_BASE_YAML' in task_config.keys(): + # Step 1: Fill templates of the JCB base YAML file + jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + + # Step 2: If algorithm is present then override the algorithm in the JEDI + # config. Otherwise, if the algorithm J2-YAML is present, fill + # its templates and merge. + if algorithm: + jcb_config['algorithm'] = algorithm + elif 'JCB_ALGO' in task_config.keys(): + jcb_config['algorithm'] = task_config.JCB_ALGO + elif 'JCB_ALGO_YAML' in task_config.keys(): + jcb_algo_config = parse_j2yaml(task_config.JCB_ALGO_YAML, task_config) + jcb_config = {**jcb_config, **jcb_algo_config} + + # Step 3: Generate the JEDI YAML using JCB + self.config = render(jcb_config) + elif 'JEDIYAML' in task_config.keys(): + # Generate JEDI YAML without using JCB + self.config = parse_j2yaml(task_config.JEDIYAML, task_config, + searchpath=self.j2tmpl_dir) + else: + raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") + + logger.debug(f"JEDI config:\n{pformat(self.config)}") + + # Save YAML to disk + logger.debug(f"Write YAML file to: {self.yaml}") + save_as_yaml(self.config, self.yaml) + logger.info(f"Wrote YAML file to: {self.yaml}") + + @logit(logger) + def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + """Execute JEDI application + + Parameters + ---------- + task_config : AttrDict + Dictionary of all configuration variables associated with a GDAS task. + aprun_cmd: str + String comprising the run command for the JEDI executable. + jedi_args (optional): List + List of strings comprising optional input arguments for the JEDI executable. + + Returns + ---------- + None + """ + + chdir(task_config.DATA) + + exec_cmd = Executable(aprun_cmd) + exec_cmd.add_default_arg(self.exe) + if jedi_args: + for arg in jedi_args: + exec_cmd.add_default_arg(arg) + exec_cmd.add_default_arg(self.yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + def link_exe(self, task_config: AttrDict[str, Any]) -> None: + """Link JEDI executable to run directory + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {task_config.JEDIEXE} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(task_config.JEDIEXE, exe_dest) + + @logit(logger) + def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: + """Compile a dictionary of observation files to copy + + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation files that are to be copied to the run directory + from the observation input directory + + Parameters + ---------- + None + + Returns + ---------- + obs_dict: Dict + a dictionary containing the list of observation files to copy for FileHandler + """ + + logger.info(f"Extracting a list of observation files from Jedi config file") + observations = find_value_in_nested_dict(self.config, 'observations') + logger.debug(f"observations:\n{pformat(observations)}") + + copylist = [] + for ob in observations['observers']: + obfile = ob['obs space']['obsdatain']['engine']['obsfile'] + basename = os.path.basename(obfile) + copylist.append([os.path.join(task_config.COM_OBS, basename), obfile]) + obs_dict = { + 'mkdir': [os.path.join(task_config.DATA, 'obs')], + 'copy': copylist + } + return obs_dict + + @logit(logger) + def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: + """Compile a dictionary of observation files to copy + + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation bias correction files that are to be copied to the run directory + from the component directory. + TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in + `analysis.py` and should be implemented in the component where this is applicable. + + Parameters + ---------- + None + + Returns + ---------- + bias_dict: Dict + a dictionary containing the list of observation bias files to copy for FileHandler + """ + + logger.info(f"Extracting a list of bias correction files from Jedi config file") + observations = find_value_in_nested_dict(self.config, 'observations') + logger.debug(f"observations:\n{pformat(observations)}") + + copylist = [] + for ob in observations['observers']: + if 'obs bias' in ob.keys(): + obfile = ob['obs bias']['input file'] + obdir = os.path.dirname(obfile) + basename = os.path.basename(obfile) + prefix = '.'.join(basename.split('.')[:-2]) + for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: + bfile = f"{prefix}.{file}" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + # TODO: Why is this specific to ATMOS? + + bias_dict = { + 'mkdir': [os.path.join(task_config.DATA, 'bc')], + 'copy': copylist + } + return bias_dict + + +@logit(logger) +def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: + """ + Recursively search through a nested dictionary and return the value for the target key. + This returns the first target key it finds. So if a key exists in a subsequent + nested dictionary, it will not be found. + + Parameters + ---------- + nested_dict : Dict + Dictionary to search + target_key : str + Key to search for + + Returns + ------- + Any + Value of the target key + + Raises + ------ + KeyError + If key is not found in dictionary + + TODO: if this gives issues due to landing on an incorrect key in the nested + dictionary, we will have to implement a more concrete method to search for a key + given a more complete address. See resolved conversations in PR 2387 + + # Example usage: + nested_dict = { + 'a': { + 'b': { + 'c': 1, + 'd': { + 'e': 2, + 'f': 3 + } + }, + 'g': 4 + }, + 'h': { + 'i': 5 + }, + 'j': { + 'k': 6 + } + } + + user_key = input("Enter the key to search for: ") + result = find_value_in_nested_dict(nested_dict, user_key) + """ + + if not isinstance(nested_dict, dict): + raise TypeError(f"Input is not of type(dict)") + + result = nested_dict.get(target_key) + if result is not None: + return result + + for value in nested_dict.values(): + if isinstance(value, dict): + try: + result = find_value_in_nested_dict(value, target_key) + if result is not None: + return result + except KeyError: + pass + + raise KeyError(f"Key '{target_key}' not found in the nested dictionary") From b2536c5d1099b01cc4f7d2d569f642a5eb2e9a66 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 8 Aug 2024 17:40:18 +0000 Subject: [PATCH 02/44] Move YAML save out of get_jedi_config method --- ush/python/pygfs/task/atm_analysis.py | 7 ++++++- ush/python/pygfs/task/atmens_analysis.py | 7 ++++++- ush/python/pygfs/task/jedi.py | 5 ----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 7268e1e126..c3dd50bab8 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -62,6 +62,7 @@ def initialize(self) -> None: This method will initialize a global atm analysis using JEDI. This includes: + - generating and saving JEDI YAML config - staging observation files - staging bias correction files - staging CRTM fix files @@ -72,9 +73,13 @@ def initialize(self) -> None: """ super().initialize() - # get JEDI variational configuration and save to YAML file + # get JEDI variational configuration self.jedi.get_config(self.task_config) + # save JEDI config to YAML file + logger.debug(f"Writing JEDI YAML file to: {self.yaml}") + save_as_yaml(jedi.config, jedi.yaml) + # link JEDI variational executable self.jedi.link_exe(self.task_config) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1114f757a4..9f94ad6b00 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -63,6 +63,7 @@ def initialize_letkf(self) -> None: This method will initialize a global atmens analysis using JEDI. This includes: + - generating and saving JEDI YAML config - staging observation files - staging bias correction files - staging CRTM fix files @@ -80,9 +81,13 @@ def initialize_letkf(self) -> None: """ super().initialize() - # get JEDI ensemble DA config dictionary and save to YAML file + # get JEDI ensemble DA config dictionary self.jedi.get_config(self.task_config) + # save JEDI config to YAML file + logger.debug(f"Writing JEDI YAML file to: {self.yaml}") + save_as_yaml(jedi.config, jedi.yaml) + # link JEDI ensemble DA executable self.jedi.link_exe(self.task_config) diff --git a/ush/python/pygfs/task/jedi.py b/ush/python/pygfs/task/jedi.py index f7d501def3..8339b34a0c 100644 --- a/ush/python/pygfs/task/jedi.py +++ b/ush/python/pygfs/task/jedi.py @@ -72,11 +72,6 @@ def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = logger.debug(f"JEDI config:\n{pformat(self.config)}") - # Save YAML to disk - logger.debug(f"Write YAML file to: {self.yaml}") - save_as_yaml(self.config, self.yaml) - logger.info(f"Wrote YAML file to: {self.yaml}") - @logit(logger) def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application From a6bc7cc2d42f71a341015d5566abcac3f7817d4b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 8 Aug 2024 18:34:44 +0000 Subject: [PATCH 03/44] pynorms --- ush/python/pygfs/task/atm_analysis.py | 8 ++++---- ush/python/pygfs/task/atmens_analysis.py | 10 +++++----- ush/python/pygfs/task/jedi.py | 18 +++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c3dd50bab8..3b80ad43ea 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -9,7 +9,7 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, - Task, + Task, parse_j2yaml, logit) from pygfs.task.jedi import JEDI @@ -55,7 +55,7 @@ def __init__(self, config): # Create JEDI object self.jedi = JEDI(self.task_config) - + @logit(logger) def initialize(self) -> None: """Initialize a global atm analysis @@ -82,7 +82,7 @@ def initialize(self) -> None: # link JEDI variational executable self.jedi.link_exe(self.task_config) - + # stage observations obs_dict = self.jedi.get_obs_dict() FileHandler(obs_dict).sync() @@ -133,7 +133,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - + @logit(logger) def finalize(self) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 9f94ad6b00..9ad01fd551 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -56,7 +56,7 @@ def __init__(self, config): # Create JEDI object self.jedi = JEDI(self.task_config) - + @logit(logger) def initialize_letkf(self) -> None: """Initialize a global atmens analysis @@ -86,11 +86,11 @@ def initialize_letkf(self) -> None: # save JEDI config to YAML file logger.debug(f"Writing JEDI YAML file to: {self.yaml}") - save_as_yaml(jedi.config, jedi.yaml) - + save_as_yaml(jedi.config, jedi.yaml) + # link JEDI ensemble DA executable self.jedi.link_exe(self.task_config) - + # stage observations obs_dict = self.jedi.get_obs_dict() FileHandler(obs_dict).sync() @@ -135,7 +135,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() self.jedi.execute(self.task_config, aprun_cmd, jedi_args) - + @logit(logger) def finalize(self) -> None: """Finalize a global atmens analysis diff --git a/ush/python/pygfs/task/jedi.py b/ush/python/pygfs/task/jedi.py index 8339b34a0c..d55eb79fbb 100644 --- a/ush/python/pygfs/task/jedi.py +++ b/ush/python/pygfs/task/jedi.py @@ -19,14 +19,14 @@ class JEDI: def __init__(self, task_config: AttrDict[str, Any]) -> None: - + _exe_name = os.path.basename(task_config.JEDIEXE) - self.exe = os.path.join(task_config.DATA, _exe_name) - self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') - self.config = {} + self.exe = os.path.join(task_config.DATA, _exe_name) + self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + self.config = {} self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') - + @logit(logger) def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> None: """Compile a JEDI configuration dictionary from a template file and save to a YAML file @@ -45,13 +45,13 @@ def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = """ logger.info(f"Generate JEDI YAML config: {self.yaml}") - + if 'JCB_BASE_YAML' in task_config.keys(): # Step 1: Fill templates of the JCB base YAML file jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) # Step 2: If algorithm is present then override the algorithm in the JEDI - # config. Otherwise, if the algorithm J2-YAML is present, fill + # config. Otherwise, if the algorithm J2-YAML is present, fill # its templates and merge. if algorithm: jcb_config['algorithm'] = algorithm @@ -89,7 +89,7 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op ---------- None """ - + chdir(task_config.DATA) exec_cmd = Executable(aprun_cmd) @@ -128,7 +128,7 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: if os.path.exists(exe_dest): rm_p(exe_dest) os.symlink(task_config.JEDIEXE, exe_dest) - + @logit(logger) def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: """Compile a dictionary of observation files to copy From f5f4d94719bf375632fef472530d5b338913f0b8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:43:06 -0400 Subject: [PATCH 04/44] Update exglobal_atmens_analysis_letkf.py From e5b74098fedc1d9c3b9ddf062d05175e2b84bedc Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 21 Aug 2024 13:44:42 +0000 Subject: [PATCH 05/44] Addressing Rahul's comments --- ush/python/pygfs/task/atm_analysis.py | 13 +++++--- ush/python/pygfs/task/atmens_analysis.py | 15 ++++++--- ush/python/pygfs/task/jedi.py | 39 ++++++++++++------------ 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 3b80ad43ea..7377f942b0 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -73,22 +73,27 @@ def initialize(self) -> None: """ super().initialize() - # get JEDI variational configuration - self.jedi.get_config(self.task_config) + # set JEDI variational configuration + logger.info(f"Generate JEDI YAML config: {self.yaml}") + jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") # save JEDI config to YAML file logger.debug(f"Writing JEDI YAML file to: {self.yaml}") save_as_yaml(jedi.config, jedi.yaml) # link JEDI variational executable + logger.debug(f"Linking JEDI variational DA executable to: {self.exe}") self.jedi.link_exe(self.task_config) # stage observations - obs_dict = self.jedi.get_obs_dict() + logger.info(f"Staging observations") + obs_dict = self.jedi.get_obs_dict(self.task_config) FileHandler(obs_dict).sync() # stage bias corrections - bias_dict = self.jedi.get_bias_dict() + logger.info(f"Staging bias corrections") + bias_dict = self.jedi.get_bias_dict(self.task_config) FileHandler(bias_dict).sync() # stage CRTM fix files diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 9ad01fd551..2de06fbc63 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -81,22 +81,27 @@ def initialize_letkf(self) -> None: """ super().initialize() - # get JEDI ensemble DA config dictionary - self.jedi.get_config(self.task_config) + # set JEDI ensemble DA config dictionary + logger.info(f"Generate JEDI YAML config: {self.yaml}") + jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML file to: {self.yaml}") + logger.info(f"Writing JEDI YAML file to: {self.yaml}") save_as_yaml(jedi.config, jedi.yaml) # link JEDI ensemble DA executable + logger.debug(f"Linking JEDI ensemble DA executable to: {self.exe}") self.jedi.link_exe(self.task_config) # stage observations - obs_dict = self.jedi.get_obs_dict() + logger.info(f"Staging observations") + obs_dict = self.jedi.get_obs_dict(self.task_config) FileHandler(obs_dict).sync() # stage bias corrections - bias_dict = self.jedi.get_bias_dict() + logger.info(f"Staging bias corrections") + bias_dict = self.jedi.get_bias_dict(self.task_config) FileHandler(bias_dict).sync() # stage CRTM fix files diff --git a/ush/python/pygfs/task/jedi.py b/ush/python/pygfs/task/jedi.py index d55eb79fbb..d061da8368 100644 --- a/ush/python/pygfs/task/jedi.py +++ b/ush/python/pygfs/task/jedi.py @@ -20,15 +20,18 @@ class JEDI: def __init__(self, task_config: AttrDict[str, Any]) -> None: + # For provenance, save incoming task_config as a private attribute of JEDI object + self._task_config = task_config + _exe_name = os.path.basename(task_config.JEDIEXE) self.exe = os.path.join(task_config.DATA, _exe_name) self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') - self.config = {} + self.config = AttrDict() self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') - + @logit(logger) - def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> None: + def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> AttrDict """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters @@ -44,8 +47,6 @@ def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None """ - logger.info(f"Generate JEDI YAML config: {self.yaml}") - if 'JCB_BASE_YAML' in task_config.keys(): # Step 1: Fill templates of the JCB base YAML file jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) @@ -69,17 +70,15 @@ def get_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = searchpath=self.j2tmpl_dir) else: raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") - - logger.debug(f"JEDI config:\n{pformat(self.config)}") - + @logit(logger) def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application Parameters ---------- - task_config : AttrDict - Dictionary of all configuration variables associated with a GDAS task. + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. aprun_cmd: str String comprising the run command for the JEDI executable. jedi_args (optional): List @@ -87,7 +86,8 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op Returns ---------- - None + jedi_config: AttrDict + Attribute-dictionary of JEDI configuration rendered from a template. """ chdir(task_config.DATA) @@ -106,9 +106,8 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op raise OSError(f"Failed to execute {exec_cmd}") except Exception: raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - + + @logit(logger) def link_exe(self, task_config: AttrDict[str, Any]) -> None: """Link JEDI executable to run directory @@ -128,7 +127,7 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: if os.path.exists(exe_dest): rm_p(exe_dest) os.symlink(task_config.JEDIEXE, exe_dest) - + @logit(logger) def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: """Compile a dictionary of observation files to copy @@ -139,7 +138,8 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: Parameters ---------- - None + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. Returns ---------- @@ -161,8 +161,9 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: 'copy': copylist } return obs_dict - + @logit(logger) + @staticmethod def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: """Compile a dictionary of observation files to copy @@ -174,7 +175,8 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: Parameters ---------- - None + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. Returns ---------- @@ -204,7 +206,6 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: } return bias_dict - @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ From e0bd76c188f13d62bdfe8aa6030e7c5724624b5d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 21 Aug 2024 14:26:09 +0000 Subject: [PATCH 06/44] Move jedi.py to its own directory and move logging from jedi.py functions to actual task --- ush/python/pygfs/jedi/__init__.py | 0 ush/python/pygfs/{task => jedi}/jedi.py | 5 --- ush/python/pygfs/task/atm_analysis.py | 40 +++++++++++++++++------- ush/python/pygfs/task/atmens_analysis.py | 31 +++++++++++------- 4 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 ush/python/pygfs/jedi/__init__.py rename ush/python/pygfs/{task => jedi}/jedi.py (96%) diff --git a/ush/python/pygfs/jedi/__init__.py b/ush/python/pygfs/jedi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/python/pygfs/task/jedi.py b/ush/python/pygfs/jedi/jedi.py similarity index 96% rename from ush/python/pygfs/task/jedi.py rename to ush/python/pygfs/jedi/jedi.py index d061da8368..3d3ca1949d 100644 --- a/ush/python/pygfs/task/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -121,7 +121,6 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: """ # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {task_config.JEDIEXE} to DATA/") logger.warn("Linking is not permitted per EE2.") exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) if os.path.exists(exe_dest): @@ -147,9 +146,7 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: a dictionary containing the list of observation files to copy for FileHandler """ - logger.info(f"Extracting a list of observation files from Jedi config file") observations = find_value_in_nested_dict(self.config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") copylist = [] for ob in observations['observers']: @@ -184,9 +181,7 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: a dictionary containing the list of observation bias files to copy for FileHandler """ - logger.info(f"Extracting a list of bias correction files from Jedi config file") observations = find_value_in_nested_dict(self.config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") copylist = [] for ob in observations['observers']: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 7377f942b0..8a2fad3829 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -12,7 +12,7 @@ Task, parse_j2yaml, logit) -from pygfs.task.jedi import JEDI +from pygfs.jedi.jedi import JEDI logger = getLogger(__name__.split('.')[-1]) @@ -74,37 +74,41 @@ def initialize(self) -> None: super().initialize() # set JEDI variational configuration - logger.info(f"Generate JEDI YAML config: {self.yaml}") + logger.info(f"Generating JEDI YAML config: {jedi.yaml}") jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.debug(f"JEDI config:\n{pformat(jedi.config)}") # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML file to: {self.yaml}") + logger.debug(f"Writing JEDI YAML config to: {jedi.yaml}") save_as_yaml(jedi.config, jedi.yaml) # link JEDI variational executable - logger.debug(f"Linking JEDI variational DA executable to: {self.exe}") + logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") self.jedi.link_exe(self.task_config) # stage observations - logger.info(f"Staging observations") + logger.info(f"Staging list of observation files generated from JEDI config") obs_dict = self.jedi.get_obs_dict(self.task_config) FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections - logger.info(f"Staging bias corrections") + logger.info(f"Staging list of bias correction files generated from JEDI config") bias_dict = self.jedi.get_bias_dict(self.task_config) FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") - crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) - FileHandler(crtm_fix_list).sync() + crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) + FileHandler(crtm_fix_dict).sync() + logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") # stage fix files logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() + jedi_fix_dict = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_dict).sync() + logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") # stage static background error files, otherwise it will assume ID matrix logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") @@ -113,17 +117,20 @@ def initialize(self) -> None: else: berror_staging_dict = {} FileHandler(berror_staging_dict).sync() + logger.debug(f"Background error files:\n{pformat(berror_staging_dict)}") # stage ensemble files for use in hybrid background error if self.task_config.DOHYBVAR: logger.debug(f"Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") fv3ens_staging_dict = parse_j2yaml(self.task_config.FV3ENS_STAGING_YAML, self.task_config) FileHandler(fv3ens_staging_dict).sync() + logger.debug(f"Ensemble files:\n{pformat(fv3ens_staging_dict)}") # stage backgrounds logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") bkg_staging_dict = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() + logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}" # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -133,6 +140,17 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() + @logit(logger) + def initialize_fv3inc(self): + # get JEDI-to-FV3 increment converter config and save to YAML file + logger.info(f"Generating JEDI YAML config: {self.yaml}") + self.jedi.get_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") + + # link JEDI-to-FV3 increment converter executable + logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") + self.jedi.link_exe(self.task_config) + @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 2de06fbc63..25221e5ac3 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os +1;95;0cimport os import glob import gzip import tarfile @@ -17,7 +17,7 @@ Executable, WorkflowException, Template, TemplateConstants) -from pygfs.task.jedi import JEDI +from pygfs.jedi.jedi import JEDI logger = getLogger(__name__.split('.')[-1]) @@ -82,42 +82,47 @@ def initialize_letkf(self) -> None: super().initialize() # set JEDI ensemble DA config dictionary - logger.info(f"Generate JEDI YAML config: {self.yaml}") + logger.info(f"Generating JEDI YAML config: {self.yaml}") jedi.set_config(self.task_config) logger.debug(f"JEDI config:\n{pformat(self.config)}") # save JEDI config to YAML file - logger.info(f"Writing JEDI YAML file to: {self.yaml}") + logger.info(f"Writing JEDI YAML config to: {self.yaml}") save_as_yaml(jedi.config, jedi.yaml) # link JEDI ensemble DA executable - logger.debug(f"Linking JEDI ensemble DA executable to: {self.exe}") + logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") self.jedi.link_exe(self.task_config) # stage observations - logger.info(f"Staging observations") + logger.info(f"Staging list of observation files generated from JEDI config") obs_dict = self.jedi.get_obs_dict(self.task_config) FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") # stage bias corrections - logger.info(f"Staging bias corrections") + logger.info(f"Staging list of bias correction files generated from JEDI config") bias_dict = self.jedi.get_bias_dict(self.task_config) FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") - crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) - FileHandler(crtm_fix_list).sync() + crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) + FileHandler(crtm_fix_dict).sync() + logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") # stage fix files logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() + jedi_fix_dict = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_dict).sync() + logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") # stage backgrounds logger.info(f"Stage ensemble member background files") bkg_staging_dict = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() + logger.debug(f"Ensemble member background files:\n{pformat(bkg_staging_dict)}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -130,15 +135,19 @@ def initialize_letkf(self) -> None: @logit(logger) def initialize_fv3inc(self): # get JEDI-to-FV3 increment converter config and save to YAML file + logger.info(f"Generating JEDI YAML config: {self.yaml}") self.jedi.get_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.config)}") # link JEDI-to-FV3 increment converter executable + logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") self.jedi.link_exe(self.task_config) @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) From e85e993d3feb58f136685d6e2723b9bf557aef63 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 21 Aug 2024 21:40:01 +0000 Subject: [PATCH 07/44] Finished debugging --- .../exglobal_atmens_analysis_initialize.py | 2 +- scripts/exglobal_atmens_analysis_letkf.py | 2 +- ush/python/pygfs/jedi/jedi.py | 5 +-- ush/python/pygfs/task/atm_analysis.py | 40 +++++++++++-------- ush/python/pygfs/task/atmens_analysis.py | 34 +++++++++------- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 590664b951..bc0d039240 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -21,4 +21,4 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize_var() + AtmEnsAnl.initialize_letkf() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 6d0112d69d..2f08e528d1 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -19,4 +19,4 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda'])) + AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda']) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 3d3ca1949d..da9cfae0cd 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -2,12 +2,11 @@ import os from logging import getLogger -from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, chdir, rm_p, - parse_j2yaml, save_as_yaml, + parse_j2yaml, logit, Task, Executable, @@ -31,7 +30,7 @@ def __init__(self, task_config: AttrDict[str, Any]) -> None: self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') @logit(logger) - def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> AttrDict + def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> AttrDict: """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8a2fad3829..e67f516c26 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -5,12 +5,14 @@ import gzip import tarfile from logging import getLogger +from pprint import pformat +from typing import Optional from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, Task, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit) from pygfs.jedi.jedi import JEDI @@ -57,7 +59,7 @@ def __init__(self, config): self.jedi = JEDI(self.task_config) @logit(logger) - def initialize(self) -> None: + def initialize_var(self) -> None: """Initialize a global atm analysis This method will initialize a global atm analysis using JEDI. @@ -73,17 +75,17 @@ def initialize(self) -> None: """ super().initialize() - # set JEDI variational configuration - logger.info(f"Generating JEDI YAML config: {jedi.yaml}") - jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(jedi.config)}") + # set JEDI config + logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML config to: {jedi.yaml}") - save_as_yaml(jedi.config, jedi.yaml) + logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) # link JEDI variational executable - logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) # stage observations @@ -130,7 +132,7 @@ def initialize(self) -> None: logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") bkg_staging_dict = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() - logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}" + logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -142,13 +144,19 @@ def initialize(self) -> None: @logit(logger) def initialize_fv3inc(self): + super().initialize() + # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.yaml}") - self.jedi.get_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") + # save JEDI config to YAML file + logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) + # link JEDI-to-FV3 increment converter executable - logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) @logit(logger) @@ -192,8 +200,8 @@ def finalize(self) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") - src = self.task_config.jedi_yaml + logger.info(f"Copying {self.jedi.yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") + src = self.jedi.yaml dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 25221e5ac3..a8d6a1c094 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -1;95;0cimport os +import os import glob import gzip import tarfile from logging import getLogger -from typing import Dict, List +from pprint import pformat +from typing import Optional from wxflow import (AttrDict, FileHandler, @@ -82,16 +83,16 @@ def initialize_letkf(self) -> None: super().initialize() # set JEDI ensemble DA config dictionary - logger.info(f"Generating JEDI YAML config: {self.yaml}") - jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.info(f"Generating JEDI config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") # save JEDI config to YAML file - logger.info(f"Writing JEDI YAML config to: {self.yaml}") - save_as_yaml(jedi.config, jedi.yaml) + logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) # link JEDI ensemble DA executable - logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) # stage observations @@ -135,19 +136,22 @@ def initialize_letkf(self) -> None: @logit(logger) def initialize_fv3inc(self): # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.yaml}") - self.jedi.get_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.config)}") + logger.info(f"Generating JEDI config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") + # save JEDI config to YAML file + logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) + # link JEDI-to-FV3 increment converter executable - logger.info(f"Linking JEDI executable {task_config.JEDIEXE} to {jedi.exe}") + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() - self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) @@ -192,8 +196,8 @@ def finalize(self) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - src = self.task_config.jedi_yaml + logger.info(f"Copying {self.jedi.yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") + src = self.jedi.yaml dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { From 43a6338ef047d0e7548be6efe7eaefadb3f9de09 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 21 Aug 2024 22:01:21 +0000 Subject: [PATCH 08/44] pynorms --- ush/python/pygfs/jedi/jedi.py | 15 ++++++++------- ush/python/pygfs/task/atm_analysis.py | 6 +++--- ush/python/pygfs/task/atmens_analysis.py | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index da9cfae0cd..34b3aaa71a 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -21,14 +21,14 @@ def __init__(self, task_config: AttrDict[str, Any]) -> None: # For provenance, save incoming task_config as a private attribute of JEDI object self._task_config = task_config - + _exe_name = os.path.basename(task_config.JEDIEXE) self.exe = os.path.join(task_config.DATA, _exe_name) self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') self.config = AttrDict() self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') - + @logit(logger) def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> AttrDict: """Compile a JEDI configuration dictionary from a template file and save to a YAML file @@ -69,7 +69,7 @@ def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = searchpath=self.j2tmpl_dir) else: raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") - + @logit(logger) def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application @@ -105,7 +105,7 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op raise OSError(f"Failed to execute {exec_cmd}") except Exception: raise WorkflowException(f"An error occured during execution of {exec_cmd}") - + @logit(logger) def link_exe(self, task_config: AttrDict[str, Any]) -> None: """Link JEDI executable to run directory @@ -125,7 +125,7 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: if os.path.exists(exe_dest): rm_p(exe_dest) os.symlink(task_config.JEDIEXE, exe_dest) - + @logit(logger) def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: """Compile a dictionary of observation files to copy @@ -137,7 +137,7 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: Parameters ---------- task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. + Attribute-dictionary of all configuration variables associated with a GDAS task. Returns ---------- @@ -157,7 +157,7 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: 'copy': copylist } return obs_dict - + @logit(logger) @staticmethod def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: @@ -200,6 +200,7 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: } return bias_dict + @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e67f516c26..301ea48058 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -145,7 +145,7 @@ def initialize_var(self) -> None: @logit(logger) def initialize_fv3inc(self): super().initialize() - + # get JEDI-to-FV3 increment converter config and save to YAML file logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) @@ -154,11 +154,11 @@ def initialize_fv3inc(self): # save JEDI config to YAML file logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") save_as_yaml(self.jedi.config, self.jedi.yaml) - + # link JEDI-to-FV3 increment converter executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) - + @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: super().execute() diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a8d6a1c094..5b4c06bdf0 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -143,7 +143,7 @@ def initialize_fv3inc(self): # save JEDI config to YAML file logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") save_as_yaml(self.jedi.config, self.jedi.yaml) - + # link JEDI-to-FV3 increment converter executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) From ef68fad268381578bfa343ed5def7a5b72fb9e08 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 22 Aug 2024 13:35:45 +0000 Subject: [PATCH 09/44] Update some comments and logging --- ush/python/pygfs/jedi/jedi.py | 1 - ush/python/pygfs/task/atm_analysis.py | 54 +++++++++++++++++++++++- ush/python/pygfs/task/atmens_analysis.py | 39 ++++++++++++++++- 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 34b3aaa71a..0fc5d2620a 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -99,7 +99,6 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op exec_cmd.add_default_arg(self.yaml) try: - logger.debug(f"Executing {exec_cmd}") exec_cmd() except OSError: raise OSError(f"Failed to execute {exec_cmd}") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 301ea48058..a2d2b7b542 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -65,6 +65,7 @@ def initialize_var(self) -> None: This method will initialize a global atm analysis using JEDI. This includes: - generating and saving JEDI YAML config + - linking the JEDI executable - staging observation files - staging bias correction files - staging CRTM fix files @@ -72,6 +73,13 @@ def initialize_var(self) -> None: - staging B error files - staging model backgrounds - creating output directories + + Parameters + ---------- + None + Returns + ---------- + None """ super().initialize() @@ -84,7 +92,7 @@ def initialize_var(self) -> None: logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") save_as_yaml(self.jedi.config, self.jedi.yaml) - # link JEDI variational executable + # link JEDI executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) @@ -144,6 +152,20 @@ def initialize_var(self) -> None: @logit(logger) def initialize_fv3inc(self): + """Initialize FV3 increment converter + + This method will initialize a global atm analysis using JEDI. + This includes: + - generating and saving JEDI YAML config + - linking the JEDI executable + + Parameters + ---------- + None + Returns + ---------- + None + """ super().initialize() # get JEDI-to-FV3 increment converter config and save to YAML file @@ -155,14 +177,35 @@ def initialize_fv3inc(self): logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") save_as_yaml(self.jedi.config, self.jedi.yaml) - # link JEDI-to-FV3 increment converter executable + # link JEDI executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + """Run JEDI executable + + This method will run the JEDI executable for the global atm analysis + or FV3 increment converter + + Parameters + ---------- + aprun_cmd : str + Run command for JEDI application on HPC system + jedi_args : List + List of additional optional arguments for JEDI application + + Returns + ---------- + None + """ super().execute() + if jedi_args: + logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") + else: + logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) @@ -174,6 +217,13 @@ def finalize(self) -> None: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - copy the updated bias correction files to ROTDIR + + Parameters + ---------- + None + Returns + ---------- + None """ super().finalize() diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 5b4c06bdf0..588fa92ecd 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -135,6 +135,23 @@ def initialize_letkf(self) -> None: @logit(logger) def initialize_fv3inc(self): + """Initialize FV3 increment converter + + This method will initialize a global atmens analysis using JEDI. + This includes: + - generating and saving JEDI YAML config + - linking the JEDI executable + + Parameters + ---------- + None + + Returns + ---------- + None + """ + super().initialize() + # get JEDI-to-FV3 increment converter config and save to YAML file logger.info(f"Generating JEDI config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) @@ -150,8 +167,28 @@ def initialize_fv3inc(self): @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + """Run JEDI executable + + This method will run the JEDI executable for either the global atm analysis + or FV3 increment converter + + Parameters + ---------- + aprun_cmd : str + Run command for JEDI application on HPC system + jedi_args : List + List of additional optional arguments for JEDI application + Returns + ---------- + None + """ super().execute() + if jedi_args: + logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") + else: + logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) @@ -165,7 +202,7 @@ def finalize(self) -> None: Parameters ---------- - Analysis: parent class for GDAS task + None Returns ---------- From 0d01ad0e413eb6f17341f2ac15decbaa541e321f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 22 Aug 2024 13:42:20 +0000 Subject: [PATCH 10/44] pynorms --- ush/python/pygfs/task/atm_analysis.py | 11 +++++++---- ush/python/pygfs/task/atmens_analysis.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index a2d2b7b542..9a6a7ec329 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -76,7 +76,8 @@ def initialize_var(self) -> None: Parameters ---------- - None + None + Returns ---------- None @@ -161,7 +162,8 @@ def initialize_fv3inc(self): Parameters ---------- - None + None + Returns ---------- None @@ -205,7 +207,7 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") else: logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) @@ -220,7 +222,8 @@ def finalize(self) -> None: Parameters ---------- - None + None + Returns ---------- None diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 588fa92ecd..1b5739158a 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -151,7 +151,7 @@ def initialize_fv3inc(self): None """ super().initialize() - + # get JEDI-to-FV3 increment converter config and save to YAML file logger.info(f"Generating JEDI config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) From d496307c11d364e997733342f460ed5579105f5d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 22 Aug 2024 13:45:03 +0000 Subject: [PATCH 11/44] pynorms --- ush/python/pygfs/task/atm_analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 9a6a7ec329..3ecd174c57 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -76,7 +76,7 @@ def initialize_var(self) -> None: Parameters ---------- - None + None Returns ---------- @@ -162,7 +162,7 @@ def initialize_fv3inc(self): Parameters ---------- - None + None Returns ---------- @@ -222,7 +222,7 @@ def finalize(self) -> None: Parameters ---------- - None + None Returns ---------- From 056d07c4fb03e736860e78333fb3501e1d7c9415 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 29 Aug 2024 14:43:34 +0000 Subject: [PATCH 12/44] Merge latest develop --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 - env/HERA.env | 10 ++++ env/HERCULES.env | 10 ++++ env/JET.env | 10 ++++ env/ORION.env | 9 ++++ env/S4.env | 10 ++++ env/WCOSS2.env | 37 +++++++++++++++ jobs/JGLOBAL_ATMENS_ANALYSIS_OBS | 35 ++++++++++++++ jobs/JGLOBAL_ATMENS_ANALYSIS_SOL | 35 ++++++++++++++ jobs/rocoto/atmensanlobs.sh | 18 +++++++ jobs/rocoto/atmensanlsol.sh | 18 +++++++ parm/archive/enkf.yaml.j2 | 29 ++++++++---- parm/config/gfs/config.atmensanlobs | 13 ++++++ parm/config/gfs/config.atmensanlsol | 13 ++++++ parm/config/gfs/config.base | 2 + parm/config/gfs/config.resources | 28 ++++++++++- parm/config/gfs/config.resources.HERA | 13 ++++++ parm/config/gfs/config.resources.HERCULES | 11 +++++ parm/config/gfs/config.resources.WCOSS2 | 11 +++++ parm/config/gfs/yaml/defaults.yaml | 6 +++ parm/stage/analysis.yaml.j2 | 7 +++ scripts/exglobal_atmens_analysis_obs.py | 23 +++++++++ scripts/exglobal_atmens_analysis_sol.py | 23 +++++++++ scripts/exglobal_stage_ic.py | 2 +- sorc/gdas.cd | 2 +- ush/python/pygfs/task/atmens_analysis.py | 24 +++++----- ush/python/pygfs/task/stage_ic.py | 4 ++ workflow/applications/applications.py | 2 +- workflow/applications/gfs_cycled.py | 5 +- workflow/rocoto/gfs_tasks.py | 57 ++++++++++++++++++++++- workflow/rocoto/tasks.py | 2 +- 31 files changed, 440 insertions(+), 31 deletions(-) create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_OBS create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_SOL create mode 100755 jobs/rocoto/atmensanlobs.sh create mode 100755 jobs/rocoto/atmensanlsol.sh create mode 100644 parm/config/gfs/config.atmensanlobs create mode 100644 parm/config/gfs/config.atmensanlsol create mode 100755 scripts/exglobal_atmens_analysis_obs.py create mode 100755 scripts/exglobal_atmens_analysis_sol.py diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index f835b34593..0b5aa7b6ac 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -18,9 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml skip_ci_on_hosts: - - hera - gaea - orion - hercules - - wcoss2 diff --git a/env/HERA.env b/env/HERA.env index 697cf21965..8d59c870cc 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -72,6 +72,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 83d934c91a..79ff8391bd 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -76,6 +76,16 @@ case ${step} in export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" ;; + "atmensanlobs") + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + ;; + "atmensanlsol") + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + ;; "atmensanlletkf") export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/JET.env b/env/JET.env index 473539ded1..4294a00de1 100755 --- a/env/JET.env +++ b/env/JET.env @@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/ORION.env b/env/ORION.env index 65a8871cdd..7838d28640 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -68,6 +68,15 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/S4.env b/env/S4.env index d0985e44ca..a29ec49fb4 100755 --- a/env/S4.env +++ b/env/S4.env @@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cf9feeca83..adff54d636 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -53,6 +53,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} @@ -89,6 +99,33 @@ elif [[ "${step}" = "esnowrecen" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${APRUN}" + +elif [[ "${step}" = "ocnanalrun" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN}" + +elif [[ "${step}" = "ocnanalchkpt" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN}" + +elif [[ "${step}" = "ocnanalecen" ]]; then + + export NTHREADS_OCNANALECEN=${NTHREADSmax} + export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}" + +elif [[ "${step}" = "marineanalletkf" ]]; then + + export NTHREADS_MARINEANALLETKF=${NTHREADSmax} + export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS b/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS new file mode 100755 index 0000000000..9d858a8a37 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlobs" -c "base atmensanl atmensanlobs" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSOBSSH:-${SCRgfs}/exglobal_atmens_analysis_obs.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL b/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL new file mode 100755 index 0000000000..415791cdd0 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlsol" -c "base atmensanl atmensanlsol" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSSOLSH:-${SCRgfs}/exglobal_atmens_analysis_sol.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/atmensanlobs.sh b/jobs/rocoto/atmensanlobs.sh new file mode 100755 index 0000000000..d02d013bcd --- /dev/null +++ b/jobs/rocoto/atmensanlobs.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlobs" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS" +status=$? +exit "${status}" diff --git a/jobs/rocoto/atmensanlsol.sh b/jobs/rocoto/atmensanlsol.sh new file mode 100755 index 0000000000..e1fe59d986 --- /dev/null +++ b/jobs/rocoto/atmensanlsol.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlsol" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL" +status=$? +exit "${status}" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 92ed0095af..d3f16e8e69 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -15,17 +15,21 @@ enkf: - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log" {% endfor %} - {% if DO_JEDIATMENS %} - {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %} - {% else %} - {% set steps = ["eobs", "eupd"] %} {% if lobsdiag_forenkf %} - {% do steps.append("ediag") %} + {% if DO_JEDIATMENS %} + {% set steps = ["atmensanlinit", "atmensanlobs", "atmensanlsol", "atmensanlfv3inc", "atmensanlfinal"] %} + {% else %} + {% set steps = ["eobs", "ediag", "eupd"] %} + {% endif %} {% else %} - {% for mem in range(1, nmem_ens + 1) %} - {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} - {% endfor %} - {% endif %} + {% if DO_JEDIATMENS %} + {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %} + {% else %} + {% set steps = ["eobs", "eupd"] %} + {% for mem in range(1, nmem_ens + 1) %} + {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} + {% endfor %} + {% endif %} {% endif %} {% for step in steps %} @@ -49,10 +53,17 @@ enkf: "oznstat.ensmean", "radstat.ensmean"] %} {% else %} + {% if lobsdiag_forenkf %} + {% set da_files = ["atmens_observer.yaml", + "atmens_solver.yaml", + "atminc.ensmean.nc", + "atmensstat"] %} + {% else %} {% set da_files = ["atmens.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} + {% endif %} {% for file in da_files %} - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" {% endfor %} diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs new file mode 100644 index 0000000000..dff3fa3095 --- /dev/null +++ b/parm/config/gfs/config.atmensanlobs @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +########## config.atmensanlobs ########## +# Pre Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlobs" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlobs + +export JCB_ALGO_YAML=@JCB_ALGO_YAML@ + +echo "END: config.atmensanlobs" diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol new file mode 100644 index 0000000000..dac161373b --- /dev/null +++ b/parm/config/gfs/config.atmensanlsol @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +########## config.atmensanlsol ########## +# Pre Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlsol" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlsol + +export JCB_ALGO_YAML=@JCB_ALGO_YAML@ + +echo "END: config.atmensanlsol" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 544113f942..81b18030fa 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -470,6 +470,7 @@ export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs foreca # The monitor jobs are not yet supported for JEDIATMVAR. if [[ ${DO_JEDIATMVAR} = "YES" ]]; then + export DO_FIT2OBS="NO" # Run fit to observations package export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring @@ -488,6 +489,7 @@ if [[ "${machine}" =~ "PW" ]]; then export DO_TRACKER="NO" export DO_GENESIS="NO" export DO_METP="NO" + export DO_WAVE="NO" fi echo "END: config.base" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 978dca6d51..2f541ff945 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -14,7 +14,7 @@ if (( $# != 1 )); then echo "stage_ic aerosol_init" echo "prep prepsnowobs prepatmiodaobs" echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" - echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" + echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" @@ -286,7 +286,7 @@ case ${step} in ntasks=1 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="3072M" + memory="4GB" ;; "atmanlvar") @@ -1004,6 +1004,30 @@ case ${step} in memory="3072M" ;; + "atmensanlobs") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlsol") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + "atmensanlletkf") export layout_x=${layout_x_atmensanl} export layout_y=${layout_y_atmensanl} diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA index 36f50508c3..e79d4c5b0a 100644 --- a/parm/config/gfs/config.resources.HERA +++ b/parm/config/gfs/config.resources.HERA @@ -11,6 +11,19 @@ case ${step} in fi ;; + "atmanlvar") + export tasks_per_node_gdas=12 + export tasks_per_node_gfs=12 + ;; + + "atmensanlobs") + export tasks_per_node=12 + ;; + + "atmensanlsol") + export tasks_per_node=12 + ;; + "eupd") case ${CASE} in "C384") diff --git a/parm/config/gfs/config.resources.HERCULES b/parm/config/gfs/config.resources.HERCULES index 7a5a74f69c..65ea508e01 100644 --- a/parm/config/gfs/config.resources.HERCULES +++ b/parm/config/gfs/config.resources.HERCULES @@ -11,6 +11,17 @@ case ${step} in export tasks_per_node=20 fi ;; + "atmanlvar") + export tasks_per_node_gdas=48 + export tasks_per_node_gfs=48 + export memory="400GB" + ;; + + "atmensanlobs") + export tasks_per_node=48 + export memory="400GB" + ;; + *) ;; esac diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2 index a0a69fa8d1..3ff019068c 100644 --- a/parm/config/gfs/config.resources.WCOSS2 +++ b/parm/config/gfs/config.resources.WCOSS2 @@ -18,6 +18,17 @@ case ${step} in fi ;; + "atmanlvar") + export tasks_per_node_gdas=48 + export tasks_per_node_gfs=48 + export memory="400GB" + ;; + + "atmensanlobs") + export tasks_per_node=48 + export memory="400GB" + ;; + "fit2obs") export tasks_per_node=3 ;; diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 24729ac43e..b423601df3 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -37,6 +37,12 @@ atmensanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 +atmensanlobs: + JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" + +atmensanlsol: + JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" + aeroanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index e014313b6d..d30389644a 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -15,5 +15,12 @@ analysis: - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} + {% if DO_JEDIATMVAR %} + {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} + {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} + - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] + {% endfor %} + {% endfor %} + {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py new file mode 100755 index 0000000000..e4b5c98952 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_atmens_analysis_obs.py +# This script creates an AtmEnsAnalysis object +# and runs the execute method +# which executes the global atm local ensemble analysis in observer mode +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.init_observer() + AtmEnsAnl.observe() diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py new file mode 100755 index 0000000000..db55959753 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_atmens_analysis_sol.py +# This script creates an AtmEnsAnalysis object +# and runs the execute method +# which executes the global atm local ensemble analysis in solver mode +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.init_solver() + AtmEnsAnl.solve() diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 5efc1bca96..d4c212a297 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -20,7 +20,7 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', 'assim_freq', 'current_cycle', 'previous_cycle', - 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', + 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR', 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 0431b26650..09594d1c03 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 0431b26650c5e5d4eb741304a05c841d3fda0ddc +Subproject commit 09594d1c032fd187f9869ac74b2b5b351112e93c diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1b5739158a..62046c36c4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -134,8 +134,8 @@ def initialize_letkf(self) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def initialize_fv3inc(self): - """Initialize FV3 increment converter + def initialize(self): + """Initialize FV3 increment converter, LETKF observer, or LETKF solver This method will initialize a global atmens analysis using JEDI. This includes: @@ -232,16 +232,18 @@ def finalize(self) -> None: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + # get list of yamls to cop to ROTDIR + yamls = glob.glob(os.path.join(self.task_config.DATA, '*atmens*yaml')) + # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.jedi.yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - src = self.jedi.yaml - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS_ENS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() + for src in yamls: + logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, os.path.basename(src)) + logger.debug(f"Copying {src} to {dest}") + yaml_copy = { + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() # create template dictionaries template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index d4d9dc3e19..37cc4138f3 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import glob import os from logging import getLogger from typing import Any, Dict, List @@ -52,6 +53,9 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists + # Add the glob.glob function for capturing filenames + stage_dict['glob'] = glob.glob + # Parse stage yaml to get list of files to copy stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 8c1f69735e..d6d7453c3c 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -172,7 +172,7 @@ def source_configs(self, run: str = "gfs", log: bool = True) -> Dict[str, Any]: files += ['config.fcst', 'config.efcs'] elif config in ['atmanlinit', 'atmanlvar', 'atmanlfv3inc']: files += ['config.atmanl', f'config.{config}'] - elif config in ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc']: + elif config in ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc']: files += ['config.atmensanl', f'config.{config}'] elif 'wave' in config: files += ['config.wave', f'config.{config}'] diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f534764245..8c3bca0fd7 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -57,7 +57,7 @@ def _get_app_configs(self): if self.do_hybvar: if self.do_jediatmens: - configs += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] + configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] @@ -165,7 +165,8 @@ def get_task_names(self): hybrid_after_eupd_tasks = [] if self.do_hybvar: if self.do_jediatmens: - hybrid_tasks += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlobs', 'atmensanlsol'] if self.lobsdiag_forenkf else ['atmensanlletkf'] else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f60ac9a549..23f334549a 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2454,6 +2454,58 @@ def atmensanlinit(self): return task + def atmensanlobs(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlinit'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanlobs') + task_name = f'{self.run}atmensanlobs' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def atmensanlsol(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanlsol') + task_name = f'{self.run}atmensanlsol' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlsol.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def atmensanlletkf(self): deps = [] @@ -2483,7 +2535,10 @@ def atmensanlletkf(self): def atmensanlfv3inc(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlletkf'} + if self.app_config.lobsdiag_forenkf: + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlsol'} + else: + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index ab89247fbb..d943cd130c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', + 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'esnowrecen', 'fcst', From 44f172cdc6a6586469817f5620cfdc6f4c354711 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 29 Aug 2024 14:44:50 +0000 Subject: [PATCH 13/44] Refactor LETKF observer to work with JEDI class --- scripts/exglobal_atmens_analysis_fv3_increment.py | 4 ++-- scripts/exglobal_atmens_analysis_obs.py | 4 ++-- scripts/exglobal_atmens_analysis_sol.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 2245288d3f..88659aed18 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -21,5 +21,5 @@ AtmEnsAnlFV3Inc = AtmEnsAnalysis(config) # Initialize and execute - AtmEnsAnlFV3Inc.initialize_fv3inc() - AtmEnsAnlFV3Inc.execute(config.APRUN_ATMENSANLFV3INC) + AtmEnsAnl.initialize() + AtmEnsAnl.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index e4b5c98952..5e04887a37 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -19,5 +19,5 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_observer() - AtmEnsAnl.observe() + AtmEnsAnl.initialize() + AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index db55959753..96e8afdbe6 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -19,5 +19,5 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_solver() - AtmEnsAnl.solve() + AtmEnsAnl.initialize() + AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 62046c36c4..384a897aa4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -152,7 +152,7 @@ def initialize(self): """ super().initialize() - # get JEDI-to-FV3 increment converter config and save to YAML file + # get JEDI config and save to YAML file logger.info(f"Generating JEDI config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") From e06f73608f864fdba8e52ab45b98ed2493fb04ff Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 29 Aug 2024 18:05:05 +0000 Subject: [PATCH 14/44] Fix small bug --- scripts/exglobal_atmens_analysis_fv3_increment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index 88659aed18..de8cea809f 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -18,7 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis object - AtmEnsAnlFV3Inc = AtmEnsAnalysis(config) + AtmEnsAnl = AtmEnsAnalysis(config) # Initialize and execute AtmEnsAnl.initialize() From 3c6d5b5b6840c9ba47502f3e2ba33856a90da6c2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 30 Aug 2024 16:53:44 +0000 Subject: [PATCH 15/44] Redo some things w/ JEDI class to address YAML archiving error --- scripts/exglobal_atm_analysis_finalize.py | 2 + .../exglobal_atm_analysis_fv3_increment.py | 8 +- scripts/exglobal_atm_analysis_initialize.py | 7 +- scripts/exglobal_atm_analysis_variational.py | 4 +- scripts/exglobal_atmens_analysis_finalize.py | 2 + .../exglobal_atmens_analysis_fv3_increment.py | 6 +- .../exglobal_atmens_analysis_initialize.py | 7 +- scripts/exglobal_atmens_analysis_letkf.py | 4 +- scripts/exglobal_atmens_analysis_obs.py | 6 +- scripts/exglobal_atmens_analysis_sol.py | 6 +- ush/python/pygfs/jedi/jedi.py | 7 +- ush/python/pygfs/task/atm_analysis.py | 97 ++++++++----------- ush/python/pygfs/task/atmens_analysis.py | 81 +++++++--------- 13 files changed, 117 insertions(+), 120 deletions(-) diff --git a/scripts/exglobal_atm_analysis_finalize.py b/scripts/exglobal_atm_analysis_finalize.py index 3f4313631c..35220928c9 100755 --- a/scripts/exglobal_atm_analysis_finalize.py +++ b/scripts/exglobal_atm_analysis_finalize.py @@ -21,4 +21,6 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) + + # Finalize JEDI variational analysis AtmAnl.finalize() diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index ffdd1a18c1..72413ddbd4 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -18,8 +18,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis object - AtmAnlFV3Inc = AtmAnalysis(config) + AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') - # Initialize and execute - AtmAnlFV3Inc.initialize_fv3inc() - AtmAnlFV3Inc.execute(config.APRUN_ATMANLFV3INC) + # Initialize and execute FV3 increment converter + AtmAnl.initialize_jedi() + AtmAnl.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 3f908372cd..9deae07bb3 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -20,5 +20,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) - AtmAnl.initialize_var() + AtmAnl = AtmAnalysis(config, 'atmanlvar') + + # Initialize JEDI variational analysis + AtmAnl.initialize_jedi() + AtmAnl.initialize_analysis() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index e99552d1a5..8359532069 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -18,5 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) + AtmAnl = AtmAnalysis(config, 'atmanlvar') + + # Execute JEDI variational analysis AtmAnl.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) diff --git a/scripts/exglobal_atmens_analysis_finalize.py b/scripts/exglobal_atmens_analysis_finalize.py index b49cb3c413..d68c260e78 100755 --- a/scripts/exglobal_atmens_analysis_finalize.py +++ b/scripts/exglobal_atmens_analysis_finalize.py @@ -21,4 +21,6 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) + + # Finalize ensemble DA analysis AtmEnsAnl.finalize() diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index de8cea809f..48eb6a6a1e 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -18,8 +18,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis object - AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlfv3inc') - # Initialize and execute - AtmEnsAnl.initialize() + # Initialize and execute JEDI FV3 increment converter + AtmEnsAnl.initialize_jedi() AtmEnsAnl.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index bc0d039240..9fcb955b5a 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -20,5 +20,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize_letkf() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + + # Initialize JEDI ensemble DA analysis + AtmEnsAnl.initialize_jedi() + AtmEnsAnl.initialize_analysis() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 2f08e528d1..45b06524fe 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -18,5 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + + # Execute the JEDI ensemble DA analysis AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 5e04887a37..4c45736cdd 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -18,6 +18,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + + # Initialize and execute JEDI ensembler DA analysis in observer mode + AtmEnsAnl.initialize_jedi() AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index 96e8afdbe6..be78e694b1 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -18,6 +18,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') + + # Initialize and execute JEDI ensemble DA analysis in solver mode + AtmEnsAnl.initialize_jedi() AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 0fc5d2620a..0199b4c235 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -17,7 +17,7 @@ class JEDI: - def __init__(self, task_config: AttrDict[str, Any]) -> None: + def __init__(self, task_config: AttrDict[str, Any], yaml_name: Optional[str] = None) -> None: # For provenance, save incoming task_config as a private attribute of JEDI object self._task_config = task_config @@ -25,7 +25,10 @@ def __init__(self, task_config: AttrDict[str, Any]) -> None: _exe_name = os.path.basename(task_config.JEDIEXE) self.exe = os.path.join(task_config.DATA, _exe_name) - self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + if yaml_name: + self.yaml = os.path.join(task_config.DATA, yaml_name + '.yaml') + else: + self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') self.config = AttrDict() self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 3ecd174c57..f3168d069b 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -24,7 +24,7 @@ class AtmAnalysis(Task): Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config): + def __init__(self, config, yaml_name=None): super().__init__(config) _res = int(self.task_config.CASE[1:]) @@ -56,23 +56,16 @@ def __init__(self, config): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = JEDI(self.task_config) + self.jedi = JEDI(self.task_config, yaml_name) @logit(logger) - def initialize_var(self) -> None: - """Initialize a global atm analysis + def initialize_jedi(self): + """Initialize JEDI application - This method will initialize a global atm analysis using JEDI. + This method will initialize a JEDI application used in the global atm analysis. This includes: - generating and saving JEDI YAML config - linking the JEDI executable - - staging observation files - - staging bias correction files - - staging CRTM fix files - - staging FV3-JEDI fix files - - staging B error files - - staging model backgrounds - - creating output directories Parameters ---------- @@ -84,7 +77,7 @@ def initialize_var(self) -> None: """ super().initialize() - # set JEDI config + # get JEDI-to-FV3 increment converter config and save to YAML file logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") @@ -96,6 +89,30 @@ def initialize_var(self) -> None: # link JEDI executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) + + @logit(logger) + def initialize_analysis(self) -> None: + """Initialize a global atm analysis + + This method will initialize a global atm analysis. + This includes: + - staging observation files + - staging bias correction files + - staging CRTM fix files + - staging FV3-JEDI fix files + - staging B error files + - staging model backgrounds + - creating output directories + + Parameters + ---------- + None + + Returns + ---------- + None + """ + super().initialize() # stage observations logger.info(f"Staging list of observation files generated from JEDI config") @@ -151,38 +168,6 @@ def initialize_var(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def initialize_fv3inc(self): - """Initialize FV3 increment converter - - This method will initialize a global atm analysis using JEDI. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - super().initialize() - - # get JEDI-to-FV3 increment converter config and save to YAML file - logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) - @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: """Run JEDI executable @@ -252,16 +237,20 @@ def finalize(self) -> None: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + # get list of yamls to cop to ROTDIR + yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) + # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.jedi.yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") - src = self.jedi.yaml - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() + for src in yamls: + logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS}") + yaml_base = os.path.splitext(os.path.basename(src))[0] + dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, dest_yaml_name) + logger.debug(f"Copying {src} to {dest}") + yaml_copy = { + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() # copy bias correction files to ROTDIR logger.info("Copy bias correction files from DATA/ to COM/") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 384a897aa4..c255b28f3d 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -28,7 +28,7 @@ class AtmEnsAnalysis(Task): Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config): + def __init__(self, config, yaml_name=None): super().__init__(config) _res = int(self.task_config.CASE_ENS[1:]) @@ -56,21 +56,16 @@ def __init__(self, config): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = JEDI(self.task_config) + self.jedi = JEDI(self.task_config, yaml_name) @logit(logger) - def initialize_letkf(self) -> None: - """Initialize a global atmens analysis + def initialize_jedi(self): + """Initialize JEDI application - This method will initialize a global atmens analysis using JEDI. + This method will initialize a JEDI application used in the global atmens analysis. This includes: - generating and saving JEDI YAML config - - staging observation files - - staging bias correction files - - staging CRTM fix files - - staging FV3-JEDI fix files - - staging model backgrounds - - creating output directories + - linking the JEDI executable Parameters ---------- @@ -80,9 +75,8 @@ def initialize_letkf(self) -> None: ---------- None """ - super().initialize() - # set JEDI ensemble DA config dictionary + # get JEDI config and save to YAML file logger.info(f"Generating JEDI config: {self.jedi.yaml}") self.jedi.set_config(self.task_config) logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") @@ -91,9 +85,32 @@ def initialize_letkf(self) -> None: logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") save_as_yaml(self.jedi.config, self.jedi.yaml) - # link JEDI ensemble DA executable + # link JEDI-to-FV3 increment converter executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) + + @logit(logger) + def initialize_analysis(self) -> None: + """Initialize a global atmens analysis + + This method will initialize a global atmens analysis. + This includes: + - staging observation files + - staging bias correction files + - staging CRTM fix files + - staging FV3-JEDI fix files + - staging model backgrounds + - creating output directories + + Parameters + ---------- + None + + Returns + ---------- + None + """ + super().initialize() # stage observations logger.info(f"Staging list of observation files generated from JEDI config") @@ -133,38 +150,6 @@ def initialize_letkf(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def initialize(self): - """Initialize FV3 increment converter, LETKF observer, or LETKF solver - - This method will initialize a global atmens analysis using JEDI. - This includes: - - generating and saving JEDI YAML config - - linking the JEDI executable - - Parameters - ---------- - None - - Returns - ---------- - None - """ - super().initialize() - - # get JEDI config and save to YAML file - logger.info(f"Generating JEDI config: {self.jedi.yaml}") - self.jedi.set_config(self.task_config) - logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") - - # save JEDI config to YAML file - logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") - save_as_yaml(self.jedi.config, self.jedi.yaml) - - # link JEDI-to-FV3 increment converter executable - logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") - self.jedi.link_exe(self.task_config) - @logit(logger) def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: """Run JEDI executable @@ -238,7 +223,9 @@ def finalize(self) -> None: # copy full YAML from executable to ROTDIR for src in yamls: logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, os.path.basename(src)) + yaml_base = os.path.splitext(os.path.basename(src))[0] + dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, dest_yaml_name) logger.debug(f"Copying {src} to {dest}") yaml_copy = { 'copy': [[src, dest]] From e0e3f403297b84d5160c91fb699f8f2244efc49f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 30 Aug 2024 17:13:44 +0000 Subject: [PATCH 16/44] pynorms --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f3168d069b..f8d19daaaa 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -89,7 +89,7 @@ def initialize_jedi(self): # link JEDI executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) - + @logit(logger) def initialize_analysis(self) -> None: """Initialize a global atm analysis @@ -250,7 +250,7 @@ def finalize(self) -> None: yaml_copy = { 'copy': [[src, dest]] } - FileHandler(yaml_copy).sync() + FileHandler(yaml_copy).sync() # copy bias correction files to ROTDIR logger.info("Copy bias correction files from DATA/ to COM/") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index c255b28f3d..386edda0dc 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -88,7 +88,7 @@ def initialize_jedi(self): # link JEDI-to-FV3 increment converter executable logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") self.jedi.link_exe(self.task_config) - + @logit(logger) def initialize_analysis(self) -> None: """Initialize a global atmens analysis From a6f4b81ed8bfe5f658878cdb84894033d41517d8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 30 Aug 2024 17:45:58 +0000 Subject: [PATCH 17/44] Proposed change to enkf.yaml.j2 --- parm/archive/enkf.yaml.j2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index d3f16e8e69..f2ef2d7442 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -54,12 +54,14 @@ enkf: "radstat.ensmean"] %} {% else %} {% if lobsdiag_forenkf %} - {% set da_files = ["atmens_observer.yaml", - "atmens_solver.yaml", + {% set da_files = ["atmensanlobs.yaml", + "atmensanlsol.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% else %} - {% set da_files = ["atmens.yaml", + {% set da_files = ["atmensanlletkf.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} From ed2613b457b611551d229b7309fb2700011f93fa Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 30 Aug 2024 17:56:49 +0000 Subject: [PATCH 18/44] Revert "Proposed change to enkf.yaml.j2" This reverts commit a6f4b81ed8bfe5f658878cdb84894033d41517d8. --- parm/archive/enkf.yaml.j2 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index f2ef2d7442..d3f16e8e69 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -54,14 +54,12 @@ enkf: "radstat.ensmean"] %} {% else %} {% if lobsdiag_forenkf %} - {% set da_files = ["atmensanlobs.yaml", - "atmensanlsol.yaml", - "atmensanlfv3inc.yaml", + {% set da_files = ["atmens_observer.yaml", + "atmens_solver.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% else %} - {% set da_files = ["atmensanlletkf.yaml", - "atmensanlfv3inc.yaml", + {% set da_files = ["atmens.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} From 86b604a0727392bc7e28796f6201698bf9a84af2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 30 Aug 2024 21:12:26 +0000 Subject: [PATCH 19/44] Fix error from my misunderstanding of how obs/sol jobs work for LETKF --- parm/archive/enkf.yaml.j2 | 8 +++++--- parm/archive/gdas.yaml.j2 | 3 ++- parm/archive/gfsa.yaml.j2 | 3 ++- scripts/exglobal_atmens_analysis_initialize.py | 5 ++++- scripts/exglobal_atmens_analysis_obs.py | 1 - 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index d3f16e8e69..f4c590b277 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -54,12 +54,14 @@ enkf: "radstat.ensmean"] %} {% else %} {% if lobsdiag_forenkf %} - {% set da_files = ["atmens_observer.yaml", - "atmens_solver.yaml", + {% set da_files = ["atmensanlobs.yaml", + "atmensanlsol.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% else %} - {% set da_files = ["atmens.yaml", + {% set da_files = ["atmensanlletkf.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index db92141ede..826f4b8b8b 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -58,7 +58,8 @@ gdas: # Analysis state {% if DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 4a86778e2e..13a03fd65b 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -32,7 +32,8 @@ gfsa: # State data {% if DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 9fcb955b5a..326fe80628 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -20,7 +20,10 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + if not config.lobsdiag_forenkf: + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + else: + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize JEDI ensemble DA analysis AtmEnsAnl.initialize_jedi() diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index 4c45736cdd..c701f8cb4e 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -21,5 +21,4 @@ AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') # Initialize and execute JEDI ensembler DA analysis in observer mode - AtmEnsAnl.initialize_jedi() AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) From 9b701952f15f8ce1d3488191195b850b65bfc5e1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 3 Sep 2024 10:40:25 +0000 Subject: [PATCH 20/44] Set JCB algorithm depending on whether running in observer-solver mode or not --- parm/archive/gdas.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 2 +- parm/config/gfs/config.atmensanl | 6 +++++- parm/config/gfs/yaml/defaults.yaml | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 826f4b8b8b..56e47e595a 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -59,7 +59,7 @@ gdas: # Analysis state {% if DO_JEDIATMVAR %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 13a03fd65b..226a7178fa 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -33,7 +33,7 @@ gfsa: # State data {% if DO_JEDIATMVAR %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index ddd3d88659..f5a1278248 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -6,7 +6,11 @@ echo "BEGIN: config.atmensanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ +if [[ ${lobsdiag_forenkf} = ".false." ]] ; then + export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@ +else + export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@ +fi export INTERP_METHOD='barycentric' diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index b423601df3..88261a9b15 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -31,7 +31,8 @@ atmanl: IO_LAYOUT_Y: 1 atmensanl: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" + JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" + JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_observer.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 From c279e45872071c382f38b43759de8fd12be82a5b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 3 Sep 2024 16:06:29 +0000 Subject: [PATCH 21/44] Fix error in JCB prototype file name --- parm/config/gfs/yaml/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 88261a9b15..05e1b24012 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -32,7 +32,7 @@ atmanl: atmensanl: JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" - JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_observer.yaml.j2" + JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 From 1e034f785a1aa74f2b6bbffc24f527d318bbbaa9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 3 Sep 2024 18:21:09 +0000 Subject: [PATCH 22/44] Address some comments --- parm/archive/enkf.yaml.j2 | 2 +- ush/python/pygfs/task/atm_analysis.py | 3 +-- ush/python/pygfs/task/atmens_analysis.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index f4c590b277..a95046d4d6 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -61,7 +61,7 @@ enkf: "atmensstat"] %} {% else %} {% set da_files = ["atmensanlletkf.yaml", - "atmensanlfv3inc.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index f8d19daaaa..a726813dd3 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -172,8 +172,7 @@ def initialize_analysis(self) -> None: def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: """Run JEDI executable - This method will run the JEDI executable for the global atm analysis - or FV3 increment converter + This method will run JEDI executables for the global atm analysis Parameters ---------- diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 386edda0dc..12ad58185e 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -154,8 +154,7 @@ def initialize_analysis(self) -> None: def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: """Run JEDI executable - This method will run the JEDI executable for either the global atm analysis - or FV3 increment converter + This method will run JEDI executables for the global atm analysis Parameters ---------- From 526b961f85bb0966df139079f446bfd27b2736ed Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 3 Sep 2024 18:32:07 +0000 Subject: [PATCH 23/44] The perils of copypasta --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 12ad58185e..8bcf866f15 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -154,7 +154,7 @@ def initialize_analysis(self) -> None: def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: """Run JEDI executable - This method will run JEDI executables for the global atm analysis + This method will run JEDI executables for the global atmens analysis Parameters ---------- From d3ecab331f467f7819b3bd96f55c76cbb1d7ae54 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 4 Sep 2024 11:03:16 +0000 Subject: [PATCH 24/44] Address comments and update GDAS hash --- sorc/gdas.cd | 2 +- ush/python/pygfs/jedi/jedi.py | 11 +++++++---- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 09594d1c03..faa95efb18 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 09594d1c032fd187f9869ac74b2b5b351112e93c +Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 0199b4c235..45e94bbc8a 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -15,7 +15,7 @@ logger = getLogger(__name__.split('.')[-1]) -class JEDI: +class Jedi: def __init__(self, task_config: AttrDict[str, Any], yaml_name: Optional[str] = None) -> None: @@ -62,7 +62,7 @@ def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = jcb_config['algorithm'] = task_config.JCB_ALGO elif 'JCB_ALGO_YAML' in task_config.keys(): jcb_algo_config = parse_j2yaml(task_config.JCB_ALGO_YAML, task_config) - jcb_config = {**jcb_config, **jcb_algo_config} + jcb_config.update(jcb_algo_config) # Step 3: Generate the JEDI YAML using JCB self.config = render(jcb_config) @@ -114,14 +114,16 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: Parameters ---------- - None + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. Returns ---------- None """ - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + # TODO: linking is not permitted per EE2. + # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] logger.warn("Linking is not permitted per EE2.") exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) if os.path.exists(exe_dest): @@ -204,6 +206,7 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: @logit(logger) +@staticmethod def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ Recursively search through a nested dictionary and return the value for the target key. diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index a726813dd3..5a9b63dd5b 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -56,7 +56,7 @@ def __init__(self, config, yaml_name=None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = JEDI(self.task_config, yaml_name) + self.jedi = Jedi(self.task_config, yaml_name) @logit(logger) def initialize_jedi(self): diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 8bcf866f15..86d2698da1 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -56,7 +56,7 @@ def __init__(self, config, yaml_name=None): self.task_config = AttrDict(**self.task_config, **local_dict) # Create JEDI object - self.jedi = JEDI(self.task_config, yaml_name) + self.jedi = Jedi(self.task_config, yaml_name) @logit(logger) def initialize_jedi(self): From 02b3734c2a38bacc6e1cae77878f707f3a877925 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 4 Sep 2024 11:11:14 +0000 Subject: [PATCH 25/44] Mistakenly had class method marked as static method --- ush/python/pygfs/jedi/jedi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 45e94bbc8a..55f8533595 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -163,7 +163,6 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: return obs_dict @logit(logger) - @staticmethod def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: """Compile a dictionary of observation files to copy From 3d3ccf0868268d31f079f19e61193d960ed2d239 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 4 Sep 2024 11:41:11 +0000 Subject: [PATCH 26/44] Update --- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5a9b63dd5b..e931bf133f 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -14,7 +14,7 @@ Task, parse_j2yaml, save_as_yaml, logit) -from pygfs.jedi.jedi import JEDI +from pygfs.jedi.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 86d2698da1..02c6e74e79 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -18,7 +18,7 @@ Executable, WorkflowException, Template, TemplateConstants) -from pygfs.jedi.jedi import JEDI +from pygfs.jedi.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) From d5e2fe04f35815e12d705b081d21f07d4d15df3f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:32:44 -0400 Subject: [PATCH 27/44] Update jedi.py --- ush/python/pygfs/jedi/jedi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 55f8533595..cd61a0ddd0 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -17,7 +17,7 @@ class Jedi: - def __init__(self, task_config: AttrDict[str, Any], yaml_name: Optional[str] = None) -> None: + def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: # For provenance, save incoming task_config as a private attribute of JEDI object self._task_config = task_config @@ -205,7 +205,6 @@ def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: @logit(logger) -@staticmethod def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: """ Recursively search through a nested dictionary and return the value for the target key. From 16868a8b335000eacd195f928b954aa3c2e783de Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:58:26 -0400 Subject: [PATCH 28/44] Update jedi.py --- ush/python/pygfs/jedi/jedi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index cd61a0ddd0..388a6866e6 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -33,7 +33,7 @@ def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> No self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') @logit(logger) - def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = None) -> AttrDict: + def set_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: """Compile a JEDI configuration dictionary from a template file and save to a YAML file Parameters @@ -74,7 +74,7 @@ def set_config(self, task_config: AttrDict[str, Any], algorithm: Optional[str] = raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") @logit(logger) - def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: """Execute JEDI application Parameters @@ -109,7 +109,7 @@ def execute(self, task_config: AttrDict[str, Any], aprun_cmd: str, jedi_args: Op raise WorkflowException(f"An error occured during execution of {exec_cmd}") @logit(logger) - def link_exe(self, task_config: AttrDict[str, Any]) -> None: + def link_exe(self, task_config: AttrDict) -> None: """Link JEDI executable to run directory Parameters @@ -131,7 +131,7 @@ def link_exe(self, task_config: AttrDict[str, Any]) -> None: os.symlink(task_config.JEDIEXE, exe_dest) @logit(logger) - def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: + def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of @@ -163,7 +163,7 @@ def get_obs_dict(self, task_config: AttrDict[str, Any]) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self, task_config: Dict[str, Any]) -> Dict[str, Any]: + def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of From a423c6fccfefa999cea6132abaadd515b982a08f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:04:50 -0400 Subject: [PATCH 29/44] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 388a6866e6..0792a0caa5 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -17,7 +17,11 @@ class Jedi: + @logit(logger, name="Jedi") def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: + """Constructor for JEDI objects + Needs a doc-block + """ # For provenance, save incoming task_config as a private attribute of JEDI object self._task_config = task_config From 35c3f493f1a65a892d6ec5a8b77895d1dd46ba73 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:05:11 -0400 Subject: [PATCH 30/44] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 0792a0caa5..3ce6abeeab 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -75,7 +75,8 @@ def set_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> self.config = parse_j2yaml(task_config.JEDIYAML, task_config, searchpath=self.j2tmpl_dir) else: - raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") + logger.exception(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + raise KeyError(f"FATAL ERROR: Task config must contain JCB_BASE_YAML or JEDIYAML") @logit(logger) def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: From 9a24b480812143857b978d275e0f3e7563bba7a8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:05:30 -0400 Subject: [PATCH 31/44] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 3ce6abeeab..c3d48be574 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -109,9 +109,9 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis try: exec_cmd() except OSError: - raise OSError(f"Failed to execute {exec_cmd}") + raise OSError(f"FATAL ERROR: Failed to execute {exec_cmd}") except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") @logit(logger) def link_exe(self, task_config: AttrDict) -> None: From f1cb728dcae16e043c31d1aabb7a689f9cd99783 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:05:59 -0400 Subject: [PATCH 32/44] Update ush/python/pygfs/jedi/jedi.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/jedi/jedi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index c3d48be574..feede565ae 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -113,8 +113,9 @@ def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[Lis except Exception: raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") + @staticmethod @logit(logger) - def link_exe(self, task_config: AttrDict) -> None: + def link_exe(task_config: AttrDict) -> None: """Link JEDI executable to run directory Parameters From 3bb5336e27c1b8cb522d885cd3ac22875b8f55e6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:11:32 -0400 Subject: [PATCH 33/44] Update ush/python/pygfs/task/atm_analysis.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e931bf133f..8c29f50f55 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -185,7 +185,6 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: ---------- None """ - super().execute() if jedi_args: logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") From ba19cdf0a0aecae2c816d3716ba04d43d86a061d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:11:48 -0400 Subject: [PATCH 34/44] Update ush/python/pygfs/task/atmens_analysis.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atmens_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 02c6e74e79..d2c0e819d8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -192,7 +192,6 @@ def finalize(self) -> None: ---------- None """ - super().finalize() # ---- tar up diags # path of output tar statfile From 287a949aa59875edd8ca9ef8701e084da433b76e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:20:19 -0400 Subject: [PATCH 35/44] Update ush/python/pygfs/task/atm_analysis.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8c29f50f55..2c2c71b5e4 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -240,7 +240,6 @@ def finalize(self) -> None: # copy full YAML from executable to ROTDIR for src in yamls: - logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS}") yaml_base = os.path.splitext(os.path.basename(src))[0] dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, dest_yaml_name) From 934d0369cc81c4913f991901a86b3df72dc7f029 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:29:10 -0400 Subject: [PATCH 36/44] Update ush/python/pygfs/task/atm_analysis.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2c2c71b5e4..b50eb79e88 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -75,7 +75,6 @@ def initialize_jedi(self): ---------- None """ - super().initialize() # get JEDI-to-FV3 increment converter config and save to YAML file logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") From 806010f8853de58705526762ca6590892e9e4140 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:31:03 -0400 Subject: [PATCH 37/44] Update ush/python/pygfs/task/atm_analysis.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/task/atm_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index b50eb79e88..51689c0844 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -234,7 +234,7 @@ def finalize(self) -> None: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - # get list of yamls to cop to ROTDIR + # get list of yamls to copy to ROTDIR yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) # copy full YAML from executable to ROTDIR From 79c4d6007be22be43e3fda2c93e5f333537602cb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Sep 2024 14:45:22 +0000 Subject: [PATCH 38/44] Address final comments --- ush/python/pygfs/jedi/__init__.py | 1 + ush/python/pygfs/task/atm_analysis.py | 22 ++++++++++++++++++++-- ush/python/pygfs/task/atmens_analysis.py | 21 +++++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/__init__.py b/ush/python/pygfs/jedi/__init__.py index e69de29bb2..5d7e85057c 100644 --- a/ush/python/pygfs/jedi/__init__.py +++ b/ush/python/pygfs/jedi/__init__.py @@ -0,0 +1 @@ +from .jedi import Jedi diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2c2c71b5e4..5e00c81c8e 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -14,7 +14,7 @@ Task, parse_j2yaml, save_as_yaml, logit) -from pygfs.jedi.jedi import Jedi +from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) @@ -24,7 +24,25 @@ class AtmAnalysis(Task): Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config, yaml_name=None): + def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): + """Initialize global atm analysis task + + This method will initialize a global atm analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute object + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + yaml_name: str, optional + name of YAML file for JEDI configuration + + Returns + ---------- + None + """ super().__init__(config) _res = int(self.task_config.CASE[1:]) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index d2c0e819d8..1ccaef805e 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -18,7 +18,7 @@ Executable, WorkflowException, Template, TemplateConstants) -from pygfs.jedi.jedi import Jedi +from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) @@ -28,7 +28,24 @@ class AtmEnsAnalysis(Task): Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config, yaml_name=None): + def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): + """Initialize global atmens analysis task + This method will initialize a global atmens analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute object + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + yaml_name: str, optional + name of YAML file for JEDI configuration + + Returns + ---------- + None + """ super().__init__(config) _res = int(self.task_config.CASE_ENS[1:]) From 00874e98780ccabf587f1ca20ab0998683feb643 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Sep 2024 14:55:52 +0000 Subject: [PATCH 39/44] Forgot some comments --- ush/python/pygfs/jedi/jedi.py | 23 ++++++++++++++++++++--- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index feede565ae..0120c50561 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -19,9 +19,26 @@ class Jedi: @logit(logger, name="Jedi") def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: - """Constructor for JEDI objects - Needs a doc-block - """ + """Constructor for JEDI objects + + This method will construct a Jedi object. + This includes: + - save a copy of task_config for provenance + - set the default JEDI YAML and executable names + - set an empty AttrDict for the JEDI config + - set the default directory for J2-YAML templates + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + yaml_name: str, optional + Name of YAML file for JEDI configuration + + Returns + ---------- + None + """ # For provenance, save incoming task_config as a private attribute of JEDI object self._task_config = task_config diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8d45f1eb7b..ff8d4bc4df 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -25,9 +25,9 @@ class AtmAnalysis(Task): """ @logit(logger, name="AtmAnalysis") def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): - """Initialize global atm analysis task + """Constructor global atm analysis task - This method will initialize a global atm analysis task. + This method will construct a global atm analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - instantiate the Jedi attribute object diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1ccaef805e..26af17dc12 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -29,8 +29,8 @@ class AtmEnsAnalysis(Task): """ @logit(logger, name="AtmEnsAnalysis") def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): - """Initialize global atmens analysis task - This method will initialize a global atmens analysis task. + """Constructor global atmens analysis task + This method will construct a global atmens analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task - instantiate the Jedi attribute object From c039881e417f987196ba1d4a03299f6e643de42f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Sep 2024 15:00:47 +0000 Subject: [PATCH 40/44] Final comment block --- ush/python/pygfs/jedi/jedi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 0120c50561..62dcb517ca 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -16,7 +16,9 @@ class Jedi: - + """ + Class for initializing and executing JEDI applications + """ @logit(logger, name="Jedi") def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: """Constructor for JEDI objects From fc437a808fe35ef9df4f2715a81d06259e20083b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Sep 2024 15:10:38 +0000 Subject: [PATCH 41/44] pynorms --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ff8d4bc4df..8871157423 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -24,7 +24,7 @@ class AtmAnalysis(Task): Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): + def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): """Constructor global atm analysis task This method will construct a global atm analysis task. @@ -42,7 +42,7 @@ def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): Returns ---------- None - """ + """ super().__init__(config) _res = int(self.task_config.CASE[1:]) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 26af17dc12..2065eca54f 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -28,8 +28,9 @@ class AtmEnsAnalysis(Task): Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): - """Constructor global atmens analysis task + def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + """Constructor global atmens analysis task + This method will construct a global atmens analysis task. This includes: - extending the task_config attribute AttrDict to include parameters required for this task @@ -40,12 +41,12 @@ def __init__(self, config: Dict[str,Any], yaml_name: Optional[str] = None): config: Dict dictionary object containing task configuration yaml_name: str, optional - name of YAML file for JEDI configuration + name of YAML file for JEDI configuration Returns ---------- None - """ + """ super().__init__(config) _res = int(self.task_config.CASE_ENS[1:]) From 95974c333af3c937eec0421b9d2d9c69af076bab Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Sep 2024 15:53:47 +0000 Subject: [PATCH 42/44] Small bug --- ush/python/pygfs/task/atm_analysis.py | 2 +- ush/python/pygfs/task/atmens_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8871157423..996bd95e08 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -6,7 +6,7 @@ import tarfile from logging import getLogger from pprint import pformat -from typing import Optional +from typing import Optional, Dict, Any from wxflow import (AttrDict, FileHandler, diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 2065eca54f..b4658554cd 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -6,7 +6,7 @@ import tarfile from logging import getLogger from pprint import pformat -from typing import Optional +from typing import Optional, Dict, Any from wxflow import (AttrDict, FileHandler, From bc52fcbc9fa2831095ca5b9469b0b42690fbcac7 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 5 Sep 2024 14:51:42 -0400 Subject: [PATCH 43/44] Update ush/python/pygfs/task/atmens_analysis.py --- ush/python/pygfs/task/atmens_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b4658554cd..55e72702b1 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -184,7 +184,6 @@ def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: ---------- None """ - super().execute() if jedi_args: logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") From f445e28d8e4d57222416b191e80651af5e1850da Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 5 Sep 2024 14:52:16 -0400 Subject: [PATCH 44/44] Update ush/python/pygfs/task/atm_analysis.py --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 996bd95e08..8d340a5b73 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -228,7 +228,6 @@ def finalize(self) -> None: ---------- None """ - super().finalize() # ---- tar up diags # path of output tar statfile