Ctest jjobs framework

ci/ctests/C48_ATM.yaml

@@ -0,0 +1,9 @@
+fcst_gfs:
+  staged_datapath: {{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ testcase }}/fcst_gfs
+  input_files: use_jinja_template_fcst
+  output_files: use_jinja_template_fcst
+
+gfs_atmos_pro:
+  staged_datapath: {{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ testcase }}/gfs_atmos_prod
+  input_files: use_jinja_template_gfs_atmos_pro
+  output_files: use_jinja_template_gfs_atmos_pro

ci/ctests/C48_S2SW.yaml

@@ -0,0 +1,9 @@
+fcst_gfs:
+  staged_datapath: {{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ testcase }}/fcst_gfs
+  input_files: use_jinja_template_fcst
+  output_files: use_jinja_template_fcst
+
+gfs_atmos_pro:
+  staged_datapath: {{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ testcase }}/gfs_atmos_prod
+  input_files: use_jinja_template_gfs_atmos_pro
+  output_files: use_jinja_template_gfs_atmos_pro

ci/ctests/CMakeLists.txt

@@ -0,0 +1,71 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(FunctionalTests)
+
+enable_testing()
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
+# Set environment variables and paths
+set(ENV{RUNTESTS} "${CMAKE_BINARY_DIR}/RUNTESTS")
+get_filename_component(HOMEgfs ${CMAKE_SOURCE_DIR}/../.. ABSOLUTE)
+set(CASE_PATH ${HOMEgfs}/ci/cases/pr)
+file(REMOVE_RECURSE ${CMAKE_BINARY_DIR}/RUNTESTS)
+
+set(CREATE_EXPERIMENT_CMD "${HOMEgfs}/workflow/create_experiment.py --overwrite")
+
+# Get the current git commit hash
+message(STATUS "Executing: git rev-parse --short HEAD")
+execute_process(
+    COMMAND git rev-parse --short HEAD
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    OUTPUT_VARIABLE GIT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+# Function to generate experiments for a given case
+function(GenerateExperiments CASE)
+    # Read the job list file for this case
+    set(JOB_LIST_FILE "${CMAKE_BINARY_DIR}/${CASE}_jobs.txt")
+
+    if(EXISTS ${JOB_LIST_FILE})
+        file(STRINGS ${JOB_LIST_FILE} JOB_NAMES)
+        message(STATUS "Processing jobs from ${JOB_LIST_FILE}:\n${JOB_NAMES}")
+
+        foreach(JOB IN LISTS JOB_NAMES)
+            string(REGEX REPLACE "^${CASE}_" "" JOB_MODIFIED ${JOB})
+            message(STATUS "Executing: ${CREATE_EXPERIMENT_CMD} --yaml ${CASE_PATH}/${CASE}.yaml") 
+            execute_process(
+                COMMAND bash -c "export pslot=${CASE}_${JOB_MODIFIED}_${GIT_HASH}; ${CREATE_EXPERIMENT_CMD} --yaml ${CASE_PATH}/${CASE}.yaml"
+            )
+        endforeach()
+    else()
+        message(WARNING "Job list file ${JOB_LIST_FILE} does not exist.")
+    endif()
+endfunction()
+
+# Function to add Rocoto boot test for a given case and job
+function(AddRocotoBootTest CASE_NAME JOB_NAME)
+    add_test(NAME jjob_test_${CASE_NAME}_${JOB_NAME}_StageData
+             COMMAND bash -c "${HOMEgfs}/ci/ctests/scripts/stage_data.sh ${CASE_NAME} ${JOB_NAME} ${GIT_HASH}")
+    add_test(NAME jjob_test_${CASE_NAME}_${JOB_NAME}
+             COMMAND bash -c "rocotoboot -w RUNTESTS/EXPDIR/${CASE_NAME}_${GIT_HASH}/workflow.xml -d RUNTESTS/EXPDIR/${CASE_NAME}_${GIT_HASH}/workflow.db -t ${JOB_NAME}")
+    set_tests_properties(jjob_test_${CASE_NAME}_${JOB_NAME} PROPERTIES DEPENDS jjob_test_${CASE_NAME}_${JOB_NAME}_StageData)         
+endfunction()
+
+# Add a test to build all components
+add_test( NAME BuildAll
+          COMMAND bash -c "cd ${HOMEgfs}/sorc; ./build_all.sh"
+)
+
+# Split CASE_LIST into individual cases
+string(REPLACE " " ";" CASE_LIST_SPLIT ${CASE_LIST})
+
+# Loop over each case in CASE_LIST_SPLIT
+foreach(CASE IN LISTS CASE_LIST_SPLIT)
+    GenerateExperiments(${CASE})
+    file(STRINGS ${CMAKE_BINARY_DIR}/${CASE}_jobs.txt JOB_NAMES)
+    foreach(JOB IN LISTS JOB_NAMES)
+        AddRocotoBootTest(${CASE} ${JOB})
+    endforeach()
+endforeach()

ci/ctests/create_functional_ctests.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import os, sys
+import shutil
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..'))
+sys.path.insert(0, _top)
+
+from argparse import ArgumentParser
+from pathlib import Path
+from wxflow import Configuration, AttrDict, parse_j2yaml, Logger, logit, which, CommandNotFoundError, ProcessError
+from workflow.hosts import Host
+from wxflow.fsutils import mkdir_p, chdir, cp
+
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
+
+def parse_args():
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    argparse.Namespace
+        The parsed command line arguments.
+    """
+    description = """Arguments for creating and updating error log files
+    """
+    parser = ArgumentParser(description=description)
+
+    parser.add_argument('--yaml', help='paths to YAML files for creating functional tests per case', required=True, type=Path, nargs='+')
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    # Locate cmake executable
+    try:
+        cmake = which("cmake")
+    except CommandNotFoundError:
+        logger.exception("cmake not found in PATH")
+        raise CommandNotFoundError("cmake not found in PATH")
+
+    # Parse command line arguments
+    args = parse_args()
+    data = AttrDict(HOMEgfs=_top)
+    data.update(os.environ)
+
+    # Initialize host and configuration
+    host = Host()
+    cfg = Configuration(f'{data.HOMEgfs}/ci/platforms')
+
+    # Create directory for tests
+    test_dir = os.path.join(_here, 'TESTS')
+    mkdir_p(test_dir)
+
+    # Process each YAML file
+    case_names = ""
+    for yaml in args.yaml:
+        platform_config = cfg.parse_config(f'config.{host.machine.lower()}')
+        case_name = os.path.basename(yaml).split('.')[0]
+        platform_config['testcase'] = case_name
+        data.update(platform_config)
+        case_cfg = parse_j2yaml(path=yaml, data=data)
+        case_cfg.update(platform_config)
+
+        # Create job list file for the case
+        top_level_entries = [key for key in case_cfg.keys() if isinstance(case_cfg[key], dict)]
+        job_list_file = f"TESTS/{case_name}_jobs.txt"
+        with open(job_list_file, 'w') as f:
+            for entry in top_level_entries:
+                f.write(f"{case_name}_{entry}\n")
+
+        case_names += f"{case_name} "
+
+        platform_config.clear()
+        data.clear()
+        case_cfg.clear()
+
+    # Run cmake to create the functional tests
+    cmake.add_default_arg([f"-S {_here}", f"-B {test_dir}", f"-DCASE_LIST='{case_names[:-1]}'"])
+    cmake()

ci/platforms/config.hera

@@ -2,6 +2,8 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+
+export STAGED_FUNCTIONAL_TEST_DIR=${GFS_CI_ROOT}/STAGED_FUNCTIONAL_TEST_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.orion

@@ -3,6 +3,7 @@
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export HPC_ACCOUNT=nems
+export STAGED_FUNCTIONAL_TEST_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_FUNCTIONAL_TEST_DIR
 export max_concurrent_cases=5
 export max_concurrent_pr=4
 

workflow/generate_workflows.sh

@@ -463,7 +463,7 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases"
 for _case in "${_yaml_list[@]}"; do
    [[ "${_verbose}" == false ]] && echo "${_case}"
    _pslot="${_case}${_tag}"
-   _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite"
+   _create_exp_cmd="${HOMEgfs}/workflow/create_experiment.py -y ${HOMEgfs}/ci/cases/pr/${_case}.yaml --overwrite"
    if [[ "${_verbose}" == true ]]; then
       pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd}
    else