diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py index 2689dbd65..13af36bdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,9 @@ """The conftest.py, providing magical fixtures to tests.""" -import datetime import inspect import json import logging import os import shutil -from functools import wraps from pathlib import Path import fmu.dataio as dio @@ -18,9 +16,10 @@ from fmu.dataio.dataio import ExportData, read_metadata from fmu.dataio.datastructure.configuration import global_configuration +from .utils import _metadata_examples + logger = logging.getLogger(__name__) -ROOTPWD = Path(".").absolute() RUN1 = "tests/data/drogon/ertrun1/realization-0/iter-0" RUN2 = "tests/data/drogon/ertrun1" RUN_PRED = "tests/data/drogon/ertrun1/realization-0/pred" @@ -55,18 +54,9 @@ def set_environ_inside_rms(monkeypatch): monkeypatch.setattr("fmu.dataio._utils.detect_inside_rms", lambda: True) -def inside_rms(func): - @pytest.mark.usefixtures("set_export_data_inside_rms", "set_environ_inside_rms") - @wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - return wrapper - - -@pytest.fixture(name="testroot", scope="session") -def fixture_testroot(): - return ROOTPWD +@pytest.fixture(scope="session") +def rootpath(request): + return request.config.rootpath def _fmu_run1_env_variables(monkeypatch, usepath="", case_only=False): @@ -83,11 +73,11 @@ def _fmu_run1_env_variables(monkeypatch, usepath="", case_only=False): @pytest.fixture(name="fmurun", scope="function") -def fixture_fmurun(tmp_path_factory, monkeypatch): +def fixture_fmurun(tmp_path_factory, monkeypatch, rootpath): """A tmp folder structure for testing; here a new fmurun without case metadata.""" tmppath = tmp_path_factory.mktemp("data") newpath = tmppath / RUN1 - shutil.copytree(ROOTPWD / RUN1, newpath) + shutil.copytree(rootpath / RUN1, newpath) _fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=False) @@ -96,11 +86,11 @@ def fixture_fmurun(tmp_path_factory, monkeypatch): @pytest.fixture(name="fmurun_prehook", scope="function") -def fixture_fmurun_prehook(tmp_path_factory, monkeypatch): +def fixture_fmurun_prehook(tmp_path_factory, monkeypatch, rootpath): """A tmp folder structure for testing; here a new fmurun without case metadata.""" tmppath = tmp_path_factory.mktemp("data") newpath = tmppath / RUN2 - shutil.copytree(ROOTPWD / RUN2, newpath) + shutil.copytree(rootpath / RUN2, newpath) _fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=True) @@ -109,11 +99,11 @@ def fixture_fmurun_prehook(tmp_path_factory, monkeypatch): @pytest.fixture(name="fmurun_w_casemetadata", scope="function") -def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch): +def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch, rootpath): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") newpath = tmppath / RUN2 - shutil.copytree(ROOTPWD / RUN2, newpath) + shutil.copytree(rootpath / RUN2, newpath) rootpath = newpath / "realization-0/iter-0" _fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False) @@ -123,11 +113,11 @@ def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch): @pytest.fixture(name="fmurun_w_casemetadata_pred", scope="function") -def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch): +def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch, rootpath): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") newpath = tmppath / RUN2 - shutil.copytree(ROOTPWD / RUN2, newpath) + shutil.copytree(rootpath / RUN2, newpath) rootpath = newpath / "realization-0/pred" _fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False) @@ -137,17 +127,17 @@ def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch): @pytest.fixture(name="fmurun_pred", scope="session") -def fixture_fmurun_pred(tmp_path_factory): +def fixture_fmurun_pred(tmp_path_factory, rootpath): """Create a tmp folder structure for testing; here a new fmurun for prediction.""" tmppath = tmp_path_factory.mktemp("data_pred") newpath = tmppath / RUN_PRED - shutil.copytree(ROOTPWD / RUN_PRED, newpath) + shutil.copytree(rootpath / RUN_PRED, newpath) logger.debug("Ran %s", _current_function_name()) return newpath @pytest.fixture(name="rmsrun_fmu_w_casemetadata", scope="session") -def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): +def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory, rootpath): """Create a tmp folder structure for testing; here existing fmurun w/ case meta! Then we locate the folder to the ...rms/model folder, pretending running RMS @@ -155,7 +145,7 @@ def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): """ tmppath = tmp_path_factory.mktemp("data3") newpath = tmppath / RUN2 - shutil.copytree(ROOTPWD / RUN2, newpath) + shutil.copytree(rootpath / RUN2, newpath) rmspath = newpath / "realization-0/iter-0/rms/model" rmspath.mkdir(parents=True, exist_ok=True) logger.debug("Active folder is %s", rmspath) @@ -163,18 +153,14 @@ def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): return rmspath -@pytest.fixture(name="rmssetup", scope="module") -def fixture_rmssetup(tmp_path_factory): +@pytest.fixture(scope="module") +def rmssetup(tmp_path_factory, global_config2_path): """Create the folder structure to mimic RMS project.""" tmppath = tmp_path_factory.mktemp("revision") rmspath = tmppath / "rms/model" rmspath.mkdir(parents=True, exist_ok=True) - - # copy a global config here - shutil.copy( - ROOTPWD / "tests/data/drogon/global_config2/global_variables.yml", rmspath - ) + shutil.copy(global_config2_path, rmspath) logger.debug("Ran %s", _current_function_name()) @@ -196,7 +182,7 @@ def fixture_rmsglobalconfig(rmssetup): @pytest.fixture(name="globalvars_norwegian_letters", scope="module") -def fixture_globalvars_norwegian_letters(tmp_path_factory): +def fixture_globalvars_norwegian_letters(tmp_path_factory, rootpath): """Read a global config with norwegian special letters w/ fmu.config utilities.""" tmppath = tmp_path_factory.mktemp("revisionxx") @@ -207,7 +193,7 @@ def fixture_globalvars_norwegian_letters(tmp_path_factory): # copy a global config with nowr letters here shutil.copy( - ROOTPWD / "tests/data/drogon/global_config2" / gname, + rootpath / "tests/data/drogon/global_config2" / gname, rmspath, ) @@ -282,10 +268,10 @@ def fixture_globalconfig1(): ).model_dump() -@pytest.fixture(name="globalconfig_asfile", scope="module") -def fixture_globalconfig_asfile() -> str: - """Global config as file for use with environment variable""" - return ROOTPWD / "tests/data/drogon/global_config2/global_variables.yml" +@pytest.fixture(scope="module") +def global_config2_path(rootpath) -> Path: + """The path to the second global config.""" + return rootpath / "tests/data/drogon/global_config2/global_variables.yml" @pytest.fixture(name="edataobj1", scope="module") @@ -310,18 +296,11 @@ def fixture_edataobj1(globalconfig1): return eobj -@pytest.fixture(name="globalconfig2", scope="module") -def fixture_globalconfig2() -> dict: +@pytest.fixture(scope="module") +def globalconfig2(global_config2_path) -> dict: """More advanced global config from file state variable in ExportData class.""" - globvar = {} - with open( - ROOTPWD / "tests/data/drogon/global_config2/global_variables.yml", - encoding="utf-8", - ) as stream: - globvar = yaml.safe_load(stream) - - logger.debug("Ran %s", _current_function_name()) - return globvar + with open(global_config2_path, encoding="utf-8") as stream: + return yaml.safe_load(stream) @pytest.fixture(name="edataobj2", scope="function") @@ -359,13 +338,16 @@ def fixture_edataobj2(globalconfig2): # ====================================================================================== -@pytest.fixture(name="schema_080", scope="session") -def fixture_schema_080(): +@pytest.fixture(scope="session") +def schema_080(rootpath): """Return 0.8.0 version of schema as json.""" - - return _parse_json(ROOTPWD / "schema/definitions/0.8.0/schema/fmu_results.json") + with open( + rootpath / "schema/definitions/0.8.0/schema/fmu_results.json", encoding="utf-8" + ) as f: + return json.load(f) +@pytest.fixture(scope="session") def metadata_examples(): """Parse all metadata examples. @@ -373,23 +355,7 @@ def metadata_examples(): Dict: Dictionary with filename as key, file contents as value. """ - - # hard code 0.8.0 for now - return { - path.name: _isoformat_all_datetimes(_parse_yaml(str(path))) - for path in ROOTPWD.glob("schema/definitions/0.8.0/examples/*.yml") - } - - -@pytest.fixture(name="metadata_examples", scope="session") -def fixture_metadata_examples(): - """Parse all metadata examples. - - Returns: - Dict: Dictionary with filename as key, file contents as value. - - """ - return metadata_examples() + return _metadata_examples() # ====================================================================================== @@ -577,7 +543,7 @@ def fixture_summary(): @pytest.fixture(name="drogon_summary") -def fixture_drogon_sum(): +def fixture_drogon_sum(rootpath): """Return pyarrow table Returns: @@ -585,8 +551,7 @@ def fixture_drogon_sum(): """ from pyarrow import feather - path = ROOTPWD / "tests/data/drogon/tabular/summary.arrow" - return feather.read_table(path) + return feather.read_table(rootpath / "tests/data/drogon/tabular/summary.arrow") @pytest.fixture(name="mock_volumes") @@ -607,7 +572,7 @@ def fixture_mock_volumes(): @pytest.fixture(name="drogon_volumes") -def fixture_drogon_volumes(): +def fixture_drogon_volumes(rootpath): """Return pyarrow table Returns: @@ -617,40 +582,6 @@ def fixture_drogon_volumes(): return Table.from_pandas( pd.read_csv( - ROOTPWD / "tests/data/drogon/tabular/geogrid--vol.csv", + rootpath / "tests/data/drogon/tabular/geogrid--vol.csv", ) ) - - -# ====================================================================================== -# Utilities -# ====================================================================================== - - -def _parse_json(schema_path): - """Parse the schema, return JSON""" - with open(schema_path, encoding="utf-8") as stream: - return json.load(stream) - - -def _parse_yaml(yaml_path): - """Parse the filename as json, return data""" - with open(yaml_path, encoding="utf-8") as stream: - data = yaml.safe_load(stream) - - return _isoformat_all_datetimes(data) - - -def _isoformat_all_datetimes(indate): - """Recursive function to isoformat all datetimes in a dictionary""" - - if isinstance(indate, list): - return [_isoformat_all_datetimes(i) for i in indate] - - if isinstance(indate, dict): - return {key: _isoformat_all_datetimes(indate[key]) for key in indate} - - if isinstance(indate, (datetime.datetime, datetime.date)): - return indate.isoformat() - - return indate diff --git a/tests/test_integration/__init__.py b/tests/test_integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_integration/conftest.py b/tests/test_integration/conftest.py new file mode 100644 index 000000000..3a1183cbb --- /dev/null +++ b/tests/test_integration/conftest.py @@ -0,0 +1,62 @@ +import os +import pathlib +import shutil +from textwrap import dedent + +import pytest + + +@pytest.fixture +def base_ert_config() -> str: + return dedent( + r""" + DEFINE user + DEFINE $DATAIO_TMP_PATH/scratch + DEFINE snakeoil + + DEFINE dev + DEFINE // + + NUM_REALIZATIONS 5 + + QUEUE_SYSTEM LOCAL + QUEUE_OPTION LOCAL MAX_RUNNING 5 + + RANDOM_SEED 123456 + + RUNPATH ///realization-/iter-/ + """ + ) + + +@pytest.fixture +def fmu_snakeoil_project(tmp_path, monkeypatch, base_ert_config, global_config2_path): + """Makes a skeleton FMU project structure into a tmp_path, copying global_config2 + into it with a basic ert config that can be appended onto.""" + monkeypatch.setenv("DATAIO_TMP_PATH", str(tmp_path)) + + os.makedirs(tmp_path / "eclipse/model") + for app in ("ert", "rms"): + os.makedirs(tmp_path / f"{app}/bin") + os.makedirs(tmp_path / f"{app}/input") + os.makedirs(tmp_path / f"{app}/model") + os.makedirs(tmp_path / "rms/model/snakeoil.rms13.1.2") + + os.makedirs(tmp_path / "fmuconfig/output") + shutil.copy(global_config2_path, tmp_path / "fmuconfig/output/") + + os.makedirs(tmp_path / "ert/bin/workflows") + pathlib.Path(tmp_path / "ert/bin/workflows/xhook_create_case_metadata").write_text( + "WF_CREATE_CASE_METADATA " + "// " # ert case root + " " # ert config path + " " # ert case dir + "", # ert username + encoding="utf-8", + ) + + pathlib.Path(tmp_path / "ert/model/snakeoil.ert").write_text( + base_ert_config, encoding="utf-8" + ) + + return tmp_path diff --git a/tests/test_integration/test_wf_create_case_metadata.py b/tests/test_integration/test_wf_create_case_metadata.py new file mode 100644 index 000000000..6514c2dbd --- /dev/null +++ b/tests/test_integration/test_wf_create_case_metadata.py @@ -0,0 +1,29 @@ +import subprocess + +import yaml + + +def test_create_case_metadata_runs_successfully(fmu_snakeoil_project, monkeypatch): + monkeypatch.chdir(fmu_snakeoil_project / "ert/model") + with open("snakeoil.ert", "a", encoding="utf-8") as f: + f.writelines( + [ + "LOAD_WORKFLOW ../bin/workflows/xhook_create_case_metadata\n" + "HOOK_WORKFLOW xhook_create_case_metadata PRE_SIMULATION\n" + ] + ) + run_result = subprocess.run( + ["ert", "test_run", "snakeoil.ert", "--disable-monitoring"], + ) + assert run_result.returncode == 0 + + fmu_case_yml = ( + fmu_snakeoil_project / "scratch/user/snakeoil/share/metadata/fmu_case.yml" + ) + assert fmu_case_yml.exists() + + with open(fmu_case_yml, encoding="utf-8") as f: + fmu_case = yaml.safe_load(f) + assert fmu_case["fmu"]["case"]["name"] == "snakeoil" + assert fmu_case["fmu"]["case"]["user"]["id"] == "user" + assert fmu_case["source"] == "fmu" diff --git a/tests/test_schema/__init__.py b/tests/test_schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_schema/test_pydantic_logic.py b/tests/test_schema/test_pydantic_logic.py index 7c77b3333..49d16bf5d 100644 --- a/tests/test_schema/test_pydantic_logic.py +++ b/tests/test_schema/test_pydantic_logic.py @@ -2,19 +2,20 @@ import logging from copy import deepcopy -import conftest import pytest from fmu.dataio.datastructure.export.content import AllowedContent from fmu.dataio.datastructure.meta import Root from fmu.dataio.datastructure.meta.enums import ContentEnum from pydantic import ValidationError +from ..utils import _metadata_examples + # pylint: disable=no-member logger = logging.getLogger(__name__) -@pytest.mark.parametrize("file, example", conftest.metadata_examples().items()) +@pytest.mark.parametrize("file, example", _metadata_examples().items()) def test_schema_example_filenames(file, example): """Assert that all examples are .yml, not .yaml""" assert file.endswith(".yml") @@ -25,7 +26,7 @@ def test_schema_example_filenames(file, example): # ====================================================================================== -@pytest.mark.parametrize("file, example", conftest.metadata_examples().items()) +@pytest.mark.parametrize("file, example", _metadata_examples().items()) def test_validate(file, example): """Confirm that examples are valid against the schema""" Root.model_validate(example) diff --git a/tests/test_units/__init__.py b/tests/test_units/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_units/test_dataio.py b/tests/test_units/test_dataio.py index a9ea56212..6c210ca32 100644 --- a/tests/test_units/test_dataio.py +++ b/tests/test_units/test_dataio.py @@ -324,9 +324,9 @@ def test_set_display_name(regsurf, globalconfig2): assert mymeta["display"]["name"] == "MyOtherDisplayName" -def test_global_config_from_env(globalconfig_asfile, monkeypatch): +def test_global_config_from_env(monkeypatch, global_config2_path): """Testing getting global config from a file""" - monkeypatch.setenv("FMU_GLOBAL_CONFIG", str(globalconfig_asfile)) + monkeypatch.setenv("FMU_GLOBAL_CONFIG", str(global_config2_path)) edata = ExportData(content="depth") # the env variable will override this assert "smda" in edata.config["masterdata"] diff --git a/tests/test_units/test_prerealization_surfaces.py b/tests/test_units/test_prerealization_surfaces.py index 8c429ff90..312970311 100644 --- a/tests/test_units/test_prerealization_surfaces.py +++ b/tests/test_units/test_prerealization_surfaces.py @@ -15,9 +15,10 @@ import fmu.dataio.dataio as dataio import pytest -from conftest import inside_rms from fmu.dataio import _utils as utils +from ..utils import inside_rms + logger = logging.getLogger(__name__) diff --git a/tests/test_units/test_rms_context.py b/tests/test_units/test_rms_context.py index 637e8c8ef..5ecd0eac6 100644 --- a/tests/test_units/test_rms_context.py +++ b/tests/test_units/test_rms_context.py @@ -9,10 +9,11 @@ import fmu.dataio.dataio as dataio import pandas as pd import pytest -from conftest import inside_rms from fmu.dataio._utils import prettyprint_dict from fmu.dataio.dataio import ValidationError +from ..utils import inside_rms + logger = logging.getLogger(__name__) logger.info("Inside RMS status %s", dataio.ExportData._inside_rms) diff --git a/tests/test_units/test_utils.py b/tests/test_units/test_utils.py index 6503db531..25caff155 100644 --- a/tests/test_units/test_utils.py +++ b/tests/test_units/test_utils.py @@ -6,10 +6,11 @@ import numpy as np import pytest -from conftest import inside_rms from fmu.dataio import _utils as utils from xtgeo import Grid, Polygons, RegularSurface +from ..utils import inside_rms + @pytest.mark.parametrize( "value, result", diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..42a490670 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,45 @@ +import datetime +from functools import wraps +from pathlib import Path + +import pytest +import yaml + + +def inside_rms(func): + @pytest.mark.usefixtures("set_export_data_inside_rms", "set_environ_inside_rms") + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def _parse_yaml(yaml_path): + """Parse the filename as json, return data""" + with open(yaml_path, encoding="utf-8") as stream: + data = yaml.safe_load(stream) + + return _isoformat_all_datetimes(data) + + +def _isoformat_all_datetimes(indate): + """Recursive function to isoformat all datetimes in a dictionary""" + + if isinstance(indate, list): + return [_isoformat_all_datetimes(i) for i in indate] + + if isinstance(indate, dict): + return {key: _isoformat_all_datetimes(indate[key]) for key in indate} + + if isinstance(indate, (datetime.datetime, datetime.date)): + return indate.isoformat() + + return indate + + +def _metadata_examples(): + return { + path.name: _isoformat_all_datetimes(_parse_yaml(path)) + for path in Path(".").absolute().glob("schema/definitions/0.8.0/examples/*.yml") + }