Merge pull request #4 from views-platform/feature_001_refactor_managers

Feature 001 refactor managers

tests/test_ensemble_manager.py

@@ -1,9 +1,9 @@
 import pytest
 from unittest.mock import patch, MagicMock
-from views_pipeline_core.managers.ensemble_manager import EnsembleManager
+from views_pipeline_core.managers.ensemble import EnsemblePathManager, EnsembleManager
 from views_pipeline_core.models.check import ensemble_model_check
-from views_pipeline_core.managers.path_manager import EnsemblePath, ModelPath
-from views_pipeline_core.managers.model_manager import ModelManager
+from views_pipeline_core.managers.model import ModelPathManager, ModelManager
+
 import logging
 import pandas as pd
 import os
@@ -99,7 +99,7 @@ class TestParametrized():
 
     @pytest.fixture
     def mock_config(self, args):
-        with patch("views_pipeline_core.managers.model_manager.ModelManager._update_single_config") as mock_update_single_config:
+        with patch("views_pipeline_core.managers.model.ModelManager._update_single_config") as mock_update_single_config:
             print(mock_update_single_config(args))
             return {
             "name": "test_model",
@@ -118,7 +118,7 @@ def mock_ensemble_manager(self, mock_model_path, args):
         manager._evaluate_ensemble = MagicMock()
         manager._forecast_ensemble = MagicMock()
         manager._eval_type = args.eval_type
-        with patch("views_pipeline_core.managers.model_manager.ModelManager._update_single_config") as mock_update_single_config:
+        with patch("views_pipeline_core.managers.model.ModelManager._update_single_config") as mock_update_single_config:
             manager.config = mock_update_single_config(args)
         manager.config = {"name": "test_model"}
         return manager
@@ -172,8 +172,8 @@ def test_get_shell_command(self, mock_model_path, args, expected_command, expect
     #     assert mock_train_model_artifact.call_count == 2
 
 
-    @patch('views_pipeline_core.managers.ensemble_manager.EnsembleManager._execute_model_tasks')
-    @patch('views_pipeline_core.managers.model_manager.ModelManager._update_single_config')
+    @patch('views_pipeline_core.managers.ensemble.EnsembleManager._execute_model_tasks')
+    @patch('views_pipeline_core.managers.model.ModelManager._update_single_config')
     @patch('views_pipeline_core.models.check.ensemble_model_check')
     def test_execute_single_run(
         self,
@@ -279,12 +279,12 @@ def test_execute_model_tasks(
     #         expected_shell_command
     #     ):
     #         # Mocking necessary components
-    #         with patch("views_pipeline_core.managers.ensemble_manager.ModelPath") as mock_model_path, \
+    #         with patch("views_pipeline_core.managers.ensemble_manager.ModelPathManager") as mock_model_path, \
     #             patch("views_pipeline_core.managers.ensemble_manager.ModelManager") as mock_model_manager, \
     #             patch("views_pipeline_core.managers.ensemble_manager.EnsembleManager._get_shell_command") as mock_get_shell_command, \
     #             patch("subprocess.run") as mock_subprocess_run:
 
-    #             # Mocking the return value of ModelPath
+    #             # Mocking the return value of ModelPathManager
     #             mock_model_path_instance = MagicMock()
     #             mock_model_path.return_value = mock_model_path_instance
 
@@ -318,7 +318,7 @@ def test_train_ensemble(self, mock_model_path, args,
         expected_command,
         expected_methods_called):
         # Create a mock for the ensemble manager
-        with patch("views_pipeline_core.managers.ensemble_manager.EnsembleManager._train_model_artifact") as mock_train_model_artifact:
+        with patch("views_pipeline_core.managers.ensemble.EnsembleManager._train_model_artifact") as mock_train_model_artifact:
             print("Mocking works:", mock_train_model_artifact)
             manager = EnsembleManager(ensemble_path=mock_model_path)
             manager.config = {

tests/test_ensemble_path.py → tests/test_ensemble_path_manager.py

@@ -2,7 +2,7 @@
 from unittest.mock import patch, MagicMock
 from pathlib import Path
 import sys
-from views_pipeline_core.managers.path_manager import EnsemblePath
+from views_pipeline_core.managers.ensemble import EnsemblePathManager
 
 @pytest.fixture
 def temp_dir(tmp_path):
@@ -33,28 +33,28 @@ def test_initialization_with_valid_name(temp_dir):
     """
     project_root, ensemble_dir = temp_dir
     # Patch the class-level attributes and methods to use the temporary directory structure
-    with patch.object(EnsemblePath, '_root', project_root):
-        with patch.object(EnsemblePath, 'get_models', return_value=project_root / "ensembles"):
-            with patch('views_pipeline_core.managers.path_manager.EnsemblePath._get_model_dir', return_value=ensemble_dir):
-                # Initialize the EnsemblePath instance with a valid ensemble name
-                ensemble_path_instance = EnsemblePath(ensemble_name_or_path="test_ensemble", validate=True)
+    with patch.object(EnsemblePathManager, '_root', project_root):
+        with patch.object(EnsemblePathManager, 'get_models', return_value=project_root / "ensembles"):
+            with patch('views_pipeline_core.managers.ensemble.EnsemblePathManager._get_model_dir', return_value=ensemble_dir):
+                # Initialize the EnsemblePathManager instance with a valid ensemble name
+                ensemble_path_instance = EnsemblePathManager(ensemble_name_or_path="test_ensemble", validate=True)
                 # Assert that the ensemble name and directories are correctly set
                 assert ensemble_path_instance.model_name == "test_ensemble"
                 assert ensemble_path_instance.root == project_root
                 assert ensemble_path_instance.model_dir == ensemble_dir
 
 def test_initialization_with_invalid_name(temp_dir):
     """
-    Test the initialization of EnsemblePath with an invalid ensemble name.
+    Test the initialization of EnsemblePathManager with an invalid ensemble name.
 
     Args:
         temp_dir (tuple): A tuple containing the project root directory and the ensemble directory.
     """
     project_root, _ = temp_dir
     # Patch the class-level attributes and methods to use the temporary directory structure
-    with patch.object(EnsemblePath, '_root', project_root):
-        with patch.object(EnsemblePath, 'get_models', return_value=project_root / "ensembles"):
-            with patch('views_pipeline_core.managers.path_manager.EnsemblePath._get_model_dir', return_value=None):
+    with patch.object(EnsemblePathManager, '_root', project_root):
+        with patch.object(EnsemblePathManager, 'get_models', return_value=project_root / "ensembles"):
+            with patch('views_pipeline_core.managers.ensemble.EnsemblePathManager._get_model_dir', return_value=None):
                 # Assert that initializing with an invalid ensemble name raises a ValueError
                 with pytest.raises(ValueError):
-                    EnsemblePath(ensemble_name_or_path="invalidensemble", validate=True)
+                    EnsemblePathManager(ensemble_name_or_path="invalidensemble", validate=True)

tests/test_files_utils.py

@@ -1,7 +1,7 @@
 import pytest
 from unittest.mock import patch, mock_open
-from views_pipeline_core.files.utils import read_log_file, create_data_fetch_log_file, create_specific_log_file, create_log_file
-from pathlib import Path
+from views_pipeline_core.files.utils import read_log_file, create_data_fetch_log_file, create_specific_log_file, create_log_file, read_dataframe, save_dataframe
+import pandas as pd
 
 def test_read_log_file():
     """
@@ -91,36 +91,48 @@ def test_create_specific_log_file(tmp_path):
         assert "Data Fetch Timestamp: 2023-10-01T12:00:00Z" in content
         assert "Deployment Status: deployed" in content
 
-# def test_create_log_file(tmp_path):
-#     path_generated = tmp_path / "generated"
-#     path_generated.mkdir()
-#     model_config = {
-#         "run_type": "test_run",
-#         "name": "test_model",
-#         "deployment_status": "deployed"
-#     }
-#     model_timestamp = "2023-10-01T12:00:00Z"
-#     data_generation_timestamp = "2023-10-01T12:00:00Z"
-#     data_fetch_timestamp = "2023-10-01T12:00:00Z"
-
-#     with patch("views_pipeline.managers.path_manager.ModelPath") as MockModelPath:
-#         mock_model_path_instance = MockModelPath.return_value
-#         mock_model_path_instance.data_generated = path_generated
-#         MockModelPath.side_effect = lambda name, validate: mock_model_path_instance if name == "test_model" and not validate else None
-
-#         create_log_file(path_generated, model_config, model_timestamp, data_generation_timestamp, data_fetch_timestamp, models=["first_model", "second_model"])
-
-#         log_file_path = path_generated / f"{model_config['run_type']}_log.txt"
-#         assert log_file_path.exists()
-
-#         with open(log_file_path, "r") as file:
-#             content = file.read()
-#             assert "Single Model Name: test_model" in content
-#             assert "Single Model Timestamp: 2023-10-01T12:00:00Z" in content
-#             assert "Data Generation Timestamp: 2023-10-01T12:00:00Z" in content
-#             assert "Data Fetch Timestamp: 2023-10-01T12:00:00Z" in content
-#             assert "Deployment Status: deployed" in content
-
-#             # Check for appended model logs
-#             assert "Single Model Name: first_model" in content
-#             assert "Single Model Name: second_model" in content
+@pytest.fixture
+def sample_dataframe():
+    return pd.DataFrame({
+        "column1": [1, 2, 3],
+        "column2": ["a", "b", "c"]
+    })
+
+# def test_save_dataframe_csv(tmp_path, sample_dataframe):
+#     save_path = tmp_path / "test.csv"
+#     save_dataframe(sample_dataframe, save_path)
+#     assert save_path.exists()
+#     df = pd.read_csv(save_path)
+#     pd.testing.assert_frame_equal(df, sample_dataframe)
+
+# def test_save_dataframe_xlsx(tmp_path, sample_dataframe):
+#     save_path = tmp_path / "test.xlsx"
+#     save_dataframe(sample_dataframe, save_path)
+#     assert save_path.exists()
+#     df = pd.read_excel(save_path)
+#     pd.testing.assert_frame_equal(df, sample_dataframe)
+
+def test_save_dataframe_parquet(tmp_path, sample_dataframe):
+    save_path = tmp_path / "test.parquet"
+    save_dataframe(sample_dataframe, save_path)
+    assert save_path.exists()
+    df = pd.read_parquet(save_path)
+    pd.testing.assert_frame_equal(df, sample_dataframe)
+
+def test_save_dataframe_pickle(tmp_path, sample_dataframe):
+    save_path = tmp_path / "test.pkl"
+    save_dataframe(sample_dataframe, save_path)
+    assert save_path.exists()
+    df = pd.read_pickle(save_path)
+    pd.testing.assert_frame_equal(df, sample_dataframe)
+
+def test_save_dataframe_invalid_extension(tmp_path, sample_dataframe):
+    save_path = tmp_path / "test.txt"
+    with pytest.raises(ValueError, match="The file extension must be provided. E.g. .parquet"):
+        save_dataframe(sample_dataframe, save_path)
+
+# def test_read_dataframe_csv(tmp_path, sample_dataframe):
+#     file_path = tmp_path / "test.csv"
+#     sample_dataframe.to_csv(file_path)
+#     df = read_dataframe(file_path)
+#     pd.testing.assert_frame_equal(df, sample_dataframe)

tests/test_model_manager.py

@@ -1,8 +1,8 @@
 import pytest
 from unittest.mock import MagicMock, patch, mock_open
-from views_pipeline_core.managers.path_manager import ModelPath, EnsemblePath
+from views_pipeline_core.managers.model import ModelManager
 from views_pipeline_core.data.dataloaders import ViewsDataLoader
-from views_pipeline_core.managers.model_manager import ModelManager
+from views_pipeline_core.managers.model import ModelManager
 import wandb
 import pandas as pd
 from pathlib import Path
@@ -15,7 +15,7 @@ def mock_model_path():
     Yields:
         MagicMock: The mock object for ModelPath.
     """
-    with patch("views_pipeline_core.managers.path_manager.ModelPath") as mock:
+    with patch("views_pipeline_core.managers.model.ModelPathManager") as mock:
         mock_instance = mock.return_value
         mock_instance.get_scripts.return_value = {
             "config_deployment.py": "path/to/config_deployment.py",
@@ -34,7 +34,7 @@ def mock_ensemble_path():
     Yields:
         MagicMock: The mock object for EnsemblePath.
     """
-    with patch("views_pipeline_core.managers.path_manager.EnsemblePath") as mock:
+    with patch("views_pipeline_core.managers.ensemble.EnsemblePathManager") as mock:
         yield mock
 
 @pytest.fixture