do not want to add an .h5 file so deepensemble also give an option to…

… create data dynamically

src/scripts/DeepEnsemble.py

@@ -169,17 +169,45 @@ def parse_args():
     rs = namespace.randomseed
     BATCH_SIZE = namespace.batchsize
     sigma = io.DataPreparation.get_sigma(noise)
+
+    # generate the df
+    data = io.DataPreparation()
+    data.sample_params_from_prior(size_df)
+    data.simulate_data(data.params, sigma, "linear_homogeneous")
+    df_array = data.get_dict()
+    # Convert non-tensor entries to tensors
+    df = {}
+    for key, value in df_array.items():
+
+        if isinstance(value, TensorDataset):
+            # Keep tensors as they are
+            df[key] = value
+        else:
+            # Convert lists to tensors
+            df[key] = torch.tensor(value)
+
+    len_df = len(df["params"][:, 0].numpy())
+    len_x = len(df["inputs"].numpy())
+    ms_array = np.repeat(df["params"][:, 0].numpy(), len_x)
+    bs_array = np.repeat(df["params"][:, 1].numpy(), len_x)
+    xs_array = np.tile(df["inputs"].numpy(), len_df)
+    ys_array = np.reshape(df["output"].numpy(), (len_df * len_x))
+
+    """
     loader = io.DataLoader()
-    data = loader.load_data_h5(
+    df = loader.load_data_h5(
         "linear_sigma_" + str(sigma) + "_size_" + str(size_df),
         path="/Users/rnevin/Documents/DeepUQ/data/",
     )
-    len_df = len(data["params"][:, 0].numpy())
-    len_x = len(data["inputs"].numpy())
-    ms_array = np.repeat(data["params"][:, 0].numpy(), len_x)
-    bs_array = np.repeat(data["params"][:, 1].numpy(), len_x)
-    xs_array = np.tile(data["inputs"].numpy(), len_df)
-    ys_array = np.reshape(data["output"].numpy(), (len_df * len_x))
+    len_df = len(df["params"][:, 0].numpy())
+    len_x = len(df["inputs"].numpy())
+    ms_array = np.repeat(df["params"][:, 0].numpy(), len_x)
+    bs_array = np.repeat(df["params"][:, 1].numpy(), len_x)
+    xs_array = np.tile(df["inputs"].numpy(), len_df)
+    ys_array = np.reshape(df["output"].numpy(), (len_df * len_x))
+    print(df)
+    STOP
+    """
     inputs = np.array([xs_array, ms_array, bs_array]).T
     model_inputs, model_outputs = io.DataPreparation.normalize(inputs,
                                                                ys_array,

test/test_DeepEnsemble.py

@@ -1,10 +1,8 @@
-import sys
 import pytest
 import os
 import subprocess
 import tempfile
 import shutil
-from scripts import evaluate, models, DeepEnsemble
 
 
 @pytest.fixture
@@ -29,6 +27,7 @@ def temp_directory():
     """
     shutil.rmtree(temp_dir)
 
+
 def test_chkpt_saved(temp_directory):
     noise_level = "low"
     n_models = 10
@@ -43,8 +42,8 @@ def test_chkpt_saved(temp_directory):
         "--n_epochs",
         str(n_epochs),
         "--save_final_checkpoint",
-        "--savefig"
-        ]
+        "--savefig",
+    ]
     # now run the subprocess
     subprocess.run(subprocess_args, check=True)
     # check if the right number of checkpoints are saved
@@ -78,7 +77,6 @@ def test_chkpt_saved(temp_directory):
         ), f"File '{file_name}' does not contain the expected substring"
 
 
-
 @pytest.mark.xfail(strict=True)
 def test_no_chkpt_saved_xfail(temp_directory):
     noise_level = "low"
@@ -127,8 +125,8 @@ def test_no_chkpt_saved(temp_directory):
     # list all files in the "models" folder
     files_in_models_folder = os.listdir(models_folder)
     # assert that the number of files is equal to 10
-    assert len(files_in_models_folder) == 0, "Expect 0 files in the 'models' folder"
-
+    assert len(files_in_models_folder) == 0, \
+        "Expect 0 files in the 'models' folder"
 
 
 def test_run_simple_ensemble(temp_directory):