diff --git a/README.md b/README.md index 51128e7..93b3b3e 100644 --- a/README.md +++ b/README.md @@ -12,22 +12,33 @@ DeepUQ is a package for injecting and measuring different types of uncertainty i ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/owner/repo/test-repo?label=test) ## Workflow -![Folder structure overview](images/folders_deepUQ.png) +![Folder structure overview](images/DeepUQWorkflow_Maggie.png) -Getting a little more specific: +The scripts can be accessed via the ipython example notebooks or via the model modules (ie `DeepEnsemble.py`). For example, to ingest data and train a Deep Ensemble: +> cd src/scripts/ -![python module overview](images/workflow_deepUQ.png) +> python DeepEnsemble.py -These modules can be accessed via the ipython example notebooks or via the model modules (ie `DeepEnsemble.py`). For example, to ingest data and train a Deep Ensemble: -> cd src/scripts/ +With no config file specified, this command will pull settings from the `default.py` file within `utils`. For the `DeepEnsemble.py` script, it will automatically select the `DefaultsDE` dictionary. + +Another option is to specify your own config file: + +> python DeepEnsemble.py --config "path/to/config/myconfig.yaml" -> python DeepEnsemble.py low 10 /Users/rnevin/Documents/DeepUQ/ --save_final_checkpoint --savefig --n_epochs=10 +Where you would modify the "path/to/config/myconfig.yaml" to specify where your own yaml lives. + +The third option is to input settings on the command line. These choices are then combined with the default settings and output in a temporary yaml. + +> python DeepEnsemble.py --noise_level "low" --n_models 10 --out_dir ./DeepUQResources/results/--save_final_checkpoint True --savefig True --n_epochs 10 This command will train a 10 network, 10 epoch ensemble on the low noise data and will save figures and final checkpoints to the specified directory. Required arguments are the noise setting (low/medium/high), the number of ensembles, and the working directory. For more information on the arguments: > python DeepEnsemble.py --help +The other available script is the `DeepEvidentialRegression.py` script: +> python DeepEvidentialRegression.py --help + ## Installation ### Clone this repo diff --git a/images/DeepUQWorkflow_Maggie.png b/images/DeepUQWorkflow_Maggie.png new file mode 100644 index 0000000..4d5c25f Binary files /dev/null and b/images/DeepUQWorkflow_Maggie.png differ diff --git a/ms.pdf b/ms.pdf deleted file mode 100644 index bc5add6..0000000 Binary files a/ms.pdf and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index 2bd02ae..93f9f9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,10 @@ authors = ["beckynevin "] readme = "README.md" license = "MIT" +[tool.poetry.scripts] +ensamble = "src.scripts.DeepEnsemble:main" +der = "src.scripts.DeepEvidentialRegression:main" + [tool.poetry.dependencies] python = ">=3.9,<3.11" jupyter = "^1.0.0" diff --git a/showyourwork.yml b/showyourwork.yml deleted file mode 100644 index 206454e..0000000 --- a/showyourwork.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 0.3.0 - -overleaf: - id: 62defbc43013ada57aa4c153 - push: - - src/tex/figures - - src/tex/output \ No newline at end of file diff --git a/src/data/__init__.py b/src/data/__init__.py new file mode 100644 index 0000000..790f378 --- /dev/null +++ b/src/data/__init__.py @@ -0,0 +1,6 @@ +from data.data import MyDataLoader, DataPreparation + +DataModules = { + "MyDataLoader": MyDataLoader, + "DataPreparation": DataPreparation +} diff --git a/src/scripts/io.py b/src/data/data.py similarity index 63% rename from src/scripts/io.py rename to src/data/data.py index 5f38eda..b5f32fd 100644 --- a/src/scripts/io.py +++ b/src/data/data.py @@ -1,107 +1,17 @@ # Contains modules used to prepare a dataset # with varying noise properties -import argparse import numpy as np from sklearn.model_selection import train_test_split import pickle from torch.distributions import Uniform -from torch.utils.data import TensorDataset import torch import h5py -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - parser.add_argument( - "size_df", - type=float, - required=False, - default=1000, - help="Used to load the associated .h5 data file", - ) - parser.add_argument( - "noise_level", - type=str, - required=False, - default="low", - help="low, medium, high or vhigh, \ - used to look up associated sigma value", - ) - parser.add_argument( - "size_df", - type=str, - nargs="?", - default="/repo/embargo", - help="Butler Repository path from which data is transferred. \ - Input str. Default = '/repo/embargo'", - ) - parser.add_argument( - "--normalize", - required=False, - action="store_true", - help="If true theres an option to normalize the dataset", - ) - parser.add_argument( - "--val_proportion", - type=float, - required=False, - default=0.1, - help="Proportion of the dataset to use as validation", - ) - parser.add_argument( - "--randomseed", - type=float, - required=False, - default=42, - help="Random seed used for shuffling the training and validation set", - ) - parser.add_argument( - "--batchsize", - type=float, - required=False, - default=100, - help="Size of batched used in the traindataloader", - ) - return parser.parse_args() - - -class ModelLoader: - def save_model_pkl(self, path, model_name, posterior): - """ - Save the pkl'ed saved posterior model - - :param path: Location to save the model - :param model_name: Name of the model - :param posterior: Model object to be saved - """ - file_name = path + model_name + ".pkl" - with open(file_name, "wb") as file: - pickle.dump(posterior, file) - - def load_model_pkl(self, path, model_name): - """ - Load the pkl'ed saved posterior model - - :param path: Location to load the model from - :param model_name: Name of the model - :return: Loaded model object that can be used with the predict function - """ - print(path) - with open(path + model_name + ".pkl", "rb") as file: - posterior = pickle.load(file) - return posterior - - def predict(input, model): - """ - - :param input: loaded object used for inference - :param model: loaded model - :return: Prediction - """ - return 0 - +class MyDataLoader: + def __init__(self): + self.data = None -class DataLoader: def save_data_pkl(self, data_name, data, path="../data/"): """ Save and load the pkl'ed training/test set @@ -197,7 +107,7 @@ def simulate_data( sigma, simulation_name, x=np.linspace(0, 100, 101), - seed=13 + seed=42 ): if simulation_name == "linear_homogeneous": # convert to numpy array (if tensor): @@ -300,35 +210,3 @@ def train_val_split( random_state=random_state, ) return x_train, x_val, y_train, y_val - - -# Example usage: -if __name__ == "__main__": - namespace = parse_args() - size_df = namespace.size_df - noise = namespace.noise_level - norm = namespace.normalize - val_prop = namespace.val_proportion - rs = namespace.randomseed - BATCH_SIZE = namespace.batchsize - sigma = DataPreparation.get_sigma(noise) - loader = DataLoader() - data = loader.load_data_h5("linear_sigma_" + str(sigma) + - "_size_" + str(size_df)) - len_df = len(data["params"][:, 0].numpy()) - len_x = len(data["inputs"].numpy()) - ms_array = np.repeat(data["params"][:, 0].numpy(), len_x) - bs_array = np.repeat(data["params"][:, 1].numpy(), len_x) - xs_array = np.tile(data["inputs"].numpy(), len_df) - ys_array = np.reshape(data["output"].numpy(), (len_df * len_x)) - inputs = np.array([xs_array, ms_array, bs_array]).T - model_inputs, model_outputs = DataPreparation.normalize(inputs, - ys_array, - norm) - x_train, x_val, y_train, y_val = DataPreparation.train_val_split( - model_inputs, model_outputs, test_size=val_prop, random_state=rs - ) - trainData = TensorDataset(torch.Tensor(x_train), torch.Tensor(y_train)) - trainDataLoader = DataLoader(trainData, - batch_size=BATCH_SIZE, - shuffle=True) diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..448baa4 --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1,5 @@ +from models.models import ModelLoader + +ModelModules = { + "ModelLoader": ModelLoader +} diff --git a/src/scripts/models.py b/src/models/models.py similarity index 84% rename from src/scripts/models.py rename to src/models/models.py index 73e5802..8140562 100644 --- a/src/scripts/models.py +++ b/src/models/models.py @@ -1,9 +1,48 @@ +# Contains modules used to prepare a dataset +# with varying noise properties import numpy as np -import torch.nn as nn +import pickle import torch +import torch.nn as nn import math +class ModelLoader: + def save_model_pkl(self, path, model_name, posterior): + """ + Save the pkl'ed saved posterior model + + :param path: Location to save the model + :param model_name: Name of the model + :param posterior: Model object to be saved + """ + file_name = path + model_name + ".pkl" + with open(file_name, "wb") as file: + pickle.dump(posterior, file) + + def load_model_pkl(self, path, model_name): + """ + Load the pkl'ed saved posterior model + + :param path: Location to load the model from + :param model_name: Name of the model + :return: Loaded model object that can be used with the predict function + """ + print(path) + with open(path + model_name + ".pkl", "rb") as file: + posterior = pickle.load(file) + return posterior + + def predict(input, model): + """ + + :param input: loaded object used for inference + :param model: loaded model + :return: Prediction + """ + return 0 + + class DERLayer(nn.Module): def __init__(self): super().__init__() @@ -209,27 +248,3 @@ def loss_bnll(mean, variance, target, beta): # beta=0.5): if beta > 0: loss = loss * (variance.detach() ** beta) return loss.sum(axis=-1) - - -''' -def get_loss(transform, beta=None): - if beta: - def beta_nll_loss(targets, outputs, beta=beta): - """Compute beta-NLL loss - """ - mu = outputs[..., 0:1] - var = transform(outputs[..., 1:2]) - loss = (K.square((targets - mu)) / var + K.log(var)) - loss = loss * K.stop_gradient(var) ** beta - return loss - return beta_nll_loss - else: - def negative_log_likelihood(targets, outputs): - """Calculate the negative loglikelihood.""" - mu = outputs[..., 0:1] - var = transform(outputs[..., 1:2]) - y = targets[..., 0:1] - loglik = - K.log(var) - K.square((y - mu)) / var - return - loglik - return negative_log_likelihood -''' diff --git a/src/scripts/DeepEnsemble.py b/src/scripts/DeepEnsemble.py index e8c679c..ba45f6d 100644 --- a/src/scripts/DeepEnsemble.py +++ b/src/scripts/DeepEnsemble.py @@ -1,39 +1,67 @@ +import os +import yaml import argparse import numpy as np import torch -from torch.utils.data import DataLoader, TensorDataset -from scripts import train, models, io +from torch.utils.data import TensorDataset, DataLoader +# from scripts import train, models, io +from train import train +from models import models +from data import DataModules +from models import ModelModules +from utils.config import Config +from utils.defaults import DefaultsDE +from data.data import DataPreparation, MyDataLoader -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) +# from plots import Plots def parse_args(): parser = argparse.ArgumentParser(description="data handling module") + # there are three options with the parser: + # 1) Read from a yaml + # 2) Reads from the command line and default file + # and dumps to yaml + + # option to pass name of config + parser.add_argument("--config", "-c", default=None) + + # data info + parser.add_argument( + "--data_path", + "-d", + default=DefaultsDE["data"]["data_path"], + choices=DataModules.keys(), + ) + parser.add_argument( + "--data_engine", + "-dl", + default=DefaultsDE["data"]["data_engine"], + choices=DataModules.keys(), + ) + + # model + # path to save the model results + parser.add_argument("--out_dir", default=DefaultsDE["common"]["out_dir"]) + parser.add_argument( + "--model_engine", + "-e", + default=DefaultsDE["model"]["model_engine"], + choices=ModelModules.keys(), + ) parser.add_argument( "--size_df", type=float, required=False, - default=1000, + default=DefaultsDE["data"]["size_df"], help="Used to load the associated .h5 data file", ) parser.add_argument( - "noise_level", + "--noise_level", type=str, - default="low", + default=DefaultsDE["data"]["noise_level"], + choices=["low", "medium", "high", "vhigh"], help="low, medium, high or vhigh, \ used to look up associated sigma value", ) @@ -41,26 +69,27 @@ def parse_args(): "--normalize", required=False, action="store_true", + default=DefaultsDE["data"]["normalize"], help="If true theres an option to normalize the dataset", ) parser.add_argument( "--val_proportion", type=float, required=False, - default=0.1, + default=DefaultsDE["data"]["val_proportion"], help="Proportion of the dataset to use as validation", ) parser.add_argument( "--randomseed", type=int, required=False, - default=42, + default=DefaultsDE["data"]["randomseed"], help="Random seed used for shuffling the training and validation set", ) parser.add_argument( "--generatedata", action="store_true", - default=False, + default=DefaultsDE["data"]["generatedata"], help="option to generate df, if not specified \ default behavior is to load from file", ) @@ -68,117 +97,172 @@ def parse_args(): "--batchsize", type=int, required=False, - default=100, + default=DefaultsDE["data"]["batchsize"], help="Size of batched used in the traindataloader", ) # now args for model parser.add_argument( - "n_models", + "--n_models", type=int, - default=100, + default=DefaultsDE["model"]["n_models"], help="Number of MVEs in the ensemble", ) parser.add_argument( "--init_lr", type=float, required=False, - default=0.001, + default=DefaultsDE["model"]["init_lr"], help="Learning rate", ) parser.add_argument( "--loss_type", type=str, required=False, - default="bnll_loss", + default=DefaultsDE["model"]["loss_type"], help="Loss types for MVE, options are no_var_loss, var_loss, \ - and bnn_loss", + and bnn_loss", ) parser.add_argument( "--BETA", type=beta_type, required=False, - default=0.5, + default=DefaultsDE["model"]["BETA"], help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "wd", - type=str, - help="Top level of directory, required arg", + there are string options: linear_decrease, \ + step_decrease_to_0.5, and step_decrease_to_1.0", ) parser.add_argument( "--model_type", type=str, required=False, - default="DE", + default=DefaultsDE["model"]["model_type"], help="Beginning of name for saved checkpoints and figures", ) parser.add_argument( "--n_epochs", type=int, required=False, - default=100, + default=DefaultsDE["model"]["n_epochs"], help="number of epochs for each MVE", ) - parser.add_argument( - "--path_to_models", - type=str, - required=False, - default="models/", - help="path to where the checkpoints are saved", - ) parser.add_argument( "--save_all_checkpoints", action="store_true", - default=False, + default=DefaultsDE["model"]["save_all_checkpoints"], help="option to save all checkpoints", ) parser.add_argument( "--save_final_checkpoint", action="store_true", # Set to True if argument is present - default=False, # Set default value to False if argument is not present + default=DefaultsDE["model"]["save_final_checkpoint"], help="option to save the final epoch checkpoint for each ensemble", ) parser.add_argument( "--overwrite_final_checkpoint", action="store_true", - default=False, + default=DefaultsDE["model"]["overwrite_final_checkpoint"], help="option to overwite already saved checkpoints", ) parser.add_argument( "--plot", action="store_true", - default=False, + default=DefaultsDE["model"]["plot"], help="option to plot in notebook", ) parser.add_argument( "--savefig", action="store_true", - default=False, + default=DefaultsDE["model"]["savefig"], help="option to save a figure of the true and predicted values", ) parser.add_argument( "--verbose", action="store_true", - default=False, + default=DefaultsDE["model"]["verbose"], help="verbose option for train", ) - return parser.parse_args() + args = parser.parse_args() + args = parser.parse_args() + if args.config is not None: + print("Reading settings from config file", args.config) + config = Config(args.config) + + else: + temp_config = DefaultsDE["common"]["temp_config"] + print( + "Reading settings from cli and default, \ + dumping to temp config: ", + temp_config, + ) + os.makedirs(os.path.dirname(temp_config), exist_ok=True) + + # check if args were specified in cli + # if not, default is from DefaultsDE dictionary + input_yaml = { + "common": {"out_dir": args.out_dir}, + "model": { + "model_engine": args.model_engine, + "model_type": args.model_type, + "loss_type": args.loss_type, + "n_models": args.n_models, + "init_lr": args.init_lr, + "BETA": args.BETA, + "n_epochs": args.n_epochs, + "save_all_checkpoints": args.save_all_checkpoints, + "save_final_checkpoint": args.save_final_checkpoint, + "overwrite_final_checkpoint": args.overwrite_final_checkpoint, + "plot": args.plot, + "savefig": args.savefig, + "verbose": args.verbose, + }, + "data": { + "data_path": args.data_path, + "data_engine": args.data_engine, + "size_df": args.size_df, + "noise_level": args.noise_level, + "val_proportion": args.val_proportion, + "randomseed": args.randomseed, + "batchsize": args.batchsize, + }, + # "plots": {key: {} for key in args.plots}, + # "metrics": {key: {} for key in args.metrics}, + } + + yaml.dump(input_yaml, open(temp_config, "w")) + config = Config(temp_config) + + return config + # return parser.parse_args() + + +def beta_type(value): + if isinstance(value, float): + return value + elif value.lower() == "linear_decrease": + return value + elif value.lower() == "step_decrease_to_0.5": + return value + elif value.lower() == "step_decrease_to_1.0": + return value + else: + raise argparse.ArgumentTypeError( + "BETA must be a float or one of 'linear_decrease', \ + 'step_decrease_to_0.5', 'step_decrease_to_1.0'" + ) if __name__ == "__main__": - namespace = parse_args() - size_df = namespace.size_df - noise = namespace.noise_level - norm = namespace.normalize - val_prop = namespace.val_proportion - rs = namespace.randomseed - BATCH_SIZE = namespace.batchsize - sigma = io.DataPreparation.get_sigma(noise) - if namespace.generatedata: + config = parse_args() + size_df = config.get_item("data", "size_df", "DE") + noise = config.get_item("data", "noise_level", "DE") + norm = config.get_item("data", "normalize", "DE", raise_exception=False) + val_prop = config.get_item("data", "val_proportion", "DE") + rs = config.get_item("data", "randomseed", "DE") + BATCH_SIZE = config.get_item("data", "batchsize", "DE") + sigma = DataPreparation.get_sigma(noise) + if config.get_item("data", "generatedata", "DE", raise_exception=False): # generate the df - data = io.DataPreparation() + data = DataPreparation() data.sample_params_from_prior(size_df) data.simulate_data(data.params, sigma, "linear_homogeneous") df_array = data.get_dict() @@ -193,7 +277,7 @@ def parse_args(): # Convert lists to tensors df[key] = torch.tensor(value) else: - loader = io.DataLoader() + loader = MyDataLoader() df = loader.load_data_h5( "linear_sigma_" + str(sigma) + "_size_" + str(size_df), path="/Users/rnevin/Documents/DeepUQ/data/", @@ -206,39 +290,59 @@ def parse_args(): ys_array = np.reshape(df["output"].numpy(), (len_df * len_x)) inputs = np.array([xs_array, ms_array, bs_array]).T - model_inputs, model_outputs = io.DataPreparation.normalize(inputs, - ys_array, - norm) - x_train, x_val, y_train, y_val = io.DataPreparation.train_val_split( + model_inputs, model_outputs = DataPreparation.normalize( + inputs, + ys_array, + norm) + x_train, x_val, y_train, y_val = DataPreparation.train_val_split( model_inputs, model_outputs, val_proportion=val_prop, random_state=rs ) trainData = TensorDataset(torch.Tensor(x_train), torch.Tensor(y_train)) - trainDataLoader = DataLoader(trainData, - batch_size=BATCH_SIZE, - shuffle=True) - print("[INFO] initializing the gal model...") + trainDataLoader = DataLoader( + trainData, + batch_size=BATCH_SIZE, + shuffle=True) # set the device we will be using to train the model DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model_name = namespace.model_type + "_noise_" + noise - model, lossFn = models.model_setup_DE(namespace.loss_type, DEVICE) + model_name = config.get_item( + "model", + "model_type", + "DE") + "_noise_" + noise + model, lossFn = models.model_setup_DE( + config.get_item("model", "loss_type", "DE"), DEVICE + ) + print( + "save final checkpoint has this value", + config.get_item("model", "save_final_checkpoint", "DE"), + ) model_ensemble = train.train_DE( trainDataLoader, x_val, y_val, - namespace.init_lr, + config.get_item("model", "init_lr", "DE"), DEVICE, - namespace.loss_type, - namespace.n_models, - namespace.wd, + config.get_item("model", "loss_type", "DE"), + config.get_item("model", "n_models", "DE"), model_name, - BETA=namespace.BETA, - EPOCHS=namespace.n_epochs, - path_to_model=namespace.path_to_models, - save_all_checkpoints=namespace.save_all_checkpoints, - save_final_checkpoint=namespace.save_final_checkpoint, - overwrite_final_checkpoint=namespace.overwrite_final_checkpoint, - plot=namespace.plot, - savefig=namespace.savefig, - verbose=namespace.verbose, + BETA=config.get_item("model", "BETA", "DE"), + EPOCHS=config.get_item("model", "n_epochs", "DE"), + path_to_model=config.get_item( + "common", + "out_dir", + "DE"), + save_all_checkpoints=config.get_item( + "model", + "save_all_checkpoints", + "DE"), + save_final_checkpoint=config.get_item( + "model", + "save_final_checkpoint", + "DE"), + overwrite_final_checkpoint=config.get_item( + "model", "overwrite_final_checkpoint", "DE" + ), + plot=config.get_item("model", "plot", "DE"), + savefig=config.get_item("model", "savefig", "DE"), + verbose=config.get_item("model", "verbose", "DE"), ) diff --git a/src/scripts/DeepEvidentialRegression.py b/src/scripts/DeepEvidentialRegression.py index 3616f67..5092253 100644 --- a/src/scripts/DeepEvidentialRegression.py +++ b/src/scripts/DeepEvidentialRegression.py @@ -1,23 +1,67 @@ +import os +import yaml import argparse import numpy as np import torch -from torch.utils.data import DataLoader, TensorDataset -from scripts import train, models, io +from torch.utils.data import TensorDataset, DataLoader + +# from scripts import train, models, io +from train import train +from models import models +from data import DataModules +from models import ModelModules +from utils.config import Config +from utils.defaults import DefaultsDER +from data.data import DataPreparation, MyDataLoader + +# from plots import Plots def parse_args(): parser = argparse.ArgumentParser(description="data handling module") + # there are three options with the parser: + # 1) Read from a yaml + # 2) Reads from the command line and default file + # and dumps to yaml + + # option to pass name of config + parser.add_argument("--config", "-c", default=None) + + # data info + parser.add_argument( + "--data_path", + "-d", + default=DefaultsDER["data"]["data_path"], + choices=DataModules.keys(), + ) + parser.add_argument( + "--data_engine", + "-dl", + default=DefaultsDER["data"]["data_engine"], + choices=DataModules.keys(), + ) + + # model + parser.add_argument("--out_dir", default=DefaultsDER["common"]["out_dir"]) + parser.add_argument( + "--model_engine", + "-e", + default=DefaultsDER["model"]["model_engine"], + choices=ModelModules.keys(), + ) + parser.add_argument( "--size_df", type=float, required=False, - default=1000, + default=DefaultsDER["data"]["size_df"], help="Used to load the associated .h5 data file", ) parser.add_argument( - "noise_level", + "--noise_level", type=str, - default="low", + default=DefaultsDER["data"]["noise_level"], + choices=["low", "medium", "high", "vhigh"], help="low, medium, high or vhigh, \ used to look up associated sigma value", ) @@ -25,26 +69,27 @@ def parse_args(): "--normalize", required=False, action="store_true", + default=DefaultsDER["data"]["normalize"], help="If true theres an option to normalize the dataset", ) parser.add_argument( "--val_proportion", type=float, required=False, - default=0.1, + default=DefaultsDER["data"]["val_proportion"], help="Proportion of the dataset to use as validation", ) parser.add_argument( "--randomseed", type=int, required=False, - default=42, + default=DefaultsDER["data"]["randomseed"], help="Random seed used for shuffling the training and validation set", ) parser.add_argument( "--generatedata", action="store_true", - default=False, + default=DefaultsDER["data"]["generatedata"], help="option to generate df, if not specified \ default behavior is to load from file", ) @@ -52,111 +97,140 @@ def parse_args(): "--batchsize", type=int, required=False, - default=100, + default=DefaultsDER["data"]["batchsize"], help="Size of batched used in the traindataloader", ) - # now args for model parser.add_argument( "--init_lr", type=float, required=False, - default=0.001, + default=DefaultsDER["model"]["init_lr"], help="Learning rate", ) - parser.add_argument( - "--coeff", - type=float, - required=False, - default=0.5, - help="Coeff, see DER lit", - ) parser.add_argument( "--loss_type", type=str, required=False, - default="SDER", - help="Loss types. \ - For MVE, options are no_var_loss, var_loss, \ - and bnn_loss. \ - For DER, options are DER or SDER", + default=DefaultsDER["model"]["loss_type"], + help="Loss types for DER", ) parser.add_argument( - "wd", - type=str, - help="Top level of directory, required arg", + "--COEFF", + type=float, + required=False, + default=DefaultsDER["model"]["COEFF"], + help="Coefficient for DER", ) parser.add_argument( "--model_type", type=str, required=False, - default="DER", + default=DefaultsDER["model"]["model_type"], help="Beginning of name for saved checkpoints and figures", ) parser.add_argument( "--n_epochs", type=int, required=False, - default=100, + default=DefaultsDER["model"]["n_epochs"], help="number of epochs for each MVE", ) - parser.add_argument( - "--path_to_models", - type=str, - required=False, - default="models/", - help="path to where the checkpoints are saved", - ) parser.add_argument( "--save_all_checkpoints", action="store_true", - default=False, + default=DefaultsDER["model"]["save_all_checkpoints"], help="option to save all checkpoints", ) parser.add_argument( "--save_final_checkpoint", action="store_true", # Set to True if argument is present - default=False, # Set default value to False if argument is not present + default=DefaultsDER["model"]["save_final_checkpoint"], help="option to save the final epoch checkpoint for each ensemble", ) parser.add_argument( "--overwrite_final_checkpoint", action="store_true", - default=False, + default=DefaultsDER["model"]["overwrite_final_checkpoint"], help="option to overwite already saved checkpoints", ) parser.add_argument( "--plot", action="store_true", - default=False, + default=DefaultsDER["model"]["plot"], help="option to plot in notebook", ) parser.add_argument( "--savefig", action="store_true", - default=False, + default=DefaultsDER["model"]["savefig"], help="option to save a figure of the true and predicted values", ) parser.add_argument( "--verbose", action="store_true", - default=False, + default=DefaultsDER["model"]["verbose"], help="verbose option for train", ) - return parser.parse_args() + args = parser.parse_args() + if args.config is not None: + config = Config(args.config) + + else: + temp_config = DefaultsDER["common"]["temp_config"] + os.makedirs(os.path.dirname(temp_config), exist_ok=True) + + input_yaml = { + "common": {"out_dir": args.out_dir}, + "model": { + "model_path": args.out_dir, + "model_engine": args.model_engine, + "model_type": args.model_type, + "loss_type": args.loss_type, + "init_lr": args.init_lr, + "COEFF": args.COEFF, + "n_epochs": args.n_epochs, + "save_all_checkpoints": args.save_all_checkpoints, + "save_final_checkpoint": args.save_final_checkpoint, + "overwrite_final_checkpoint": args.overwrite_final_checkpoint, + "plot": args.plot, + "savefig": args.savefig, + "verbose": args.verbose, + }, + "data": { + "data_path": args.data_path, + "data_engine": args.data_engine, + "size_df": args.size_df, + "noise_level": args.noise_level, + "val_proportion": args.val_proportion, + "randomseed": args.randomseed, + "batchsize": args.batchsize, + }, + # "plots": {key: {} for key in args.plots}, + # "metrics": {key: {} for key in args.metrics}, + } + + yaml.dump(input_yaml, open(temp_config, "w")) + config = Config(temp_config) + + return config if __name__ == "__main__": - namespace = parse_args() - size_df = namespace.size_df - noise = namespace.noise_level - norm = namespace.normalize - val_prop = namespace.val_proportion - rs = namespace.randomseed - BATCH_SIZE = namespace.batchsize - sigma = io.DataPreparation.get_sigma(noise) - if namespace.generatedata: + config = parse_args() + size_df = config.get_item("data", "size_df", "DER") + noise = config.get_item("data", "noise_level", "DER") + norm = config.get_item("data", "normalize", "DER", raise_exception=False) + val_prop = config.get_item("data", "val_proportion", "DER") + rs = config.get_item("data", "randomseed", "DER") + BATCH_SIZE = config.get_item("data", "batchsize", "DER") + sigma = DataPreparation.get_sigma(noise) + print( + "generated data", + config.get_item("data", "generatedata", "DER", raise_exception=False), + ) + if config.get_item("data", "generatedata", "DER", raise_exception=False): # generate the df - data = io.DataPreparation() + data = DataPreparation() data.sample_params_from_prior(size_df) data.simulate_data(data.params, sigma, "linear_homogeneous") df_array = data.get_dict() @@ -171,12 +245,11 @@ def parse_args(): # Convert lists to tensors df[key] = torch.tensor(value) else: - loader = io.DataLoader() + loader = MyDataLoader() df = loader.load_data_h5( "linear_sigma_" + str(sigma) + "_size_" + str(size_df), path="/Users/rnevin/Documents/DeepUQ/data/", ) - print('df', df) len_df = len(df["params"][:, 0].numpy()) len_x = len(df["inputs"].numpy()) ms_array = np.repeat(df["params"][:, 0].numpy(), len_x) @@ -184,57 +257,51 @@ def parse_args(): xs_array = np.tile(df["inputs"].numpy(), len_df) ys_array = np.reshape(df["output"].numpy(), (len_df * len_x)) inputs = np.array([xs_array, ms_array, bs_array]).T - model_inputs, model_outputs = io.DataPreparation.normalize(inputs, - ys_array, - norm) - x_train, x_val, y_train, y_val = io.DataPreparation.train_val_split( + model_inputs, model_outputs = DataPreparation.normalize( + inputs, + ys_array, + norm) + x_train, x_val, y_train, y_val = DataPreparation.train_val_split( model_inputs, model_outputs, val_proportion=val_prop, random_state=rs ) trainData = TensorDataset(torch.Tensor(x_train), torch.Tensor(y_train)) - trainDataLoader = DataLoader(trainData, - batch_size=BATCH_SIZE, - shuffle=True) + trainDataLoader = DataLoader( + trainData, + batch_size=BATCH_SIZE, + shuffle=True) print("[INFO] initializing the gal model...") # set the device we will be using to train the model DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - model_name = namespace.model_type + "_noise_" + noise - model, lossFn = models.model_setup_DER(namespace.loss_type, DEVICE) + model_name = config.get_item( + "model", + "model_type", + "DER") + "_noise_" + noise + model, lossFn = models.model_setup_DER( + config.get_item("model", "loss_type", "DER"), DEVICE + ) model_ensemble = train.train_DER( trainDataLoader, x_val, y_val, - namespace.init_lr, + config.get_item("model", "init_lr", "DER"), DEVICE, - namespace.coeff, - namespace.loss_type, - namespace.wd, + config.get_item("model", "COEFF", "DER"), + config.get_item("model", "loss_type", "DER"), model_name, - EPOCHS=namespace.n_epochs, - path_to_model=namespace.path_to_models, - save_all_checkpoints=namespace.save_all_checkpoints, - save_final_checkpoint=namespace.save_final_checkpoint, - overwrite_final_checkpoint=namespace.overwrite_final_checkpoint, - plot=namespace.plot, - savefig=namespace.savefig, - verbose=namespace.verbose, - ) - ''' - trainDataLoader, - x_val, - y_val, - INIT_LR, - DEVICE, - COEFF, - loss_type, - wd, - model_name="DER", - EPOCHS=100, - path_to_model="models/", - save_all_checkpoints=False, - save_final_checkpoint=False, - overwrite_final_checkpoint=False, - plot=True, - savefig=True, - verbose=True - ''' + EPOCHS=config.get_item("model", "n_epochs", "DER"), + path_to_model=config.get_item("common", "out_dir", "DER"), + save_all_checkpoints=config.get_item( + "model", + "save_all_checkpoints", + "DER"), + save_final_checkpoint=config.get_item( + "model", + "save_final_checkpoint", + "DER"), + overwrite_final_checkpoint=config.get_item( + "model", "overwrite_final_checkpoint", "DER" + ), + plot=config.get_item("model", "plot", "DER"), + savefig=config.get_item("model", "savefig", "DER"), + verbose=config.get_item("model", "verbose", "DER"), + ) diff --git a/src/scripts/paths.py b/src/scripts/paths.py deleted file mode 100644 index 939c710..0000000 --- a/src/scripts/paths.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Exposes common paths useful for manipulating datasets and generating figures. - -""" -from pathlib import Path - -# Absolute path to the top level of the repository -root = Path(__file__).resolve().parents[2].absolute() - -# Absolute path to the `src` folder -src = root / "src" - -# Absolute path to the `src/data` folder (contains datasets) -data = src / "data" - -# Absolute path to the `src/static` folder (contains static images) -static = src / "static" - -# Absolute path to the `src/scripts` folder (contains figure/pipeline scripts) -scripts = src / "scripts" - -# Absolute path to the `src/tex` folder (contains the manuscript) -tex = src / "tex" - -# Absolute path to the `src/tex/figures` folder (contains figure output) -figures = tex / "figures" - -# Absolute path to the `src/tex/output` folder -# (contains other user-defined output) -output = tex / "output" diff --git a/src/static/.gitignore b/src/static/.gitignore deleted file mode 100644 index e167a34..0000000 --- a/src/static/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Anything is game in this folder -!* diff --git a/src/tex/.gitignore b/src/tex/.gitignore deleted file mode 100644 index 841b7c7..0000000 --- a/src/tex/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Don't track TeX temporaries -*latexindent* \ No newline at end of file diff --git a/src/tex/bib.bib b/src/tex/bib.bib deleted file mode 100644 index 5ca6881..0000000 --- a/src/tex/bib.bib +++ /dev/null @@ -1,37 +0,0 @@ -@article{Hunter:2007, - Author = {Hunter, J. D.}, - Title = {Matplotlib: A 2D graphics environment}, - Journal = {Computing in Science \& Engineering}, - Volume = {9}, - Number = {3}, - Pages = {90--95}, - abstract = {Matplotlib is a 2D graphics package used for Python for - application development, interactive scripting, and publication-quality - image generation across user interfaces and operating systems.}, - publisher = {IEEE COMPUTER SOC}, - doi = {10.1109/MCSE.2007.55}, - year = 2007 - } - -@article{ harris2020array, - title = {Array programming with {NumPy}}, - author = {Charles R. Harris and K. Jarrod Millman and St{\'{e}}fan J. - van der Walt and Ralf Gommers and Pauli Virtanen and David - Cournapeau and Eric Wieser and Julian Taylor and Sebastian - Berg and Nathaniel J. Smith and Robert Kern and Matti Picus - and Stephan Hoyer and Marten H. van Kerkwijk and Matthew - Brett and Allan Haldane and Jaime Fern{\'{a}}ndez del - R{\'{i}}o and Mark Wiebe and Pearu Peterson and Pierre - G{\'{e}}rard-Marchant and Kevin Sheppard and Tyler Reddy and - Warren Weckesser and Hameer Abbasi and Christoph Gohlke and - Travis E. Oliphant}, - year = {2020}, - month = sep, - journal = {Nature}, - volume = {585}, - number = {7825}, - pages = {357--362}, - doi = {10.1038/s41586-020-2649-2}, - publisher = {Springer Science and Business Media {LLC}}, - url = {https://doi.org/10.1038/s41586-020-2649-2} -} \ No newline at end of file diff --git a/src/tex/figures/.gitignore b/src/tex/figures/.gitignore deleted file mode 100644 index 9d0f65c..0000000 --- a/src/tex/figures/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Nothing should be tracked in this folder... -* - -# Except the gitignore file itself! -!.gitignore \ No newline at end of file diff --git a/src/tex/ms.tex b/src/tex/ms.tex deleted file mode 100644 index 93c6a58..0000000 --- a/src/tex/ms.tex +++ /dev/null @@ -1,144 +0,0 @@ - -\documentclass[twocolumn]{aastex631} - -% Import showyourwork magic -\usepackage{showyourwork} - -\usepackage[utf8]{inputenc} -\usepackage{amsmath} -\usepackage{unicode-math} - - -% Recommended, but optional, packages for figures and better typesetting: -\usepackage{microtype} -\usepackage{graphicx} -\usepackage{subfigure} -\usepackage{booktabs} % for professional tables -\usepackage{multirow} - -% hyperref makes hyperlinks in the resulting PDF. -% xurl can wrap the link if it spans a column (especially in citations). -\usepackage{hyperref} -\usepackage{xurl} - - -% This command creates a new command \editor{} that highlights any of the text in {} with a maroon color, so it can easily be spotted during internal review - -\usepackage[textsize=tiny]{todonotes} -\newcommand{\editor}[1]{{\color{purple} #1}} - -\begin{document} - -\title{DeepUQ} % Define the title itself, so it may be used in headers - -\author{Author 1 \thanks{Becky Nevin, rnevin@fnal.gov}} - - -\begin{abstract} - This project aims to create a framework for calibrating uncertainty expectations in various ML and statistical models. - It builds on DeepBench's pendulum module, controlling error injection, calculating an analytic expectation for the error impact on the final confidence intervals, and comparing this expectation to that produced by various uncertainty-aware ML and statistical techniques. - Here, we pursue the following modeling techniques: hierarchical and non-hierarchical Hamiltonian Monte Carlo sampling (\texttt{numpyro}), hierarchical and non-hierarchical simulation-based inference (\texttt{mackelab}), and deep ensembles. - We inject aleatoric error on the pendulum parameters individually ($L$, $\theta_0$, and $a_g$) at a variety of levels (1\%, 10\%, and 50\%). - We compare this to expectations of aleatoric and epistemic error from the various ML techniques and explore the bias and confidence of the models compared to our analytic expectation. -\end{abstract} - -\section{Introduction} -Cite other UQ techniques, mostly Caldeira \& Nord. - -\subsection{Error injection, (post-hoc) calibration, reliability} -Historically, aleatoric uncertainty is represented as $\epsilon$ in linear regression. -It is an additive uncertainty, not necessarily associated with a certain parameter. -This type of uncertainty can be homoskedastic or heteroskedastic. -If $\epsilon$ is a multiplicative term modifying an input parameter, the uncertainty is considered to be heteroskedastic, since it is tied to the parameter value. -In a linear regression setting, epistemic uncertainty is accounted for by the error on the $\beta$, or slope coefficient (\citealt{Nagl2022}). - -Many methods focus on creating software that will produce an uncertainty prediction. -However, a critical missed step is to calibrate this uncertainty prediction, testing its reliability against an expectation. -Several authors have investigated this, focusing on the calibration of classification problems, including Guao et al. 2017, Wegner et al. 2020, and Zhang et al. 2020. - -\begin{itemize} - \item Guo et al. 2017; this is the paper that presents temperature scaling as a method to quantify calibration of deep neural networks. - They also find that while neural networks today are more accurate than they were a decade ago, they are no longer well-calibrated, meaning that the confidence is substantially higher than accuracy. - So here, we are comparing confidence, which is a measurement of probabilities associate with the predicted labels, with the accuracy, both of which you get from the output. - So they are not propagating an error expecation to; they are instead comparing output to output. - \item Wegner et al. 2020 - \item Zhang et al. 2020. -\end{itemize} - -To read: -\begin{itemize} - \item Ghanem et al. 2017 is a handbook on UQ -\end{itemize} - -\section{Methods} -\subsection{Uncertainty definition and injection} -\subsection{Modeling techniques} -\subsubsection{HMC Sampling} -\subsubsection{SBI} -\subsubsection{DE} - -\section{Analysis} -\subsection{} - - -%\editor{Here is an quick comment that may appear, indicating an addition by an editor.} - - -%\subsubsection{Equations} - -%Large equations should be numbered and included in an equation block such that -%\begin{align} -% E=mc^2 \label{eq:1} \\ -% F=ma \label{eq:2} -%\end{align} - - -\section {Acknowledgements} - -Make sure to cite \cite{harris2020array} all of your sources \cite{Hunter:2007}. - - -You can also optionally provide contributions by person: - -\paragraph{Becky Nevin} -Author 1 contributed X Y and Z - -\paragraph{Author 2} -Author 2 contributed A B and C - -If you work with the DeepSkies research group; please include the following text: - -\emph{We acknowledge the Deep Skies Lab as a community of multi-domain experts and collaborators who’ve facilitated an environment of open discussion, idea-generation, and collaboration. This community was important for the development of this project.} - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% bibliography -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -% Style of the bib may change based on the publications requirements - -\bibliography{bib} - - - % Ending the multicol format before the appendix - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% APPENDIX -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\newpage -\appendix -\section{Appendix} - - -% Todo Example of running show your work function within the tex to produce table - -\subsection{Table References} - -% Todo: Show your work table drawing results from a function - -\end{document} \ No newline at end of file diff --git a/src/tex/output/.gitignore b/src/tex/output/.gitignore deleted file mode 100644 index 9d0f65c..0000000 --- a/src/tex/output/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Nothing should be tracked in this folder... -* - -# Except the gitignore file itself! -!.gitignore \ No newline at end of file diff --git a/src/tex/showyourwork.sty b/src/tex/showyourwork.sty deleted file mode 100644 index 8432d72..0000000 --- a/src/tex/showyourwork.sty +++ /dev/null @@ -1,13 +0,0 @@ -\NeedsTeXFormat{LaTeX2e} -\ProvidesPackage{showyourwork}[2022/01/12 Open source science articles] - -\IfFileExists{./showyourwork.tex}{ - \input{showyourwork.tex} -}{ - \newcommand\GitHubURL{} - \newcommand\GitHubSHA{} - \newcommand\GitHubIcon{} - \newcommand\showyourwork{} - \newcommand\script[1]{} - \newcommand\variable[1]{} -} \ No newline at end of file diff --git a/src/scripts/train.py b/src/train/train.py similarity index 95% rename from src/scripts/train.py rename to src/train/train.py index 1a6b2ce..5a89d97 100644 --- a/src/scripts/train.py +++ b/src/train/train.py @@ -1,10 +1,9 @@ -import argparse import torch import time import glob import numpy as np import matplotlib.pyplot as plt -from scripts import models +from models import models def train_DER( @@ -15,7 +14,6 @@ def train_DER( DEVICE, COEFF, loss_type, - wd, model_name="DER", EPOCHS=100, path_to_model="models/", @@ -31,7 +29,8 @@ def train_DER( # option to skip running the model if you don't care about # saving all checkpoints and only want to save the final final_chk = ( - path_to_model + str(path_to_model) + + "checkpoints/" + str(model_name) + "_loss_" + str(loss_type) @@ -226,8 +225,9 @@ def train_DER( if savefig: # ax1.errorbar(200, 600, yerr=5, # color='red', capsize=2) + print('path to model', path_to_model) plt.savefig( - str(wd) + str(path_to_model) + "images/animations/" + str(model_name) + "_loss_" @@ -254,8 +254,8 @@ def train_DER( "std_u_al_validation": std_u_al_val, "std_u_ep_validation": std_u_ep_val, }, - str(wd) - + "models/" + str(path_to_model) + + "checkpoints/" + str(model_name) + "_loss_" + str(loss_type) @@ -278,8 +278,8 @@ def train_DER( "std_u_al_validation": std_u_al_val, "std_u_ep_validation": std_u_ep_val, }, - str(wd) - + "models/" + str(path_to_model) + + "checkpoints/" + str(model_name) + "_loss_" + str(loss_type) @@ -302,7 +302,6 @@ def train_DE( DEVICE, loss_type, n_models, - wd, model_name="DE", BETA=None, EPOCHS=100, @@ -337,7 +336,8 @@ def train_DE( # saving all checkpoints and only want to save the final if loss_type == "bnll_loss": final_chk = ( - path_to_model + str(path_to_model) + + "checkpoints/" + str(model_name) + "_beta_" + str(BETA) @@ -349,7 +349,8 @@ def train_DE( ) else: final_chk = ( - path_to_model + str(path_to_model) + + "checkpoints/" + str(model_name) + "_nmodel_" + str(m) @@ -610,7 +611,7 @@ def train_DE( # ax1.errorbar(200, 600, yerr=5, # color='red', capsize=2) plt.savefig( - str(wd) + str(path_to_model) + "images/animations/" + str(model_name) + "_nmodel_" @@ -640,8 +641,8 @@ def train_DE( "x_val": x_val, "y_val": y_val, }, - str(wd) - + "models/" + str(path_to_model) + + 'checkpoints/' + str(model_name) + "_beta_" + str(BETA) @@ -665,8 +666,8 @@ def train_DE( "x_val": x_val, "y_val": y_val, }, - str(wd) - + "models/" + str(path_to_model) + + "checkpoints/" + str(model_name) + "_nmodel_" + str(m) @@ -690,8 +691,8 @@ def train_DE( "x_val": x_val, "y_val": y_val, }, - str(wd) - + "models/" + str(path_to_model) + + "checkpoints/" + str(model_name) + "_beta_" + str(BETA) @@ -715,8 +716,8 @@ def train_DE( "x_val": x_val, "y_val": y_val, }, - str(wd) - + "models/" + str(path_to_model) + + "checkpoints/" + str(model_name) + "_nmodel_" + str(m) @@ -734,21 +735,3 @@ def train_DE( print(endTime - startTime) return model_ensemble - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--data_source", - type=str, - help="Data used to train the model") - parser.add_argument( - "--n_epochs", - type=int, - help="Integer number of epochs to train the model" - ) - - args = parser.parse_args() - - # eventually change the bottom to train_model, - # which will contain train_DE and train_DER - train_DER(data_source=args.data_source, n_epochs=args.n_epochs) diff --git a/Snakefile b/src/utils/__init__.py similarity index 100% rename from Snakefile rename to src/utils/__init__.py diff --git a/src/utils/config.py b/src/utils/config.py new file mode 100644 index 0000000..fc45114 --- /dev/null +++ b/src/utils/config.py @@ -0,0 +1,73 @@ +from typing import Optional +import os +import yaml +from utils.defaults import DefaultsDE, DefaultsDER + + +def get_item(section, item, raise_exception=True): + return Config().get_item(section, item, raise_exception) + + +def get_section(section, raise_exception=True): + return Config().get_section(section, raise_exception) + + +class Config: + ENV_VAR_PATH = "DeepUQ_Config" + + def __init__(self, config_path: Optional[str] = None) -> None: + # okay what Maggie is doing here is a little trick or "cheat" + # where the config_path is saved to the ENV_VAR_PATH + # the first time this thing is called and then later it + # can be loaded from this temp location saving on memory + if config_path is not None: + # Add it to the env vars in case we need to get it later. + os.environ[self.ENV_VAR_PATH] = config_path + else: + # Get it from the env vars + try: + config_path = os.environ[self.ENV_VAR_PATH] + except KeyError: + assert False, \ + "Cannot load config from enviroment. \ + Hint: Have you set the config path \ + by passing a str path to Config?" + self.config = self._read_config(config_path) + self._validate_config() + + def _validate_config(self): + # Validate common + # TODO + pass + + def _read_config(self, path): + assert os.path.exists(path), f"Config path at {path} does not exist." + with open(path, 'r') as f: + config = yaml.safe_load(f) + return config + + # if raise_exception is True, then throws an error if we're missing + # otherwise, pull value from the defaults.py + def get_item(self, section, item, defaulttype, raise_exception=True): + try: + return self.config[section][item] + except KeyError as e: + if raise_exception: + raise KeyError(f"Configuration File missing parameter {e}") + else: + return { + "DER": DefaultsDER, + "DE": DefaultsDE + }[defaulttype][section][item] + + def get_section(self, section, defaulttype, raise_exception=True): + try: + return self.config[section] + except KeyError as e: + if raise_exception: + raise KeyError(e) + else: + return { + "DER": DefaultsDER, + "DE": DefaultsDE + }[defaulttype][section] diff --git a/src/utils/defaults.py b/src/utils/defaults.py new file mode 100644 index 0000000..a8135b9 --- /dev/null +++ b/src/utils/defaults.py @@ -0,0 +1,108 @@ +DefaultsDE = { + "common": { + "out_dir": "./DeepUQResources/results/", + "temp_config": "./DeepUQResources/temp/temp_config_DE.yml", + }, + "data": { + "data_path": "./data/", + "data_engine": "DataLoader", + "size_df": 1000, + "noise_level": "low", + "normalize": False, + "val_proportion": 0.1, + "randomseed": 42, + "batchsize": 100, + "generatedata": False, + }, + "model": { + "model_engine": "DE", + "model_type": "DE", + "loss_type": "bnll_loss", + "n_models": 100, + "init_lr": 0.001, + "BETA": 0.5, + "n_epochs": 100, + "save_all_checkpoints": False, + "save_final_checkpoint": False, + "overwrite_final_checkpoint": False, + "plot": False, + "savefig": False, + "verbose": False, + }, + "plots_common": { + "axis_spines": False, + "tight_layout": True, + "default_colorway": "viridis", + "plot_style": "fast", + "parameter_labels": ["$m$", "$b$"], + "parameter_colors": ["#9C92A3", "#0F5257"], + "line_style_cycle": ["-", "-."], + "figure_size": [6, 6], + }, + "plots": {"CDFRanks": {}, + "Ranks": {"num_bins": None}, + "CoverageFraction": {}}, + "metrics_common": { + "use_progress_bar": False, + "samples_per_inference": 1000, + "percentiles": [75, 85, 95], + }, + "metrics": { + "AllSBC": {}, + "CoverageFraction": {}, + }, +} +DefaultsDER = { + "common": { + "out_dir": "./DeepUQResources/results/", + "temp_config": "./DeepUQResources/temp/temp_config_DER.yml", + }, + "data": { + "data_path": "./data/", + "data_engine": "DataLoader", + "size_df": 1000, + "noise_level": "low", + "normalize": False, + "val_proportion": 0.1, + "randomseed": 42, + "batchsize": 100, + "generatedata": False, + }, + "model": { + # the engines are the classes, defined + "model_engine": "DER", + "model_type": "DER", + "loss_type": "SDER", + "init_lr": 0.001, + "COEFF": 0.5, + "n_epochs": 100, + "save_all_checkpoints": False, + "save_final_checkpoint": False, + "overwrite_final_checkpoint": False, + "plot": False, + "savefig": False, + "verbose": False, + }, + "plots_common": { + "axis_spines": False, + "tight_layout": True, + "default_colorway": "viridis", + "plot_style": "fast", + "parameter_labels": ["$m$", "$b$"], + "parameter_colors": ["#9C92A3", "#0F5257"], + "line_style_cycle": ["-", "-."], + "figure_size": [6, 6], + }, + "plots": {"CDFRanks": {}, + "Ranks": {"num_bins": None}, + "CoverageFraction": {}}, + "metrics_common": { + "use_progress_bar": False, + "samples_per_inference": 1000, + "percentiles": [75, 85, 95], + }, + "metrics": { + "AllSBC": {}, + "CoverageFraction": {}, + }, +} diff --git a/test/test_DeepEnsemble.py b/test/test_DeepEnsemble.py index 2334cb8..af224b1 100644 --- a/test/test_DeepEnsemble.py +++ b/test/test_DeepEnsemble.py @@ -3,6 +3,7 @@ import subprocess import tempfile import shutil +import yaml @pytest.fixture @@ -11,7 +12,10 @@ def temp_directory(): temp_dir = tempfile.mkdtemp() # Create subdirectories within the temporary directory - models_dir = os.path.join(temp_dir, "models") + yaml_dir = os.path.join(temp_dir, "yamls") + os.makedirs(yaml_dir) + + models_dir = os.path.join(temp_dir, "checkpoints") os.makedirs(models_dir) animations_dir = os.path.join(temp_dir, "images", "animations") @@ -28,39 +32,71 @@ def temp_directory(): shutil.rmtree(temp_dir) -def test_DER_chkpt_saved(temp_directory): - noise_level = "low" - wd = str(temp_directory) + "/" +def create_test_config(temp_directory, n_epochs): + print("dumping temp yaml") + print("temp_dir", temp_directory) + input_yaml = { + "common": {"out_dir": str(temp_directory)}, + "model": { + "model_engine": "DE", + "model_type": "DE", + "loss_type": "bnll_loss", + "init_lr": 0.001, + "BETA": 0.5, + "n_models": 2, + "n_epochs": n_epochs, + "save_all_checkpoints": False, + "save_final_checkpoint": True, + "overwrite_final_checkpoint": True, + "plot": False, + "savefig": True, + "verbose": False, + }, + "data": { + "data_path": "./data", + "data_engine": "DataLoader", + "size_df": 1000, + "noise_level": "low", + "val_proportion": 0.1, + "randomseed": 42, + "batchsize": 100, + }, + } + print("theoretically dumping here", str(temp_directory) + "yamls/DE.yaml") + yaml.dump(input_yaml, open(str(temp_directory) + "yamls/DE.yaml", "w")) + + +def test_DE_from_config(temp_directory): + # create the test config dynamically + # make the temporary config file n_epochs = 2 + n_models = 2 + create_test_config(temp_directory + "/", n_epochs) subprocess_args = [ "python", - "src/scripts/DeepEvidentialRegression.py", - noise_level, - wd, - "--n_epochs", - str(n_epochs), - "--save_final_checkpoint", - "--savefig", - "--generatedata" + "src/scripts/DeepEnsemble.py", + "--config", + str(temp_directory) + "/yamls/DE.yaml", ] # now run the subprocess subprocess.run(subprocess_args, check=True) # check if the right number of checkpoints are saved - models_folder = os.path.join(temp_directory, "models") + models_folder = os.path.join(temp_directory, "checkpoints") + print("this is the checkpoints folder", models_folder) # list all files in the "models" folder files_in_models_folder = os.listdir(models_folder) + print("files in checkpoints folder", files_in_models_folder) # assert that the number of files is equal to 10 assert ( - len(files_in_models_folder) == 1 - ), "Expected 1 file in the 'models' folder" + len(files_in_models_folder) == n_models + ), f"Expected {n_models} file in the 'checkpoints' folder" # check if the right number of images were saved animations_folder = os.path.join(temp_directory, "images/animations") files_in_animations_folder = os.listdir(animations_folder) - # assert that the number of files is equal to 10 assert ( - len(files_in_animations_folder) == 1 - ), "Expected 1 file in the 'images/animations' folder" + len(files_in_animations_folder) == n_models + ), f"Expected {n_models} file in the 'images/animations' folder" # also check that all files in here have the same name elements expected_substring = "epoch_" + str(n_epochs - 1) @@ -78,31 +114,33 @@ def test_DER_chkpt_saved(temp_directory): def test_DE_chkpt_saved(temp_directory): noise_level = "low" - n_models = 10 - wd = str(temp_directory) + "/" + n_models = 2 n_epochs = 2 subprocess_args = [ "python", "src/scripts/DeepEnsemble.py", + "--noise_level", noise_level, + "--n_models", str(n_models), - wd, + "--out_dir", + str(temp_directory) + "/", "--n_epochs", str(n_epochs), "--save_final_checkpoint", "--savefig", - "--generatedata" + "--generatedata", ] # now run the subprocess subprocess.run(subprocess_args, check=True) # check if the right number of checkpoints are saved - models_folder = os.path.join(temp_directory, "models") + models_folder = os.path.join(temp_directory, "checkpoints") # list all files in the "models" folder files_in_models_folder = os.listdir(models_folder) # assert that the number of files is equal to 10 assert ( len(files_in_models_folder) == n_models - ), "Expected 10 files in the 'models' folder" + ), f"Expected {n_models} files in the 'checkpoints' folder" # check if the right number of images were saved animations_folder = os.path.join(temp_directory, "images/animations") @@ -110,7 +148,7 @@ def test_DE_chkpt_saved(temp_directory): # assert that the number of files is equal to 10 assert ( len(files_in_animations_folder) == n_models - ), "Expected 10 files in the 'images/animations' folder" + ), f"Expected {n_models} files in the 'images/animations' folder" # also check that all files in here have the same name elements expected_substring = "epoch_" + str(n_epochs - 1) @@ -129,18 +167,20 @@ def test_DE_chkpt_saved(temp_directory): @pytest.mark.xfail(strict=True) def test_DE_no_chkpt_saved_xfail(temp_directory): noise_level = "low" - n_models = 10 - wd = str(temp_directory) + "/" + n_models = 2 n_epochs = 2 subprocess_args = [ "python", "src/scripts/DeepEnsemble.py", + "--noise_level", noise_level, + "--n_models", str(n_models), - wd, + "--out_dir", + str(temp_directory) + "/", "--n_epochs", str(n_epochs), - "--generatedata" + "--generatedata", ] # now run the subprocess subprocess.run(subprocess_args, check=True) @@ -151,28 +191,30 @@ def test_DE_no_chkpt_saved_xfail(temp_directory): # assert that the number of files is equal to 10 assert ( len(files_in_models_folder) == n_models - ), "Expected 10 files in the 'models' folder" + ), f"Expected {n_models} files in the 'models' folder" def test_DE_no_chkpt_saved(temp_directory): noise_level = "low" - n_models = 10 - wd = str(temp_directory) + "/" + n_models = 2 n_epochs = 2 subprocess_args = [ "python", "src/scripts/DeepEnsemble.py", + "--noise_level", noise_level, + "--n_models", str(n_models), - wd, + "--out_dir", + str(temp_directory) + "/", "--n_epochs", str(n_epochs), - "--generatedata" + "--generatedata", ] # now run the subprocess subprocess.run(subprocess_args, check=True) # check if the right number of checkpoints are saved - models_folder = os.path.join(temp_directory, "models") + models_folder = os.path.join(temp_directory, "checkpoints") # list all files in the "models" folder files_in_models_folder = os.listdir(models_folder) # assert that the number of files is equal to 10 @@ -182,37 +224,19 @@ def test_DE_no_chkpt_saved(temp_directory): def test_DE_run_simple_ensemble(temp_directory): noise_level = "low" - n_models = "10" - # here = os.getcwd() - # wd = self.temp_path - # os.path.dirname(here) + str(temp_directory) + '/' - wd = str(temp_directory) + "/" + n_models = 2 subprocess_args = [ "python", "src/scripts/DeepEnsemble.py", + "--noise_level", noise_level, - n_models, - wd, - "--n_epochs", - "2", - "--generatedata" - ] - # now run the subprocess - subprocess.run(subprocess_args, check=True) - - -@pytest.mark.xfail(strict=True) -def test_DE_missing_req_arg(temp_directory): - noise_level = "low" - n_models = "10" - subprocess_args = [ - "python", - "src/scripts/DeepEnsemble.py", - noise_level, - n_models, + "--n_models", + str(n_models), + "--out_dir", + str(temp_directory) + "/", "--n_epochs", "2", - "--generatedata" + "--generatedata", ] # now run the subprocess subprocess.run(subprocess_args, check=True) diff --git a/test/test_DeepEvidentialRegression.py b/test/test_DeepEvidentialRegression.py new file mode 100644 index 0000000..5a17619 --- /dev/null +++ b/test/test_DeepEvidentialRegression.py @@ -0,0 +1,154 @@ +import pytest +import os +import subprocess +import tempfile +import shutil +import yaml + + +@pytest.fixture +def temp_directory(): + # Setup: Create a temporary directory with one folder level + temp_dir = tempfile.mkdtemp() + + # Create subdirectories within the temporary directory + yaml_dir = os.path.join(temp_dir, "yamls") + os.makedirs(yaml_dir) + + models_dir = os.path.join(temp_dir, "checkpoints") + os.makedirs(models_dir) + + animations_dir = os.path.join(temp_dir, "images", "animations") + os.makedirs(animations_dir) + + yield temp_dir # Provide the temporary directory path to the test function + + # Teardown: Remove the temporary directory and its contents + """ + for dir_path in [models_dir, animations_dir, temp_dir]: + os.rmdir(dir_path) + # Teardown: Remove the temporary directory and its contents + """ + shutil.rmtree(temp_dir) + + +def create_test_config(temp_directory, n_epochs): + print("dumping temp yaml") + print("temp_dir", temp_directory) + input_yaml = { + "common": {"out_dir": str(temp_directory)}, # +"results/"}, + "model": { + "model_engine": "DER", + "model_type": "DER", + "loss_type": "DER", + "init_lr": 0.001, + "COEFF": 0.5, + "n_epochs": n_epochs, + "save_all_checkpoints": False, + "save_final_checkpoint": True, + "overwrite_final_checkpoint": True, + "plot": False, + "savefig": True, + "verbose": False, + }, + "data": { + "data_path": "./data", + "data_engine": "DataLoader", + "size_df": 1000, + "noise_level": "low", + "val_proportion": 0.1, + "randomseed": 42, + "batchsize": 100, + }, + } + print("theoretically dumping here", str(temp_directory) + "yamls/DER.yaml") + yaml.dump(input_yaml, open(str(temp_directory) + "yamls/DER.yaml", "w")) + + +def test_DER_chkpt_saved(temp_directory): + noise_level = "low" + n_epochs = 2 + subprocess_args = [ + "python", + "src/scripts/DeepEvidentialRegression.py", + "--noise_level", + noise_level, + "--out_dir", + str(temp_directory) + "/", + "--n_epochs", + str(n_epochs), + "--save_final_checkpoint", + "--savefig", + "--generatedata", + ] + # now run the subprocess + subprocess.run(subprocess_args, check=True) + # check if the right number of checkpoints are saved + models_folder = os.path.join(temp_directory, "checkpoints") + # list all files in the "models" folder + files_in_models_folder = os.listdir(models_folder) + # assert that the number of files is equal to 10 + assert len(files_in_models_folder) == 1, \ + "Expected 1 file in the 'models' folder" + + # check if the right number of images were saved + animations_folder = os.path.join(temp_directory, "images/animations") + files_in_animations_folder = os.listdir(animations_folder) + # assert that the number of files is equal to 10 + assert ( + len(files_in_animations_folder) == 1 + ), "Expected 1 file in the 'images/animations' folder" + + # also check that all files in here have the same name elements + expected_substring = "epoch_" + str(n_epochs - 1) + for file_name in files_in_models_folder: + assert ( + expected_substring in file_name + ), f"File '{file_name}' does not contain the expected substring" + + # also check that all files in here have the same name elements + for file_name in files_in_animations_folder: + assert ( + expected_substring in file_name + ), f"File '{file_name}' does not contain the expected substring" + + +def test_DER_from_config(temp_directory): + # create the test config dynamically + # make the temporary config file + n_epochs = 2 + create_test_config(temp_directory + "/", n_epochs) + subprocess_args = [ + "python", + "src/scripts/DeepEvidentialRegression.py", + "--config", + str(temp_directory) + "/yamls/DER.yaml", + ] + # now run the subprocess + subprocess.run(subprocess_args, check=True) + # check if the right number of checkpoints are saved + models_folder = os.path.join(temp_directory, "checkpoints") + # list all files in the "models" folder + files_in_models_folder = os.listdir(models_folder) + # assert that the number of files is equal to 10 + assert len(files_in_models_folder) == 1, \ + "Expected 1 file in the 'models' folder" + # check if the right number of images were saved + animations_folder = os.path.join(temp_directory, "images/animations") + files_in_animations_folder = os.listdir(animations_folder) + # assert that the number of files is equal to 10 + assert ( + len(files_in_animations_folder) == 1 + ), "Expected 1 file in the 'images/animations' folder" + # also check that all files in here have the same name elements + expected_substring = "epoch_" + str(n_epochs - 1) + for file_name in files_in_models_folder: + assert ( + expected_substring in file_name + ), f"File '{file_name}' does not contain the expected substring" + + # also check that all files in here have the same name elements + for file_name in files_in_animations_folder: + assert ( + expected_substring in file_name + ), f"File '{file_name}' does not contain the expected substring"