Merge pull request #40 from pastas/dev

Major refactor

pastastore/store.py

@@ -1,7 +1,7 @@
 import json
 import os
 import warnings
-from typing import Optional, Tuple, Union
+from typing import List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -129,7 +129,8 @@ def get_nearest_oseries(self, names: Optional[Union[list, str]] = None,
 
     def get_distances(self, oseries: Optional[Union[list, str]] = None,
                       stresses: Optional[Union[list, str]] = None,
-                      kind: Optional[str] = None) -> FrameorSeriesUnion:
+                      kind: Optional[Union[str, List[str]]] = None) \
+            -> FrameorSeriesUnion:
         """Method to obtain the distances in meters between the oseries and
         stresses.
 

pastastore/util.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Optional
+from typing import Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -12,6 +12,10 @@ def _custom_warning(message, category=UserWarning, filename='', lineno=-1,
     print(f"{filename}:{lineno}: {category.__name__}: {message}")
 
 
+class ItemInLibraryException(Exception):
+    pass
+
+
 def delete_pystore_connector(path: Optional[str] = None,
                              name: Optional[str] = None,
                              conn=None,
@@ -83,7 +87,7 @@ def delete_arctic_connector(connstr: Optional[str] = None,
         name = conn.name
         connstr = conn.connstr
     elif name is None or connstr is None:
-        raise ValueError("Please provide 'name' and 'connstr' OR 'conn'!")
+        raise ValueError("Provide 'name' and 'connstr' OR 'conn'!")
 
     arc = arctic.Arctic(connstr)
 
@@ -94,8 +98,10 @@ def delete_arctic_connector(connstr: Optional[str] = None,
         for ilib in arc.list_libraries():
             if ilib.split(".")[0] == name:
                 libs.append(ilib)
-    else:
+    elif name is not None:
         libs = [name + "." + ilib for ilib in libraries]
+    else:
+        raise ValueError("Provide 'name' and 'connstr' OR 'conn'!")
 
     for lib in libs:
         arc.delete_library(lib)
@@ -203,7 +209,8 @@ def empty_library(pstore, libname: str,
 
 def validate_names(s: Optional[str] = None, d: Optional[dict] = None,
                    replace_space: Optional[str] = "_",
-                   deletechars: Optional[str] = None, **kwargs) -> str:
+                   deletechars: Optional[str] = None, **kwargs) \
+        -> Union[str, Dict]:
     """Remove invalid characters from string or dictionary keys.
 
     Parameters
@@ -237,7 +244,7 @@ def validate_names(s: Optional[str] = None, d: Optional[dict] = None,
             new_dict[validator(k)[0]] = v
         return new_dict
     else:
-        raise ValueError("Provide one of 's' or 'd'!")
+        raise ValueError("Provide one of 's' (string) or 'd' (dict)!")
 
 
 def compare_models(ml1, ml2, stats=None, detailed_comparison=False):
@@ -353,3 +360,50 @@ def compare_models(ml1, ml2, stats=None, detailed_comparison=False):
         return df
     else:
         return df["comparison"].all()
+
+
+def copy_database(conn1, conn2, libraries: Optional[List[str]] = None,
+                  overwrite: bool = False, progressbar: bool = False) -> None:
+    """Copy libraries from one database to another.
+
+    Parameters
+    ----------
+    conn1 : pastastore.*Connector
+        source Connector containing link to current database containing data
+    conn2 : pastastore.*Connector
+        destination Connector with link to database to which you want to copy
+    libraries : Optional[List[str]], optional
+        list of str containing names of libraries to copy, by default None,
+        which copies all libraries: ['oseries', 'stresses', 'models']
+    overwrite : bool, optional
+        overwrite data in destination database, by default False
+    progressbar : bool, optional
+        show progressbars, by default False
+
+    Raises
+    ------
+    ValueError
+        if library name is not understood
+    """
+    if libraries is None:
+        libraries = ["oseries", "stresses", "models"]
+
+    for lib in libraries:
+        if lib == "oseries":
+            for name in (tqdm(conn1.oseries_names, desc="copying oseries") if
+                         progressbar else conn1.oseries_names):
+                o, meta = conn1.get_oseries(name, return_metadata=True)
+                conn2.add_oseries(o, name, metadata=meta, overwrite=overwrite)
+        elif lib == "stresses":
+            for name in (tqdm(conn1.stresses_names, desc="copying oseries") if
+                         progressbar else conn1.stresses_names):
+                s, meta = conn1.get_stresses(name, return_metadata=True)
+                conn2.add_stress(s, name, kind=meta["kind"], metadata=meta,
+                                 overwrite=overwrite)
+        elif lib == "models":
+            for name in (tqdm(conn1.model_names, desc="copying oseries") if
+                         progressbar else conn1.model_names):
+                mldict = conn1.get_models(name, return_dict=True)
+                conn2.add_model(mldict, overwrite=overwrite)
+        else:
+            raise ValueError(f"Library name '{lib}' not recognized!")

pastastore/version.py

@@ -1 +1 @@
-__version__ = "0.5.0"
+__version__ = "0.6.0"

tests/conftest.py

@@ -1,114 +1,102 @@
-import pytest
-import pystore
-import arctic
 import pandas as pd
-
 import pastastore as pst
-
+import pystore
+import pytest
 
 params = ["arctic", "pystore", "dict", "pas"]
-# params = ["pas"]
+# params = ["dict"]
 
 
 def initialize_project(conn):
 
-    prj = pst.PastaStore("test_project", conn)
+    pstore = pst.PastaStore("test_project", conn)
 
     # oseries 1
     o = pd.read_csv("./tests/data/obs.csv", index_col=0, parse_dates=True)
-    prj.add_oseries(o, "oseries1", metadata={"x": 100000,
-                                             "y": 400000})
+    pstore.add_oseries(o, "oseries1", metadata={"x": 100000,
+                                                "y": 400000})
     # oseries 2
     o = pd.read_csv("./tests/data/head_nb1.csv", index_col=0, parse_dates=True)
-    prj.add_oseries(o, "oseries2", metadata={"x": 100300,
-                                             "y": 400400})
+    pstore.add_oseries(o, "oseries2", metadata={"x": 100300,
+                                                "y": 400400})
 
     # oseries 3
     o = pd.read_csv("./tests/data/gw_obs.csv", index_col=0, parse_dates=True)
-    prj.add_oseries(o, "oseries3", metadata={"x": 165554,
-                                             "y": 422685})
+    pstore.add_oseries(o, "oseries3", metadata={"x": 165554,
+                                                "y": 422685})
 
     # prec 1
     s = pd.read_csv("./tests/data/rain.csv", index_col=0, parse_dates=True)
-    prj.add_stress(s, "prec1", kind="prec", metadata={"x": 100000,
-                                                      "y": 400000})
+    pstore.add_stress(s, "prec1", kind="prec", metadata={"x": 100000,
+                                                         "y": 400000})
 
     # prec 2
     s = pd.read_csv("./tests/data/rain_nb1.csv", index_col=0, parse_dates=True)
-    prj.add_stress(s, "prec2", kind="prec", metadata={"x": 100300,
-                                                      "y": 400400})
+    pstore.add_stress(s, "prec2", kind="prec", metadata={"x": 100300,
+                                                         "y": 400400})
 
     # evap 1
     s = pd.read_csv("./tests/data/evap.csv", index_col=0, parse_dates=True)
-    prj.add_stress(s, "evap1", kind="evap", metadata={"x": 100000,
-                                                      "y": 400000})
+    pstore.add_stress(s, "evap1", kind="evap", metadata={"x": 100000,
+                                                         "y": 400000})
 
     # evap 2
     s = pd.read_csv("./tests/data/evap_nb1.csv", index_col=0, parse_dates=True)
-    prj.add_stress(s, "evap2", kind="evap", metadata={"x": 100300,
-                                                      "y": 400400})
+    pstore.add_stress(s, "evap2", kind="evap", metadata={"x": 100300,
+                                                         "y": 400400})
 
     # well 1
     s = pd.read_csv("./tests/data/well.csv", index_col=0, parse_dates=True)
-    prj.add_stress(s, "well1", kind="well", metadata={"x": 164691,
-                                                      "y": 423579})
+    pstore.add_stress(s, "well1", kind="well", metadata={"x": 164691,
+                                                         "y": 423579})
 
-    return prj
+    return pstore
 
 
 @pytest.fixture(scope="module", params=params)
-def pr(request):
+def conn(request):
     """Fixture that yields connection object.
     """
     name = f"test_{request.param}"
     # connect to dbase
     if request.param == "arctic":
         connstr = "mongodb://localhost:27017/"
-        pr = pst.ArcticConnector(name, connstr)
+        conn = pst.ArcticConnector(name, connstr)
     elif request.param == "pystore":
         path = "./tests/data/pystore"
-        pr = pst.PystoreConnector(name, path)
+        conn = pst.PystoreConnector(name, path)
     elif request.param == "dict":
-        pr = pst.DictConnector(name)
+        conn = pst.DictConnector(name)
     elif request.param == "pas":
-        pr = pst.PasConnector(name, "./tests/data/pas")
+        conn = pst.PasConnector(name, "./tests/data/pas")
     else:
         raise ValueError("Unrecognized parameter!")
-    pr.type = request.param  # added here for defining test dependencies
-    yield pr
+    conn.type = request.param  # added here for defining test dependencies
+    yield conn
 
 
 @pytest.fixture(scope="module", params=params)
-def prj(request):
+def pstore(request):
     if request.param == "arctic":
         connstr = "mongodb://localhost:27017/"
         name = "test_project"
-        arc = arctic.Arctic(connstr)
-        if name in [lib.split(".")[0] for lib in arc.list_libraries()]:
-            connector = pst.ArcticConnector(name, connstr)
-            prj = pst.PastaStore(name, connector)
-        else:
-            connector = pst.ArcticConnector(name, connstr)
-            prj = initialize_project(connector)
+        connector = pst.ArcticConnector(name, connstr)
+        pstore = initialize_project(connector)
     elif request.param == "pystore":
         name = "test_project"
         path = "./tests/data/pystore"
         pystore.set_path(path)
-        if name in pystore.list_stores():
-            connector = pst.PystoreConnector(name, path)
-            prj = pst.PastaStore(name, connector)
-        else:
-            connector = pst.PystoreConnector(name, path)
-            prj = initialize_project(connector)
+        connector = pst.PystoreConnector(name, path)
+        pstore = initialize_project(connector)
     elif request.param == "dict":
         name = "test_project"
         connector = pst.DictConnector(name)
-        prj = initialize_project(connector)
+        pstore = initialize_project(connector)
     elif request.param == "pas":
         name = "test_project"
         connector = pst.PasConnector(name, "./tests/data/pas")
-        prj = initialize_project(connector)
+        pstore = initialize_project(connector)
     else:
         raise ValueError("Unrecognized parameter!")
-    prj.type = request.param  # added here for defining test dependencies
-    yield prj
+    pstore.type = request.param  # added here for defining test dependencies
+    yield pstore