Make this repo installable with all python versions (#53)

* Remove torchvision from requirements.in

* Add torchvision to dev.in

* Remove numpy pin

* Remove python version pin

* Bump version

* Update notebooks and squirrel_plugin (torchvision)

* Catch import error for squirrel plugin

* Strip notebook metadata

examples/01.Getting_Started.ipynb

@@ -25,7 +25,7 @@
     "\n",
     "    restart_colab = False\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core matplotlib # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" matplotlib # noqa\n",
     "    import matplotlib\n",
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
@@ -145,19 +145,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 4
 }

examples/02.Catalog.ipynb

@@ -31,7 +31,7 @@
     "    import matplotlib\n",
     "    import squirrel\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-core matplotlib # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" matplotlib # noqa\n",
     "    import matplotlib\n",
     "    import squirrel\n",
     "\n",
@@ -258,19 +258,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/03.Pytorch_Model_Training.ipynb

@@ -31,7 +31,7 @@
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" # noqa\n",
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
     "\n",
@@ -352,19 +352,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/04.PytorchLightning_Model_Training.ipynb

@@ -32,7 +32,7 @@
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pytorch_lightning # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pytorch_lightning # noqa\n",
     "    import pytorch_lightning as pl\n",
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
@@ -149,11 +149,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

examples/05.Plugins.ipynb

@@ -23,7 +23,7 @@
     "    import networkx\n",
     "    import squirrel\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-core networkx # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" networkx # noqa\n",
     "    import networkx\n",
     "    import squirrel\n",
     "\n",
@@ -356,19 +356,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/06.SquirrelStore_with_Spark.ipynb

@@ -20,7 +20,7 @@
     "    import squirrel_datasets_core\n",
     "    import pyspark\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pyspark # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pyspark # noqa\n",
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
     "    import pyspark\n",
@@ -310,19 +310,7 @@
    "source": []
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/07.Performance_Guideline.ipynb

@@ -356,19 +356,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 5
 }

examples/08.Spark_Preprocessing.ipynb

@@ -21,7 +21,7 @@
     "    import numpy as np\n",
     "    import matplotlib.pyplot as plt\n",
     "except:\n",
-    "    !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core numpy matplotlib # noqa\n",
+    "    !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" numpy matplotlib # noqa\n",
     "    import squirrel\n",
     "    import squirrel_datasets_core\n",
     "    import matplotlib.pyplot as plt\n",
@@ -131,19 +131,7 @@
    ]
   }
  ],
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
-  }
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 2
 }

requirements.dev.in

@@ -15,4 +15,4 @@ notebook>=6.4.12
 jupyter
 networkx
 pre-commit==2.16.0
-pip-tools
+pip-tools

requirements.in

@@ -2,8 +2,7 @@ datasets
 docutils
 fire
 hub
-numpy>=1.22
+numpy
 pillow
 scipy # required by torch dataset "caltech"
 squirrel-core[gcp, zarr]>=0.14.2
-torchvision

setup.py

@@ -121,7 +121,6 @@ def parse_req(spec: str) -> str:
     setup(
         name="squirrel_datasets_core",
         version=version,
-        python_requires=">=3.8.0",
         description="Squirrel public datasets collection",
         long_description=long_description,
         long_description_content_type="text/markdown",

src/squirrel_datasets_core/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.9"
+__version__ = "0.1.10"

src/squirrel_datasets_core/squirrel_plugin.py

@@ -7,21 +7,53 @@
 from squirrel.framework.plugins.hookimpl import hookimpl
 
 
+def get_hub_driver() -> Driver:
+    """Imports and returns the hub driver class"""
+    from squirrel_datasets_core.driver.hub import HubDriver
+
+    return HubDriver
+
+
+def get_huggingface_driver() -> Driver:
+    """Imports and returns the huggingface driver class"""
+    from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver
+
+    return HuggingfaceDriver
+
+
+def get_torchvision_driver() -> Driver:
+    """Imports and returns the torchvision driver class"""
+    from squirrel_datasets_core.driver.torchvision import TorchvisionDriver
+
+    return TorchvisionDriver
+
+
 @hookimpl
 def squirrel_drivers() -> List[Type[Driver]]:
     """Custom drivers added by this package."""
     import squirrel_datasets_core.datasets as ds
-    from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver
-    from squirrel_datasets_core.driver.torchvision import TorchvisionDriver
-    from squirrel_datasets_core.driver.hub import HubDriver
 
-    drivers = [TorchvisionDriver, HuggingfaceDriver, HubDriver]
+    drivers = []
+    add_drivers = {
+        "hub": get_hub_driver,
+        "huggingface": get_huggingface_driver,
+        "torchvision": get_torchvision_driver,
+    }
+
+    for d in add_drivers:
+        try:
+            drivers.append(add_drivers[d]())
+        except ImportError as e:
+            print(f"Failed to import {d} driver with error: {e}")
+
     for m in pkgutil.iter_modules(ds.__path__):
         try:
             d = importlib.import_module(f"{ds.__package__}.{m.name}").DRIVERS
             drivers += d
         except AttributeError:
             pass
+        except ImportError as e:
+            print(f"Failed to import module {m.name} with error: {e}")
 
     return drivers