From 800dd70125a0803ee18f41ad3539d36c87e4eef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Winfried=20L=C3=B6tzsch?= Date: Thu, 1 Sep 2022 12:59:52 +0200 Subject: [PATCH] Make this repo installable with all python versions (#53) * Remove torchvision from requirements.in * Add torchvision to dev.in * Remove numpy pin * Remove python version pin * Bump version * Update notebooks and squirrel_plugin (torchvision) * Catch import error for squirrel plugin * Strip notebook metadata --- examples/01.Getting_Started.ipynb | 16 +------- examples/02.Catalog.ipynb | 16 +------- examples/03.Pytorch_Model_Training.ipynb | 16 +------- .../04.PytorchLightning_Model_Training.ipynb | 8 +--- examples/05.Plugins.ipynb | 16 +------- examples/06.SquirrelStore_with_Spark.ipynb | 16 +------- examples/07.Performance_Guideline.ipynb | 14 +------ examples/08.Spark_Preprocessing.ipynb | 16 +------- requirements.dev.in | 2 +- requirements.in | 3 +- setup.py | 1 - src/squirrel_datasets_core/__init__.py | 2 +- src/squirrel_datasets_core/squirrel_plugin.py | 40 +++++++++++++++++-- 13 files changed, 54 insertions(+), 112 deletions(-) diff --git a/examples/01.Getting_Started.ipynb b/examples/01.Getting_Started.ipynb index f4c5868..ce8dc6d 100644 --- a/examples/01.Getting_Started.ipynb +++ b/examples/01.Getting_Started.ipynb @@ -25,7 +25,7 @@ "\n", " restart_colab = False\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core matplotlib # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" matplotlib # noqa\n", " import matplotlib\n", " import squirrel\n", " import squirrel_datasets_core\n", @@ -145,19 +145,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 4 } diff --git a/examples/02.Catalog.ipynb b/examples/02.Catalog.ipynb index 7f91301..d213cad 100644 --- a/examples/02.Catalog.ipynb +++ b/examples/02.Catalog.ipynb @@ -31,7 +31,7 @@ " import matplotlib\n", " import squirrel\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-core matplotlib # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" matplotlib # noqa\n", " import matplotlib\n", " import squirrel\n", "\n", @@ -258,19 +258,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/03.Pytorch_Model_Training.ipynb b/examples/03.Pytorch_Model_Training.ipynb index e996ada..33f75ae 100644 --- a/examples/03.Pytorch_Model_Training.ipynb +++ b/examples/03.Pytorch_Model_Training.ipynb @@ -31,7 +31,7 @@ " import squirrel\n", " import squirrel_datasets_core\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" # noqa\n", " import squirrel\n", " import squirrel_datasets_core\n", "\n", @@ -352,19 +352,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/04.PytorchLightning_Model_Training.ipynb b/examples/04.PytorchLightning_Model_Training.ipynb index e3a00e5..57bffd3 100644 --- a/examples/04.PytorchLightning_Model_Training.ipynb +++ b/examples/04.PytorchLightning_Model_Training.ipynb @@ -32,7 +32,7 @@ " import squirrel\n", " import squirrel_datasets_core\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pytorch_lightning # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pytorch_lightning # noqa\n", " import pytorch_lightning as pl\n", " import squirrel\n", " import squirrel_datasets_core\n", @@ -149,11 +149,7 @@ ] } ], - "metadata": { - "language_info": { - "name": "python" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 0 } diff --git a/examples/05.Plugins.ipynb b/examples/05.Plugins.ipynb index 45912ff..dc25e1c 100644 --- a/examples/05.Plugins.ipynb +++ b/examples/05.Plugins.ipynb @@ -23,7 +23,7 @@ " import networkx\n", " import squirrel\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-core networkx # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" networkx # noqa\n", " import networkx\n", " import squirrel\n", "\n", @@ -356,19 +356,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/06.SquirrelStore_with_Spark.ipynb b/examples/06.SquirrelStore_with_Spark.ipynb index c4e563d..8a4ae12 100644 --- a/examples/06.SquirrelStore_with_Spark.ipynb +++ b/examples/06.SquirrelStore_with_Spark.ipynb @@ -20,7 +20,7 @@ " import squirrel_datasets_core\n", " import pyspark\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pyspark # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pyspark # noqa\n", " import squirrel\n", " import squirrel_datasets_core\n", " import pyspark\n", @@ -310,19 +310,7 @@ "source": [] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/07.Performance_Guideline.ipynb b/examples/07.Performance_Guideline.ipynb index 7182a4a..512a7f1 100644 --- a/examples/07.Performance_Guideline.ipynb +++ b/examples/07.Performance_Guideline.ipynb @@ -356,19 +356,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/examples/08.Spark_Preprocessing.ipynb b/examples/08.Spark_Preprocessing.ipynb index 1876da8..45ec447 100644 --- a/examples/08.Spark_Preprocessing.ipynb +++ b/examples/08.Spark_Preprocessing.ipynb @@ -21,7 +21,7 @@ " import numpy as np\n", " import matplotlib.pyplot as plt\n", "except:\n", - " !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core numpy matplotlib # noqa\n", + " !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" numpy matplotlib # noqa\n", " import squirrel\n", " import squirrel_datasets_core\n", " import matplotlib.pyplot as plt\n", @@ -131,19 +131,7 @@ ] } ], - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 2 } diff --git a/requirements.dev.in b/requirements.dev.in index c139912..d368947 100644 --- a/requirements.dev.in +++ b/requirements.dev.in @@ -15,4 +15,4 @@ notebook>=6.4.12 jupyter networkx pre-commit==2.16.0 -pip-tools \ No newline at end of file +pip-tools diff --git a/requirements.in b/requirements.in index 6806522..d1b8d9c 100644 --- a/requirements.in +++ b/requirements.in @@ -2,8 +2,7 @@ datasets docutils fire hub -numpy>=1.22 +numpy pillow scipy # required by torch dataset "caltech" squirrel-core[gcp, zarr]>=0.14.2 -torchvision diff --git a/setup.py b/setup.py index 5da8d81..087a895 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,6 @@ def parse_req(spec: str) -> str: setup( name="squirrel_datasets_core", version=version, - python_requires=">=3.8.0", description="Squirrel public datasets collection", long_description=long_description, long_description_content_type="text/markdown", diff --git a/src/squirrel_datasets_core/__init__.py b/src/squirrel_datasets_core/__init__.py index c11f861..569b121 100644 --- a/src/squirrel_datasets_core/__init__.py +++ b/src/squirrel_datasets_core/__init__.py @@ -1 +1 @@ -__version__ = "0.1.9" +__version__ = "0.1.10" diff --git a/src/squirrel_datasets_core/squirrel_plugin.py b/src/squirrel_datasets_core/squirrel_plugin.py index ec2ce43..4bca4c4 100644 --- a/src/squirrel_datasets_core/squirrel_plugin.py +++ b/src/squirrel_datasets_core/squirrel_plugin.py @@ -7,21 +7,53 @@ from squirrel.framework.plugins.hookimpl import hookimpl +def get_hub_driver() -> Driver: + """Imports and returns the hub driver class""" + from squirrel_datasets_core.driver.hub import HubDriver + + return HubDriver + + +def get_huggingface_driver() -> Driver: + """Imports and returns the huggingface driver class""" + from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver + + return HuggingfaceDriver + + +def get_torchvision_driver() -> Driver: + """Imports and returns the torchvision driver class""" + from squirrel_datasets_core.driver.torchvision import TorchvisionDriver + + return TorchvisionDriver + + @hookimpl def squirrel_drivers() -> List[Type[Driver]]: """Custom drivers added by this package.""" import squirrel_datasets_core.datasets as ds - from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver - from squirrel_datasets_core.driver.torchvision import TorchvisionDriver - from squirrel_datasets_core.driver.hub import HubDriver - drivers = [TorchvisionDriver, HuggingfaceDriver, HubDriver] + drivers = [] + add_drivers = { + "hub": get_hub_driver, + "huggingface": get_huggingface_driver, + "torchvision": get_torchvision_driver, + } + + for d in add_drivers: + try: + drivers.append(add_drivers[d]()) + except ImportError as e: + print(f"Failed to import {d} driver with error: {e}") + for m in pkgutil.iter_modules(ds.__path__): try: d = importlib.import_module(f"{ds.__package__}.{m.name}").DRIVERS drivers += d except AttributeError: pass + except ImportError as e: + print(f"Failed to import module {m.name} with error: {e}") return drivers