Skip to content
This repository has been archived by the owner on Dec 5, 2024. It is now read-only.

Make this repo installable with all python versions #53

Merged
merged 8 commits into from
Sep 1, 2022
Merged

Make this repo installable with all python versions #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cbe971e
Remove torchvision from requirements.in
Aug 31, 2022
d16fdd5
Add torchvision to dev.in
Aug 31, 2022
509ccf9
Remove numpy pin
Aug 31, 2022
765bde1
Remove python version pin
Aug 31, 2022
76c917f
Bump version
Aug 31, 2022
c5daefd
Update notebooks and squirrel_plugin (torchvision)
Aug 31, 2022
aa945d1
Catch import error for squirrel plugin
Sep 1, 2022
ed8ca85
Strip notebook metadata
Sep 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions examples/01.Getting_Started.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"\n",
" restart_colab = False\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core matplotlib # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" matplotlib # noqa\n",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity: Are quotes necessary?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It works both ways

" import matplotlib\n",
" import squirrel\n",
" import squirrel_datasets_core\n",
Expand Down Expand Up @@ -145,19 +145,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}
16 changes: 2 additions & 14 deletions examples/02.Catalog.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
" import matplotlib\n",
" import squirrel\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-core matplotlib # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" matplotlib # noqa\n",
" import matplotlib\n",
" import squirrel\n",
"\n",
Expand Down Expand Up @@ -258,19 +258,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
16 changes: 2 additions & 14 deletions examples/03.Pytorch_Model_Training.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
" import squirrel\n",
" import squirrel_datasets_core\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" # noqa\n",
" import squirrel\n",
" import squirrel_datasets_core\n",
"\n",
Expand Down Expand Up @@ -352,19 +352,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
8 changes: 2 additions & 6 deletions examples/04.PytorchLightning_Model_Training.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
" import squirrel\n",
" import squirrel_datasets_core\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pytorch_lightning # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pytorch_lightning # noqa\n",
" import pytorch_lightning as pl\n",
" import squirrel\n",
" import squirrel_datasets_core\n",
Expand Down Expand Up @@ -149,11 +149,7 @@
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 0
}
16 changes: 2 additions & 14 deletions examples/05.Plugins.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
" import networkx\n",
" import squirrel\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-core networkx # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-core[torchvision]\" networkx # noqa\n",
" import networkx\n",
" import squirrel\n",
"\n",
Expand Down Expand Up @@ -356,19 +356,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
16 changes: 2 additions & 14 deletions examples/06.SquirrelStore_with_Spark.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
" import squirrel_datasets_core\n",
" import pyspark\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core pyspark # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" pyspark # noqa\n",
" import squirrel\n",
" import squirrel_datasets_core\n",
" import pyspark\n",
Expand Down Expand Up @@ -310,19 +310,7 @@
"source": []
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
14 changes: 1 addition & 13 deletions examples/07.Performance_Guideline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,19 +356,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
16 changes: 2 additions & 14 deletions examples/08.Spark_Preprocessing.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
" import numpy as np\n",
" import matplotlib.pyplot as plt\n",
"except:\n",
" !pip install -q --ignore-requires-python --upgrade squirrel-datasets-core numpy matplotlib # noqa\n",
" !pip install -q --ignore-requires-python --upgrade \"squirrel-datasets-core[torchvision]\" numpy matplotlib # noqa\n",
" import squirrel\n",
" import squirrel_datasets_core\n",
" import matplotlib.pyplot as plt\n",
Expand Down Expand Up @@ -131,19 +131,7 @@
]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3"
}
},
"metadata": {},
"nbformat": 4,
"nbformat_minor": 2
}
2 changes: 1 addition & 1 deletion requirements.dev.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ notebook>=6.4.12
jupyter
networkx
pre-commit==2.16.0
pip-tools
pip-tools
3 changes: 1 addition & 2 deletions requirements.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ datasets
docutils
fire
hub
numpy>=1.22
numpy
pillow
scipy # required by torch dataset "caltech"
squirrel-core[gcp, zarr]>=0.14.2
torchvision
1 change: 0 additions & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@ def parse_req(spec: str) -> str:
setup(
name="squirrel_datasets_core",
version=version,
python_requires=">=3.8.0",
description="Squirrel public datasets collection",
long_description=long_description,
long_description_content_type="text/markdown",
Expand Down
2 changes: 1 addition & 1 deletion src/squirrel_datasets_core/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.1.9"
__version__ = "0.1.10"
40 changes: 36 additions & 4 deletions src/squirrel_datasets_core/squirrel_plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,53 @@
from squirrel.framework.plugins.hookimpl import hookimpl


def get_hub_driver() -> Driver:
"""Imports and returns the hub driver class"""
from squirrel_datasets_core.driver.hub import HubDriver

return HubDriver


def get_huggingface_driver() -> Driver:
"""Imports and returns the huggingface driver class"""
from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver

return HuggingfaceDriver


def get_torchvision_driver() -> Driver:
"""Imports and returns the torchvision driver class"""
from squirrel_datasets_core.driver.torchvision import TorchvisionDriver

return TorchvisionDriver


@hookimpl
def squirrel_drivers() -> List[Type[Driver]]:
"""Custom drivers added by this package."""
import squirrel_datasets_core.datasets as ds
from squirrel_datasets_core.driver.huggingface import HuggingfaceDriver
from squirrel_datasets_core.driver.torchvision import TorchvisionDriver
from squirrel_datasets_core.driver.hub import HubDriver

drivers = [TorchvisionDriver, HuggingfaceDriver, HubDriver]
drivers = []
add_drivers = {
"hub": get_hub_driver,
"huggingface": get_huggingface_driver,
"torchvision": get_torchvision_driver,
}

for d in add_drivers:
try:
drivers.append(add_drivers[d]())
except ImportError as e:
print(f"Failed to import {d} driver with error: {e}")

for m in pkgutil.iter_modules(ds.__path__):
try:
d = importlib.import_module(f"{ds.__package__}.{m.name}").DRIVERS
drivers += d
except AttributeError:
pass
except ImportError as e:
print(f"Failed to import module {m.name} with error: {e}")

return drivers

Expand Down