diff --git a/.github/workflows/cd_release.yml b/.github/workflows/cd_release.yml index fd9eccde..e8cded9b 100644 --- a/.github/workflows/cd_release.yml +++ b/.github/workflows/cd_release.yml @@ -19,12 +19,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 @@ -47,9 +47,9 @@ jobs: - name: Update version and tag run: | - invoke setver --ver="${GITHUB_REF#refs/tags/}" + invoke setver --version="${GITHUB_REF#refs/tags/}" - git add oteapi/__init__.py CHANGELOG.md + git add oteapi_dlite/__init__.py CHANGELOG.md git commit -m "Release ${GITHUB_REF#refs/tags/} - Changelog" TAG_MSG=.github/utils/release_tag_msg.txt @@ -101,13 +101,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ env.PUBLISH_UPDATE_BRANCH }} - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 diff --git a/.github/workflows/ci_automerge_dependabot.yml b/.github/workflows/ci_automerge_dependabot.yml index 9767c244..4c857fc8 100644 --- a/.github/workflows/ci_automerge_dependabot.yml +++ b/.github/workflows/ci_automerge_dependabot.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/ci_cd_updated_master.yml b/.github/workflows/ci_cd_updated_master.yml index 242b37ac..a5dfccd0 100644 --- a/.github/workflows/ci_cd_updated_master.yml +++ b/.github/workflows/ci_cd_updated_master.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ env.DEPENDABOT_BRANCH }} fetch-depth: 0 @@ -78,13 +78,13 @@ jobs: - name: Checkout repository if: env.RELEASE_RUN == 'false' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python 3.9 if: env.RELEASE_RUN == 'false' - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index d290c694..c06f6915 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -19,12 +19,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: master - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 @@ -87,7 +87,7 @@ jobs: - name: Create PR id: cpr - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_PAT }} commit-message: New @dependabot-fueled updates diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index ede94a73..14c7059a 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -14,10 +14,10 @@ jobs: # timeout-minutes: 2 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 @@ -47,12 +47,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 2 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 @@ -85,12 +85,12 @@ jobs: python-version: ["3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 2 - name: Set up Python ${{ matrix.python-version}} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version}} @@ -120,12 +120,12 @@ jobs: python-version: ["3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 2 - name: Set up Python ${{ matrix.python-version}} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version}} @@ -151,9 +151,9 @@ jobs: # timeout-minutes: 5 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 @@ -170,11 +170,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6ee7ad5..5bd88a98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,19 +25,19 @@ repos: args: ["--profile", "black", "--filter-files", "--skip-gitignore"] - repo: https://github.com/ambv/black - rev: 21.12b0 + rev: 22.1.0 hooks: - id: black - repo: https://github.com/PyCQA/bandit - rev: 1.7.2 + rev: 1.7.4 hooks: - id: bandit args: ["-r"] files: ^oteapi_dlite/.*$ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.931 + rev: v0.941 hooks: - id: mypy exclude: ^tests/.*$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd98f8f..f31df191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,85 @@ # Changelog -This file will hold an auto-generated changelog for you project. +## [Unreleased](https://github.com/EMMC-ASBL/oteapi-dlite/tree/HEAD) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.7...HEAD) + +**Fixed bugs:** + +- Tests are failing when using the `oteapi-core`@master [\#32](https://github.com/EMMC-ASBL/oteapi-dlite/issues/32) +- Fix Excel parser [\#8](https://github.com/EMMC-ASBL/oteapi-dlite/issues/8) + +## [v0.0.7](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.7) (2022-03-17) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.6...v0.0.7) + +**Merged pull requests:** + +- Fix tests + excel parser [\#33](https://github.com/EMMC-ASBL/oteapi-dlite/pull/33) ([CasperWA](https://github.com/CasperWA)) + +## [v0.0.6](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.6) (2022-03-16) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.5...v0.0.6) + +**Fixed bugs:** + +- Documentation issue [\#25](https://github.com/EMMC-ASBL/oteapi-dlite/issues/25) + +**Merged pull requests:** + +- Fix setup.cfg [\#31](https://github.com/EMMC-ASBL/oteapi-dlite/pull/31) ([jesper-friis](https://github.com/jesper-friis)) +- \[Auto-generated\] Update dependencies [\#30](https://github.com/EMMC-ASBL/oteapi-dlite/pull/30) ([TEAM4-0](https://github.com/TEAM4-0)) +- Corrected syntax in setup.cfg [\#28](https://github.com/EMMC-ASBL/oteapi-dlite/pull/28) ([jesper-friis](https://github.com/jesper-friis)) + +## [v0.0.5](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.5) (2022-03-11) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.4...v0.0.5) + +**Merged pull requests:** + +- Updated strategy names [\#27](https://github.com/EMMC-ASBL/oteapi-dlite/pull/27) ([jesper-friis](https://github.com/jesper-friis)) +- \[Auto-generated\] Update dependencies [\#26](https://github.com/EMMC-ASBL/oteapi-dlite/pull/26) ([TEAM4-0](https://github.com/TEAM4-0)) +- Updated dlite image parse strategy according to changes in the image parse strategy in oteapi core PR \#108 [\#18](https://github.com/EMMC-ASBL/oteapi-dlite/pull/18) ([jesper-friis](https://github.com/jesper-friis)) + +## [v0.0.4](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.4) (2022-03-07) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.3...v0.0.4) + +**Fixed bugs:** + +- Release workflow failing [\#22](https://github.com/EMMC-ASBL/oteapi-dlite/issues/22) + +**Merged pull requests:** + +- Fix folder reference in workflow [\#24](https://github.com/EMMC-ASBL/oteapi-dlite/pull/24) ([CasperWA](https://github.com/CasperWA)) +- Fix option name for `invoke setver` [\#23](https://github.com/EMMC-ASBL/oteapi-dlite/pull/23) ([CasperWA](https://github.com/CasperWA)) + +## [v0.0.3](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.3) (2022-03-05) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/v0.0.2...v0.0.3) + +**Merged pull requests:** + +- Add serialise filter [\#17](https://github.com/EMMC-ASBL/oteapi-dlite/pull/17) ([jesper-friis](https://github.com/jesper-friis)) +- Added a simple filter that adds a collection to the session [\#15](https://github.com/EMMC-ASBL/oteapi-dlite/pull/15) ([jesper-friis](https://github.com/jesper-friis)) + +## [v0.0.2](https://github.com/EMMC-ASBL/oteapi-dlite/tree/v0.0.2) (2022-03-05) + +[Full Changelog](https://github.com/EMMC-ASBL/oteapi-dlite/compare/04843680b62d6bb3bc16e5ff644edbcc40892f31...v0.0.2) + +**Closed issues:** + +- Add CI/CD workflow\(s\) [\#3](https://github.com/EMMC-ASBL/oteapi-dlite/issues/3) + +**Merged pull requests:** + +- Relaxed dependencies [\#16](https://github.com/EMMC-ASBL/oteapi-dlite/pull/16) ([jesper-friis](https://github.com/jesper-friis)) +- fix ci cd workflow [\#14](https://github.com/EMMC-ASBL/oteapi-dlite/pull/14) ([jesper-friis](https://github.com/jesper-friis)) +- Add MANIFEST.in [\#9](https://github.com/EMMC-ASBL/oteapi-dlite/pull/9) ([CasperWA](https://github.com/CasperWA)) +- Image parser [\#7](https://github.com/EMMC-ASBL/oteapi-dlite/pull/7) ([TorgeirUstad](https://github.com/TorgeirUstad)) +- Add CI/CD [\#4](https://github.com/EMMC-ASBL/oteapi-dlite/pull/4) ([jesper-friis](https://github.com/jesper-friis)) +- Parse excel [\#1](https://github.com/EMMC-ASBL/oteapi-dlite/pull/1) ([jesper-friis](https://github.com/jesper-friis)) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/README.md b/README.md index 73a8d99f..82374bfd 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,8 @@ OTEAPI DLite Plugin has been created via the [cookiecutter](https://cookiecutter OTEAPI DLite Plugin has been supported by the following projects: -- **OntoTrans** (2020-2024) that receives funding from the European Union’s Horizon 2020 Research and Innovation Programme, under Grant Agreement n. 862136. +* __OntoTrans__ (2020-2024) that receives funding from the European Union’s Horizon 2020 Research and Innovation Programme, under Grant Agreement no. 862136. + +* __VIPCOAT__ (2021-2025) receives funding from the European Union’s Horizon 2020 Research and Innovation Programme - DT-NMBP-11-2020 Open Innovation Platform for Materials Modelling, under Grant Agreement no: 952903. + +* __OpenModel__ (2021-2025) receives funding from the European Union’s Horizon 2020 Research and Innovation Programme - DT-NMBP-11-2020 Open Innovation Platform for Materials Modelling, under Grant Agreement no: 953167. diff --git a/docs/index.md b/docs/index.md index ff61693f..d6795697 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,4 +31,8 @@ OTEAPI DLite Plugin has been created via the [cookiecutter](https://cookiecutter OTEAPI DLite Plugin has been supported by the following projects: -- **OntoTrans** (2020-2024) that receives funding from the European Union’s Horizon 2020 Research and Innovation Programme, under Grant Agreement n. 862136. +* __OntoTrans__ (2020-2024) that receives funding from the European Union’s Horizon 2020 Research and Innovation Programme, under Grant Agreement no. 862136. + +* __VIPCOAT__ (2021-2025) receives funding from the European Union’s Horizon 2020 Research and Innovation Programme - DT-NMBP-11-2020 Open Innovation Platform for Materials Modelling, under Grant Agreement no: 952903. + +* __OpenModel__ (2021-2025) receives funding from the European Union’s Horizon 2020 Research and Innovation Programme - DT-NMBP-11-2020 Open Innovation Platform for Materials Modelling, under Grant Agreement no: 953167. diff --git a/oteapi_dlite/__init__.py b/oteapi_dlite/__init__.py index d28cd111..29f0dff3 100644 --- a/oteapi_dlite/__init__.py +++ b/oteapi_dlite/__init__.py @@ -6,6 +6,6 @@ Created from cookiecutter-oteapi-plugin, SINTEF, 2022 """ -__version__ = "0.0.3" +__version__ = "0.0.7" __author__ = "team4.0@sintef.no" __author_email__ = "team4.0@sintef.no" diff --git a/oteapi_dlite/strategies/filter.py b/oteapi_dlite/strategies/filter.py index 6a6a929d..a7cb6318 100644 --- a/oteapi_dlite/strategies/filter.py +++ b/oteapi_dlite/strategies/filter.py @@ -1,16 +1,16 @@ """Trivial filter that adds an empty collection to the session.""" # pylint: disable=no-self-use,unused-argument -from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict import dlite +from oteapi.models import FilterConfig +from pydantic import Field +from pydantic.dataclasses import dataclass from oteapi_dlite.models import DLiteSessionUpdate if TYPE_CHECKING: - from typing import Any, Dict, Optional - - from oteapi.models import FilterConfig + from typing import Optional @dataclass @@ -19,11 +19,18 @@ class CreateCollectionStrategy: **Registers strategies**: - - `("filterType", "create_collection")` + - `("filterType", "dlite/create-collection")` """ - filter_config: "FilterConfig" + filter_config: FilterConfig + + # Find a better way to keep collections alive!!! + # Need to be `Any`, because otherwise `pydantic` complains. + collection_refs: Dict[str, Any] = Field( + {}, + description="A dictionary of DLite Collections.", + ) def initialize( self, session: "Optional[Dict[str, Any]]" = None @@ -37,7 +44,7 @@ def initialize( # Save reference to the collection to ensure that it lives as long as # the session does - session["_collection_ref"] = coll + self.collection_refs[coll.uuid] = coll return DLiteSessionUpdate(collection_id=coll.uuid) diff --git a/oteapi_dlite/strategies/parse_excel.py b/oteapi_dlite/strategies/parse_excel.py index 38dc573d..722bf62a 100644 --- a/oteapi_dlite/strategies/parse_excel.py +++ b/oteapi_dlite/strategies/parse_excel.py @@ -1,7 +1,6 @@ """Strategy for parsing an Excel spreadsheet to a DLite instance.""" # pylint: disable=no-self-use,unused-argument import re -from dataclasses import dataclass from random import getrandbits from typing import TYPE_CHECKING, Optional @@ -11,6 +10,7 @@ from oteapi.models import AttrDict, ResourceConfig, SessionUpdate from oteapi.strategies.parse.excel_xlsx import XLSXParseConfig, XLSXParseStrategy from pydantic import Field, HttpUrl +from pydantic.dataclasses import dataclass from oteapi_dlite.models import DLiteSessionUpdate from oteapi_dlite.utils import dict2recarray @@ -76,7 +76,7 @@ class DLiteExcelStrategy: """ - parse_config: "ResourceConfig" + parse_config: DLiteExcelParseResourceConfig def initialize(self, session: "Optional[Dict[str, Any]]" = None) -> SessionUpdate: """Initialize.""" @@ -100,11 +100,15 @@ def get(self, session: "Optional[Dict[str, Any]]" = None) -> SessionUpdate: if session is None: raise ValueError("Missing session") - config = DLiteExcelParseConfig(**self.parse_config.configuration) + config = self.parse_config.configuration - xlsx_session = self.parse_config.copy() - xlsx_session.configuration = config.excel_config - parser: "IParseStrategy" = XLSXParseStrategy(xlsx_session) + xlsx_config = self.parse_config.dict() + xlsx_config["configuration"] = config.excel_config + xlsx_config[ + "mediaType" + ] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + print(xlsx_config) + parser: "IParseStrategy" = XLSXParseStrategy(xlsx_config) columns = parser.get(session)["data"] names, units = zip(*[split_column_name(column) for column in columns]) diff --git a/oteapi_dlite/strategies/parse_image.py b/oteapi_dlite/strategies/parse_image.py index a0becf6b..4ae1b006 100644 --- a/oteapi_dlite/strategies/parse_image.py +++ b/oteapi_dlite/strategies/parse_image.py @@ -1,15 +1,18 @@ """Strategy class for parsing an image to a DLite instance.""" -# pylint: disable=no-self-use,unused-argument -from dataclasses import dataclass -from io import BytesIO -from typing import TYPE_CHECKING, Optional, Tuple +# pylint: disable=no-self-use +import logging +from typing import TYPE_CHECKING import dlite -import numpy as np -from oteapi.datacache.datacache import DataCache -from oteapi.strategies.parse.image import ImageDataParseStrategy -from PIL import Image -from pydantic import BaseModel, Field +from oteapi.datacache import DataCache +from oteapi.models import ResourceConfig +from oteapi.strategies.parse.image import ( + ImageDataParseStrategy, + ImageParserConfig, + ImageParserResourceConfig, +) +from pydantic import Extra, Field +from pydantic.dataclasses import dataclass from oteapi_dlite.models import DLiteSessionUpdate from oteapi_dlite.utils import get_meta @@ -17,38 +20,45 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any, Dict - from dlite import Instance - from oteapi.models.resourceconfig import ResourceConfig +LOGGER = logging.getLogger("oteapi_dlite.strategies") +LOGGER.setLevel(logging.DEBUG) -class DLiteImageConfig(BaseModel): + +class DLiteImageConfig(ImageParserConfig): """Configuration for DLite image parser.""" - crop: Optional[Tuple] = Field( - None, description="Cropping rectangle. The whole image if None." - ) image_label: str = Field( "image", description="Label to assign to the image in the collection.", ) +class DLiteImageResourceConfig(ResourceConfig): + """Resource config for DLite image parser.""" + + configuration: DLiteImageConfig = Field( + DLiteImageConfig(), + description="Image parse strategy-specific configuration.", + ) + + @dataclass class DLiteImageParseStrategy: """Parse strategy for image files. **Registers strategies**: - - `("mediaType", "image/gif")` - - `("mediaType", "image/jpeg")` - - `("mediaType", "image/jpg")` - - `("mediaType", "image/jp2")` - - `("mediaType", "image/png")` - - `("mediaType", "image/tiff")` + - `("mediaType", "image/vnd.dlite-gif")` + - `("mediaType", "image/vnd.dlite-jpeg")` + - `("mediaType", "image/vnd.dlite-jpg")` + - `("mediaType", "image/vnd.dlite-jp2")` + - `("mediaType", "image/vnd.dlite-png")` + - `("mediaType", "image/vnd.dlite-tiff")` """ - parse_config: "ResourceConfig" + parse_config: DLiteImageResourceConfig def initialize(self, session: "Dict[str, Any]" = None) -> DLiteSessionUpdate: """Initialize.""" @@ -70,41 +80,34 @@ def get(self, session: "Dict[str, Any]" = None) -> DLiteSessionUpdate: Returns: DLite instance. - """ if session is None: raise ValueError("Missing session") - if "key" in session: - key = session["key"] - elif "key" in self.parse_config.configuration: - key = self.parse_config.configuration["key"] - else: - raise RuntimeError("Image parser needs an image to parse") - - image_config = DLiteImageConfig(**self.parse_config.configuration) - - with DataCache().getfile( - key, suffix=self.parse_config.mediaType.split("/")[1] - ) as tmp_file: - if image_config.crop: - tmp_config = self.parse_config.copy() - tmp_config.configuration["filename"] = tmp_file.name - tmp_config.configuration["localpath"] = tmp_file.parent - image = Image.open( - BytesIO(ImageDataParseStrategy(tmp_config).get().content) - ) - else: - image = Image.open(tmp_file).copy() - - data = np.asarray(image) - if np.ndim(data) == 2: - data.shape = (data.shape[0], data.shape[1], 1) + config = self.parse_config.configuration + + # Configuration for ImageDataParseStrategy in oteapi-core + conf = self.parse_config.dict() + conf["configuration"] = ImageParserConfig( + **config.dict(), + extra=Extra.ignore, + ) + conf["mediaType"] = "image/" + conf["mediaType"].split("-")[-1] + core_config = ImageParserResourceConfig(**conf) + + ImageDataParseStrategy(core_config).initialize(session) + output = ImageDataParseStrategy(core_config).get(session) + + cache = DataCache() + data = cache.get(output["image_key"]) + meta = get_meta("http://onto-ns.com/meta/1.0/Image") - inst = meta(dims=[image.height, image.width, len(image.getbands())]) + inst = meta(dims=data.shape) inst["data"] = data + LOGGER.info("session: %s", session) + coll = dlite.get_collection(session["collection_id"]) - coll.add(image_config.image_label, inst) + coll.add(config.image_label, inst) return DLiteSessionUpdate(collection_id=coll.uuid) diff --git a/oteapi_dlite/strategies/serialise.py b/oteapi_dlite/strategies/serialise.py index af112d9f..fc53fa19 100644 --- a/oteapi_dlite/strategies/serialise.py +++ b/oteapi_dlite/strategies/serialise.py @@ -1,12 +1,12 @@ """Filter for serialisation using DLite.""" # pylint: disable=no-self-use -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Optional, Sequence import dlite from oteapi.models import AttrDict, FilterConfig from pydantic import Field +from pydantic.dataclasses import dataclass from oteapi_dlite.models import DLiteSessionUpdate @@ -57,7 +57,7 @@ class SerialiseStrategy: """ - filter_config: "SerialiseFilterConfig" + filter_config: SerialiseFilterConfig def initialize( self, session: "Optional[Dict[str, Any]]" = None diff --git a/requirements.txt b/requirements.txt index 17a2f98e..7e40bb1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ DLite-Python>=0.3.3,<1 -numpy>=1.22.2 +numpy>=1.21,<2 oteapi-core>=0.1.2 Pillow>=9.0.1,<10 diff --git a/requirements_dev.txt b/requirements_dev.txt index c70753b7..8e8e2e2b 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ pre-commit~=2.17 pylint~=2.12 -pytest~=6.2 +pytest~=7.1 pytest-cov~=3.0 diff --git a/requirements_docs.txt b/requirements_docs.txt index a2078166..9632040e 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -1,6 +1,6 @@ -invoke~=1.6 +invoke~=1.7 mike~=1.1 mkdocs~=1.2 -mkdocs-awesome-pages-plugin~=2.6 -mkdocs-material~=8.1 -mkdocstrings[python]~=0.18.0 +mkdocs-awesome-pages-plugin~=2.7 +mkdocs-material~=8.2 +mkdocstrings[python]~=0.18.1 diff --git a/setup.cfg b/setup.cfg index 63875e72..17f0e888 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,11 @@ [options.entry_points] -oteapi.parse_strategy = - oteapi_dlite.image/gif = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.image/jpeg = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.image/jpg = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.image/jp2 = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.image/png = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.image/tiff = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy - oteapi_dlite.application/vnd.openxmlformats-officedocument.spreadsheetml.sheet = oteapi_dlite.strategies.parse_excel:DLiteExcelStrategy +oteapi.parse = + oteapi_dlite.image/vnd.dlite-gif = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.image/vnd.dlite-jpeg = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.image/vnd.dlite-jpg = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.image/vnd.dlite-jp2 = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.image/vnd.dlite-png = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.image/vnd.dlite-tiff = oteapi_dlite.strategies.parse_image:DLiteImageParseStrategy + oteapi_dlite.application/vnd.dlite-xlsx = oteapi_dlite.strategies.parse_excel:DLiteExcelStrategy +oteapi.filter = + oteapi_dlite.dlite/create-collection = oteapi_dlite.strategies.filter:CreateCollectionStrategy diff --git a/tests/.gitignore b/tests/.gitignore deleted file mode 100644 index a9a5aecf..00000000 --- a/tests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp diff --git a/tests/conftest.py b/tests/conftest.py index def979ed..481a634c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,11 +27,3 @@ def repo_dir() -> "Path": def static_files(repo_dir: "Path") -> "Path": """Absolute path to the static directory filled with test files.""" return repo_dir / "tests" / "static" - - -@pytest.fixture(scope="session") -def tmp_dir(repo_dir: "Path") -> "Path": - """Absolute path to directory output files created by the tests.""" - tmpdir = repo_dir / "tests" / "tmp" - tmpdir.mkdir(exist_ok=True) - return tmpdir diff --git a/tests/strategies/test_filter.py b/tests/strategies/test_filter.py index 8dee696b..c85277e3 100644 --- a/tests/strategies/test_filter.py +++ b/tests/strategies/test_filter.py @@ -1,18 +1,21 @@ """Tests filter strategies.""" +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from oteapi.interfaces import IFilterStrategy -def test_create_collection(): + +def test_create_collection() -> None: """Test the create_collection filter.""" import dlite - from oteapi.models.filterconfig import FilterConfig from oteapi_dlite.strategies.filter import CreateCollectionStrategy - config = FilterConfig(filterType="create_collection") + config = {"filterType": "dlite/create_collection"} session = {} - collfilter = CreateCollectionStrategy(config) + collfilter: "IFilterStrategy" = CreateCollectionStrategy(config) session.update(collfilter.initialize(session)) assert "collection_id" in session diff --git a/tests/strategies/test_parse_excel.py b/tests/strategies/test_parse_excel.py index f6a576d0..3777a9e1 100644 --- a/tests/strategies/test_parse_excel.py +++ b/tests/strategies/test_parse_excel.py @@ -4,28 +4,29 @@ if TYPE_CHECKING: from pathlib import Path + from oteapi.interfaces import IParseStrategy + def test_parse_excel(static_files: "Path") -> None: """Test excel parse strategy.""" import dlite import numpy as np - from oteapi.models import ResourceConfig from oteapi_dlite.strategies.parse_excel import DLiteExcelStrategy sample_file = static_files / "test_parse_excel.xlsx" - config = ResourceConfig( - downloadUrl=sample_file.as_uri(), - mediaType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - configuration={ + config = { + "downloadUrl": sample_file.as_uri(), + "mediaType": "application/vnd.dlite-xlsx", + "configuration": { "excel_config": { "worksheet": "Sheet1", "header_row": "1", "row_from": "2", }, }, - ) + } coll = dlite.Collection() session = {"collection_id": coll.uuid} @@ -34,7 +35,7 @@ def test_parse_excel(static_files: "Path") -> None: session.update(parser.initialize(session)) # Note that initialize() and get() are called on different parser instances... - parser = DLiteExcelStrategy(config) + parser: "IParseStrategy" = DLiteExcelStrategy(config) parser.get(session) inst = coll.get("excel-data") diff --git a/tests/strategies/test_parse_image.py b/tests/strategies/test_parse_image.py index e532be48..75f8d756 100644 --- a/tests/strategies/test_parse_image.py +++ b/tests/strategies/test_parse_image.py @@ -8,6 +8,10 @@ from pathlib import Path from typing import Optional, Tuple + from oteapi.interfaces import IParseStrategy + + from oteapi_dlite.models import DLiteSessionUpdate + def test_image_config() -> None: """Test the DLiteImageConfig class.""" @@ -33,11 +37,11 @@ def test_image_config() -> None: "test_file, target_file", ( # ("sample_1280_853.gif", "sample_150_100.gif"), - # ("sample_1280_853.jpeg", "sample_150_100.jpeg"), - # ("sample_1280_853.jpg", "sample_150_100.jpeg"), + ("sample_1280_853.jpeg", "sample_150_100.jpeg"), + ("sample_1280_853.jpg", "sample_150_100.jpeg"), # ("sample1.jp2", "sample1_150_100.jp2"), DISABLED BECAUSE SLOW ("sample_640_426.png", None), - # ("sample_640_426.tiff", None), + ("sample_640_426.tiff", None), ), ) def test_image( @@ -47,10 +51,12 @@ def test_image( static_files: "Path", ) -> None: """Test parsing an image format.""" + if crop_rect and (target_file is None or "jpeg" in target_file): + pytest.skip("This variable combination is not supported") + import dlite import numpy as np from oteapi.datacache import DataCache - from oteapi.models import ResourceConfig from PIL import Image from oteapi_dlite.strategies.parse_image import DLiteImageParseStrategy @@ -58,26 +64,26 @@ def test_image( sample_file = static_files / test_file orig_key = DataCache().add(sample_file.read_bytes()) - config = ResourceConfig( - downloadUrl="file://dummy.host/" + str(sample_file), - mediaType="image/" + test_file.rpartition(".")[2], - configuration={ + config = { + "downloadUrl": f"file://dummy.host/{sample_file}", + "mediaType": f"image/vnd.dlite-{sample_file.suffix.lstrip('.')}", + "configuration": { "image_label": "test_image", "crop": crop_rect, }, - ) + } coll = dlite.Collection() session = { "collection_id": coll.uuid, "key": orig_key, } - parser = DLiteImageParseStrategy(config) - output = parser.get(session) + parser: "IParseStrategy" = DLiteImageParseStrategy(config) + output: "DLiteSessionUpdate" = parser.get(session) assert "collection_id" in output assert output.collection_id == coll.uuid - coll2 = dlite.get_collection(session["collection_id"]) - inst = coll2.get("test_image") + coll2: dlite.Collection = dlite.get_collection(session["collection_id"]) + inst: dlite.Instance = coll2.get("test_image") # Compare data instance contents to expected values assert inst.meta.uri.startswith("http://onto-ns.com/meta") diff --git a/tests/strategies/test_serialise.py b/tests/strategies/test_serialise.py index f85dcb7f..691b179d 100644 --- a/tests/strategies/test_serialise.py +++ b/tests/strategies/test_serialise.py @@ -1,7 +1,13 @@ """Tests serialise strategy.""" +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from pathlib import Path -def test_serialise(tmp_dir): + from oteapi.interfaces import IFilterStrategy + + +def test_serialise(tmp_path: "Path") -> None: """Test the serialise filter.""" import dlite @@ -15,7 +21,7 @@ def test_serialise(tmp_dir): filterType="dlite_serialise", configuration={ "driver": "json", - "location": str(tmp_dir / "coll.json"), + "location": str(tmp_path / "coll.json"), "options": "mode=w", # "labels": ["image"], }, @@ -24,7 +30,7 @@ def test_serialise(tmp_dir): coll = dlite.Collection() session = {"collection_id": coll.uuid} - serialiser = SerialiseStrategy(config) + serialiser: "IFilterStrategy" = SerialiseStrategy(config) session.update(serialiser.initialize(session)) # Imitate other filters adding stuff to the collection @@ -36,7 +42,7 @@ def test_serialise(tmp_dir): image.data = [[[1], [2]], [[3], [4]]] coll.add("image", image) - serialiser = SerialiseStrategy(config) + serialiser: "IFilterStrategy" = SerialiseStrategy(config) session.update(serialiser.get(session)) - assert (tmp_dir / "coll.json").exists() + assert (tmp_path / "coll.json").exists()