From 6469a9a234f0b61a8b6a5a787e690af86a977075 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 25 May 2022 17:40:26 -0400 Subject: [PATCH 01/10] clarify type requirements for search intersects parameter, update some documentation --- CHANGELOG.md | 1 + README.md | 8 ++++++-- docs/usage.rst | 4 ++-- pystac_client/item_search.py | 30 ++++++++++++++++++++++-------- tests/test_item_search.py | 8 +++++++- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ff761b6..5308a389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Bumped PySTAC dependency to >= 1.4.0 [#147](https://github.com/stac-utils/pystac-client/pull/147) - Search `filter-lang` defaults to `cql2-json` instead of `cql-json` - Search `filter-lang` will be set to `cql2-json` if the `filter` is a dict, or `cql2-text` if it is a string +- Search parameter `intersects` is now typed to only accept a str, dict, or object that implements `__geo_interface__` ## Removed diff --git a/README.md b/README.md index 30e91db2..64de8cce 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -STAC Client -=============== +# STAC Client [![CI](https://github.com/stac-utils/pystac-client/actions/workflows/continuous-integration.yml/badge.svg)](https://github.com/stac-utils/pystac-client/actions/workflows/continuous-integration.yml) [![Release](https://github.com/stac-utils/pystac-client/actions/workflows/release.yml/badge.svg)](https://github.com/stac-utils/pystac-client/actions/workflows/release.yml) @@ -10,6 +9,11 @@ STAC Client A Python client for working with [STAC](https://stacspec.org/) Catalogs and APIs. +- [Installation](#installation) +- [Documentation](#documentation) +- [Development](#development) + - [Pull Requests](#pull-requests) + ## Installation Install from PyPi. Other than [PySTAC](https://pystac.readthedocs.io) itself, the only dependencies for pystac-client is the Python [requests](https://docs.python-requests.org) and [dateutil](https://dateutil.readthedocs.io) libraries. diff --git a/docs/usage.rst b/docs/usage.rst index 311076dd..3bbd8e93 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -129,7 +129,7 @@ The :meth:`pystac_client.Client.search` method provides an interface for making .. code-block:: python >>> from pystac_client import API - >>> api = API.from_file('https://eod-catalog-svc-prod.astraea.earth') + >>> api = API.from_file('https://planetarycomputer.microsoft.com/api/stac/v1') >>> results = api.search( ... bbox=[-73.21, 43.99, -73.12, 44.05], ... datetime=['2019-01-01T00:00:00Z', '2019-01-02T00:00:00Z'], @@ -169,7 +169,7 @@ implementation of this ``"next"`` link parsing assumes that the link follows the described in the `STAC API - Item Search: Paging `__ section. See the :mod:`Paging ` docs for details on how to customize this behavior. -Query Filter +Query Extension ------------ If the Catalog supports the `Query diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index c2f34f21..77a8d668 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -6,7 +6,7 @@ from collections.abc import Iterable, Mapping from copy import deepcopy from datetime import timezone, datetime as datetime_ -from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union +from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union, Protocol import warnings from pystac import Collection, Item, ItemCollection @@ -22,6 +22,12 @@ r"(?P(T|t)\d{2}:\d{2}:\d{2}(\.\d+)?" r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?") + +class GeoInterface(Protocol): + def __geo_interface__(self) -> dict: + ... + + DatetimeOrTimestamp = Optional[Union[datetime_, str]] Datetime = Union[Tuple[str], Tuple[str, str]] DatetimeLike = Union[DatetimeOrTimestamp, Tuple[DatetimeOrTimestamp, DatetimeOrTimestamp], @@ -37,7 +43,7 @@ IDsLike = Union[IDs, str, List[str], Iterator[str]] Intersects = dict -IntersectsLike = Union[str, Intersects, object] +IntersectsLike = Union[str, Intersects, GeoInterface] Query = dict QueryLike = Union[Query, List[str]] @@ -132,7 +138,9 @@ class ItemSearch: - ``2017/2018`` expands to ``2017-01-01T00:00:00Z/2018-12-31T23:59:59Z`` - ``2017-06/2017-07`` expands to ``2017-06-01T00:00:00Z/2017-07-31T23:59:59Z`` - ``2017-06-10/2017-06-11`` expands to ``2017-06-10T00:00:00Z/2017-06-11T23:59:59Z`` - intersects: A GeoJSON-like dictionary or JSON string. Results filtered to only those intersecting the geometry + intersects: A string or dictionary representing a GeoJSON geometry, or an object that implements a + ``__geo_interface__`` property as supported by several libraries including Shapely, ArcPy, PySAL, and + geojson. Results filtered to only those intersecting the geometry. ids: List of Item ids to return. All other filter parameters that further restrict the number of search results (except ``limit``) are ignored. collections: List of one or more Collection IDs or :class:`pystac.Collection` instances. Only Items in one @@ -142,11 +150,11 @@ class ItemSearch: filter_lang: Language variant used in the filter body. If `filter` is a dictionary or not provided, defaults to 'cql2-json'. If `filter` is a string, defaults to `cql2-text`. sortby: A single field or list of fields to sort the response by - fields: A list of fields to return in the response. Note this may result in invalid JSON. - Use `get_all_items_as_dict` to avoid errors - max_items: The maximum number of items to get, even if there are more matched items + fields: A list of fields to include in the response. Note this may result in invalid STAC objects, as they + may not have required fields. Use `get_all_items_as_dict` to avoid object unmarshalling errors. + max_items: The maximum number of items to get, even if there are more matched items. method: The http method, 'GET' or 'POST' - stac_io: An instance of of StacIO for retrieving results. Normally comes from the Client that returns this ItemSearch + stac_io: An instance of StacIO for retrieving results. Normally comes from the Client that returns this ItemSearch client: An instance of a root Client used to set the root on resulting Items """ def __init__(self, @@ -409,9 +417,15 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: def _format_intersects(value: Optional[IntersectsLike]) -> Optional[Intersects]: if value is None: return None + if isinstance(value, dict): + return deepcopy(value) if isinstance(value, str): return json.loads(value) - return deepcopy(getattr(value, '__geo_interface__', value)) + if hasattr(value, '__geo_interface__'): + return deepcopy(getattr(value, '__geo_interface__')) + raise Exception( + "intersects must be of type None, str, dict, or an object that implements __geo_interface__" + ) @lru_cache(1) def matched(self) -> int: diff --git a/tests/test_item_search.py b/tests/test_item_search.py index 336c2ab3..8e5d2915 100644 --- a/tests/test_item_search.py +++ b/tests/test_item_search.py @@ -260,6 +260,10 @@ def test_intersects_json_string(self): search = ItemSearch(url=SEARCH_URL, intersects=json.dumps(INTERSECTS_EXAMPLE)) assert search._parameters['intersects'] == INTERSECTS_EXAMPLE + def test_intersects_non_geo_interface_object(self): + with pytest.raises(Exception): + ItemSearch(url=SEARCH_URL, intersects=object()) + def test_filter_lang_default_for_dict(self): search = ItemSearch(url=SEARCH_URL, filter={}) assert search._parameters['filter-lang'] == 'cql2-json' @@ -377,7 +381,9 @@ def test_intersects_results(self): # Geo-interface object class MockGeoObject: - __geo_interface__ = intersects_dict + @property + def __geo_interface__(self): + return intersects_dict intersects_obj = MockGeoObject() search = ItemSearch(url=SEARCH_URL, intersects=intersects_obj, collections='naip') From 69f1b3afddf1e42b8398d3d87639a709ec91552d Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 08:47:54 -0400 Subject: [PATCH 02/10] add todo about runtime checkable --- pystac_client/item_search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 77a8d668..38c3d889 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -23,6 +23,7 @@ r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?") +# todo: add runtime_checkable when class GeoInterface(Protocol): def __geo_interface__(self) -> dict: ... From f1635b8dac50c43650294266825ae50515e59362 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:21:36 -0400 Subject: [PATCH 03/10] Update pystac_client/item_search.py Co-authored-by: Pete Gadomski --- pystac_client/item_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 38c3d889..45ce6f92 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -23,7 +23,7 @@ r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?") -# todo: add runtime_checkable when +# todo: add runtime_checkable when we drop 3.7 support class GeoInterface(Protocol): def __geo_interface__(self) -> dict: ... From 6d5a0eefd5095f0928a2613cc4d81f631d406ea5 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:25:50 -0400 Subject: [PATCH 04/10] refactor GeoInterface to support py 3.7 --- pystac_client/item_search.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 45ce6f92..162eed61 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -6,7 +6,7 @@ from collections.abc import Iterable, Mapping from copy import deepcopy from datetime import timezone, datetime as datetime_ -from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union, Protocol +from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union import warnings from pystac import Collection, Item, ItemCollection @@ -24,9 +24,9 @@ # todo: add runtime_checkable when we drop 3.7 support -class GeoInterface(Protocol): - def __geo_interface__(self) -> dict: - ... +# class GeoInterface(Protocol): +# def __geo_interface__(self) -> dict: +# ... DatetimeOrTimestamp = Optional[Union[datetime_, str]] @@ -44,7 +44,7 @@ def __geo_interface__(self) -> dict: IDsLike = Union[IDs, str, List[str], Iterator[str]] Intersects = dict -IntersectsLike = Union[str, Intersects, GeoInterface] +IntersectsLike = Union[str, object, Intersects] #, GeoInterface] Query = dict QueryLike = Union[Query, List[str]] From 4cc74c7b1a9905476e550425e4e3bbaca4d0ee85 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:33:05 -0400 Subject: [PATCH 05/10] add python 3.10 and 3.11 to setup.py and github ci config --- .github/workflows/continuous-integration.yml | 8 +++++++- setup.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 9889282e..fbedec9f 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -11,8 +11,14 @@ jobs: name: build runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11-dev" steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index ae85e01f..5c2c632e 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: OS Independent", "Natural Language :: English", "Development Status :: 3 - Alpha", From 4f52225c53655e6e439b124dd09b42b874c77f29 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:39:16 -0400 Subject: [PATCH 06/10] add todo to re-add GeoInterface typing --- pystac_client/item_search.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 162eed61..a391535e 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -22,13 +22,11 @@ r"(?P(T|t)\d{2}:\d{2}:\d{2}(\.\d+)?" r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?") - # todo: add runtime_checkable when we drop 3.7 support # class GeoInterface(Protocol): # def __geo_interface__(self) -> dict: # ... - DatetimeOrTimestamp = Optional[Union[datetime_, str]] Datetime = Union[Tuple[str], Tuple[str, str]] DatetimeLike = Union[DatetimeOrTimestamp, Tuple[DatetimeOrTimestamp, DatetimeOrTimestamp], @@ -44,7 +42,8 @@ IDsLike = Union[IDs, str, List[str], Iterator[str]] Intersects = dict -IntersectsLike = Union[str, object, Intersects] #, GeoInterface] +IntersectsLike = Union[str, object, Intersects] +# todo: after 3.7 is dropped, replace object with GeoInterface Query = dict QueryLike = Union[Query, List[str]] From e3dd4d031938277eccd62b9bc69aff714d1a39b7 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:48:52 -0400 Subject: [PATCH 07/10] add rust toolchain for 3.11 build --- .github/workflows/continuous-integration.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index fbedec9f..fa5af062 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -20,7 +20,7 @@ jobs: - "3.10" - "3.11-dev" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 @@ -29,6 +29,16 @@ jobs: cache: 'pip' cache-dependency-path: 'setup.py' + - uses: actions-rs/toolchain@v1 + # Wheels for orjson are not available for Python 3.11. This sets up the Rust + # toolchain, so we can build orjson wheel from source. + if: ${{ startsWith(matrix.python-version, '3.11')}} + with: + toolchain: stable + override: true + default: true + profile: minimal + - name: Execute linters and test suites run: ./scripts/cibuild From a3b8a4527fd5ab7c75f728d0cfa927f4b30a17d6 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:54:14 -0400 Subject: [PATCH 08/10] remove 3.11 support --- .github/workflows/continuous-integration.yml | 12 +----------- setup.py | 1 - 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index fa5af062..c7d77613 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: - "3.8" - "3.9" - "3.10" - - "3.11-dev" +# - "3.11-dev" steps: - uses: actions/checkout@v3 @@ -29,16 +29,6 @@ jobs: cache: 'pip' cache-dependency-path: 'setup.py' - - uses: actions-rs/toolchain@v1 - # Wheels for orjson are not available for Python 3.11. This sets up the Rust - # toolchain, so we can build orjson wheel from source. - if: ${{ startsWith(matrix.python-version, '3.11')}} - with: - toolchain: stable - override: true - default: true - profile: minimal - - name: Execute linters and test suites run: ./scripts/cibuild diff --git a/setup.py b/setup.py index 5c2c632e..ae85e01f 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,6 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Operating System :: OS Independent", "Natural Language :: English", "Development Status :: 3 - Alpha", From 05d2f7dc3d00ede99f546e9f7bb6ee2a61230783 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 09:55:26 -0400 Subject: [PATCH 09/10] add windows to actions matrix --- .github/workflows/continuous-integration.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index c7d77613..55ad98f3 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -9,7 +9,7 @@ on: jobs: build: name: build - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -19,6 +19,9 @@ jobs: - "3.9" - "3.10" # - "3.11-dev" + os: + - ubuntu-latest + - windows-latest steps: - uses: actions/checkout@v3 From aca4f7c64b4ca4f99b357dc205c1294bd4417614 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 10:02:29 -0400 Subject: [PATCH 10/10] remove windows from matrix --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 55ad98f3..84985391 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -21,7 +21,7 @@ jobs: # - "3.11-dev" os: - ubuntu-latest - - windows-latest +# - windows-latest steps: - uses: actions/checkout@v3