diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d222c663..4ee11226 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,8 @@ Changelog Next ==== +- Allow finer grain for the requests cache (#487) + Version 1.2.27 - 2024-09-10 =========================== diff --git a/examples/preset.py b/examples/preset.py index 16a77dcf..f63497d3 100644 --- a/examples/preset.py +++ b/examples/preset.py @@ -31,7 +31,7 @@ SQL = """ SELECT * FROM - "https://d90230ca.us1a.app-sdx.preset.io/api/v1/chart/" + "https://12345678.us1a.app.preset.io/api/v1/chart/" LIMIT 12 """ for row in cursor.execute(SQL): diff --git a/src/shillelagh/adapters/api/github.py b/src/shillelagh/adapters/api/github.py index 8354c81d..2a851a6f 100644 --- a/src/shillelagh/adapters/api/github.py +++ b/src/shillelagh/adapters/api/github.py @@ -10,12 +10,12 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypedDict import jsonpath -import requests_cache from shillelagh.adapters.base import Adapter from shillelagh.exceptions import ProgrammingError from shillelagh.fields import Boolean, DateTime, Field, Integer, String, StringDateTime from shillelagh.filters import Equal, Filter +from shillelagh.lib import get_session from shillelagh.typing import RequestedOrder, Row _logger = logging.getLogger(__name__) @@ -220,11 +220,10 @@ def __init__( # pylint: disable=too-many-arguments self.resource = resource self.access_token = access_token - # use a cache for the API requests - self._session = requests_cache.CachedSession( + self._session = get_session( + request_headers={}, cache_name="github_cache", - backend="sqlite", - expire_after=180, + expire_after=timedelta(minutes=3), ) def get_columns(self) -> Dict[str, Field]: diff --git a/src/shillelagh/adapters/api/socrata.py b/src/shillelagh/adapters/api/socrata.py index 0e7e32b2..657d1353 100644 --- a/src/shillelagh/adapters/api/socrata.py +++ b/src/shillelagh/adapters/api/socrata.py @@ -7,10 +7,10 @@ import logging import re import urllib.parse +from datetime import timedelta from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union -import requests_cache from requests import Request from typing_extensions import TypedDict @@ -18,7 +18,7 @@ from shillelagh.exceptions import ImpossibleFilterError, ProgrammingError from shillelagh.fields import Field, Order, String, StringDate from shillelagh.filters import Equal, Filter, IsNotNull, IsNull, Like, NotEqual, Range -from shillelagh.lib import SimpleCostModel, build_sql, flatten +from shillelagh.lib import SimpleCostModel, build_sql, flatten, get_session from shillelagh.typing import RequestedOrder, Row _logger = logging.getLogger(__name__) @@ -126,10 +126,10 @@ def __init__(self, netloc: str, dataset_id: str, app_token: Optional[str] = None self.app_token = app_token # use a cache for the API requests - self._session = requests_cache.CachedSession( + self._session = get_session( + request_headers={}, cache_name="socrata_cache", - backend="sqlite", - expire_after=180, + expire_after=timedelta(minutes=3), ) self._set_columns() diff --git a/src/shillelagh/adapters/api/weatherapi.py b/src/shillelagh/adapters/api/weatherapi.py index d28a4978..4d93c591 100644 --- a/src/shillelagh/adapters/api/weatherapi.py +++ b/src/shillelagh/adapters/api/weatherapi.py @@ -9,12 +9,12 @@ import dateutil.parser import dateutil.tz -import requests_cache from shillelagh.adapters.base import Adapter from shillelagh.exceptions import ImpossibleFilterError from shillelagh.fields import DateTime, Float, IntBoolean, Integer, Order, String from shillelagh.filters import Filter, Impossible, Operator, Range +from shillelagh.lib import get_session from shillelagh.typing import RequestedOrder, Row _logger = logging.getLogger(__name__) @@ -150,10 +150,10 @@ def __init__(self, location: str, api_key: str, window: int = 7): # use a cache, since the adapter does a lot of similar API requests, # and the data should rarely (never?) change - self._session = requests_cache.CachedSession( + self._session = get_session( + request_headers={}, cache_name="weatherapi_cache", - backend="sqlite", - expire_after=180, + expire_after=timedelta(minutes=3), ) def get_cost( diff --git a/src/shillelagh/lib.py b/src/shillelagh/lib.py index 8e2c13ab..409efc37 100644 --- a/src/shillelagh/lib.py +++ b/src/shillelagh/lib.py @@ -611,16 +611,26 @@ def best_index_object_available() -> bool: return bool(Version(apsw.apswversion()) >= Version("3.41.0.0")) +def create_namespaced_cache_key(cache_name: str) -> str: + """ + Get the cache name with a specific namespace. + + This function does nothing. It can be monkeypatched to add a namespace to the cache, + if one is needed -- eg, when using the library in a multi-tenant environment. + """ + return cache_name + + def get_session( request_headers: Dict[str, str], cache_name: str, expire_after: timedelta = CACHE_EXPIRATION, -) -> requests_cache.CachedSession: # E: line too long (81 > 79 characters) +) -> requests_cache.CachedSession: """ Return a cached session. """ session = requests_cache.CachedSession( - cache_name=cache_name, + cache_name=create_namespaced_cache_key(cache_name), backend="sqlite", expire_after=( requests_cache.DO_NOT_CACHE diff --git a/tests/adapters/api/github_test.py b/tests/adapters/api/github_test.py index 43a5fee8..9c6311ff 100644 --- a/tests/adapters/api/github_test.py +++ b/tests/adapters/api/github_test.py @@ -28,7 +28,7 @@ def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None: Test a simple request. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -204,7 +204,7 @@ def test_github_single_resource(mocker: MockerFixture, requests_mock: Mocker) -> Test a request to a single resource. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -247,7 +247,7 @@ def test_github_single_resource_with_offset( Test a request to a single resource. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -273,7 +273,7 @@ def test_github_rate_limit(mocker: MockerFixture, requests_mock: Mocker) -> None Test that the adapter was rate limited by the API. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -315,7 +315,7 @@ def test_github_auth_token(mocker: MockerFixture, requests_mock: Mocker) -> None Test a simple request. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -358,7 +358,7 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) -> """ mocker.patch("shillelagh.adapters.api.github.PAGE_SIZE", new=5) mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -553,7 +553,7 @@ def test_github_missing_field(mocker: MockerFixture, requests_mock: Mocker) -> N For example, some issues don't have the ``draft`` field in the response. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -590,7 +590,7 @@ def test_github_json_field(mocker: MockerFixture, requests_mock: Mocker) -> None Test a request when the response has a JSON field. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) @@ -674,7 +674,7 @@ def test_github_participation(mocker: MockerFixture, requests_mock: Mocker) -> N Test a request to the participation stats. """ mocker.patch( - "shillelagh.adapters.api.github.requests_cache.CachedSession", + "shillelagh.adapters.api.github.get_session", return_value=Session(), ) diff --git a/tests/adapters/api/socrata_test.py b/tests/adapters/api/socrata_test.py index 7cb51981..17c27f1c 100644 --- a/tests/adapters/api/socrata_test.py +++ b/tests/adapters/api/socrata_test.py @@ -23,7 +23,7 @@ def test_socrata(mocker: MockerFixture, requests_mock: Mocker) -> None: Test a simple query. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -62,7 +62,7 @@ def test_socrata_app_token_url(mocker: MockerFixture, requests_mock: Mocker) -> Test app token being passed via the URL. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -96,7 +96,7 @@ def test_socrata_app_token_connection( Test app token being passed via the connection instead of the URL. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -129,7 +129,7 @@ def test_socrata_no_data(mocker: MockerFixture, requests_mock: Mocker) -> None: Test that some queries return no data. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -160,7 +160,7 @@ def test_socrata_impossible(mocker: MockerFixture, requests_mock: Mocker) -> Non Test that impossible queries return no data. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -190,7 +190,7 @@ def test_socrata_invalid_query(mocker: MockerFixture, requests_mock: Mocker) -> Test that invalid queries are handled correctly. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", return_value=Session(), ) @@ -258,7 +258,7 @@ def test_get_cost(mocker: MockerFixture) -> None: Test ``get_cost``. """ mocker.patch( - "shillelagh.adapters.api.socrata.requests_cache.CachedSession", + "shillelagh.adapters.api.socrata.get_session", ) adapter = SocrataAPI("netloc", "dataset", "XXX") diff --git a/tests/adapters/api/weatherapi_test.py b/tests/adapters/api/weatherapi_test.py index a64ae536..7ca8711c 100644 --- a/tests/adapters/api/weatherapi_test.py +++ b/tests/adapters/api/weatherapi_test.py @@ -25,7 +25,7 @@ def test_weatherapi(mocker: MockerFixture, requests_mock: Mocker) -> None: Test the adapter. """ mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", return_value=Session(), ) @@ -108,7 +108,7 @@ def test_weatherapi_api_error(mocker: MockerFixture, requests_mock: Mocker) -> N Test handling errors in the API. """ mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", return_value=Session(), ) @@ -532,7 +532,7 @@ def test_dispatch(mocker: MockerFixture, requests_mock: Mocker) -> None: Test the dispatcher. """ mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", return_value=Session(), ) @@ -593,7 +593,7 @@ def test_dispatch_api_key_connection( Test passing the key via the adapter kwargs. """ mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", return_value=Session(), ) @@ -625,7 +625,7 @@ def test_dispatch_impossible(mocker: MockerFixture) -> None: any network requests. """ session = mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", ) connection = connect( @@ -728,7 +728,7 @@ def test_get_cost(mocker: MockerFixture) -> None: Test ``get_cost``. """ mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", + "shillelagh.adapters.api.weatherapi.get_session", return_value=Session(), ) @@ -761,9 +761,7 @@ def test_window(mocker: MockerFixture) -> None: """ Test the default window size of days to fetch data. """ - session = mocker.patch( - "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession", - ) + session = mocker.patch("shillelagh.adapters.api.weatherapi.get_session") session.return_value.get.return_value.json.return_value = weatherapi_response adapter = WeatherAPI("location", "XXX") diff --git a/tests/lib_test.py b/tests/lib_test.py index b009b5d5..91bbbf2d 100644 --- a/tests/lib_test.py +++ b/tests/lib_test.py @@ -2,6 +2,7 @@ Tests for shillelagh.lib. """ +from datetime import timedelta from typing import Any, Dict, Iterator, List, Tuple import pytest @@ -31,6 +32,7 @@ escape_string, filter_data, find_adapter, + get_session, is_not_null, is_null, serialize, @@ -485,3 +487,32 @@ def test_apply_limit_and_offset() -> None: rows = apply_limit_and_offset(iter(range(10)), offset=2) assert list(rows) == [2, 3, 4, 5, 6, 7, 8, 9] + + +def test_get_session(mocker: MockerFixture) -> None: + """ + Test ``get_session``. + """ + requests_cache = mocker.patch("shillelagh.lib.requests_cache") + + get_session({}, "test", timedelta(seconds=10)) + requests_cache.CachedSession.assert_called_once_with( + cache_name="test", + backend="sqlite", + expire_after=10, + ) + + +def test_get_session_namespaced(mocker: MockerFixture) -> None: + """ + Test ``get_session`` with a namespaced cache key. + """ + requests_cache = mocker.patch("shillelagh.lib.requests_cache") + mocker.patch("shillelagh.lib.create_namespaced_cache_key", return_value="ns") + + get_session({}, "test", timedelta(seconds=10)) + requests_cache.CachedSession.assert_called_once_with( + cache_name="ns", + backend="sqlite", + expire_after=10, + )