From af71269c83d1915cfab99494f5d496483c427038 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 1 Jul 2020 22:10:10 +0200 Subject: [PATCH 01/24] remove duplicate testing whether IIASA API is available (to skip module) --- tests/test_iiasa.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 0704491d3..a75544975 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -11,12 +11,6 @@ if IIASA_UNAVAILABLE: pytest.skip('IIASA database API unavailable', allow_module_level=True) -# verify whether IIASA database API can be reached, skip tests otherwise -try: - iiasa.Connection() -except SSLError: - pytest.skip('IIASA database API unavailable', allow_module_level=True) - # check to see if we can do online testing of db authentication TEST_ENV_USER = 'IIASA_CONN_TEST_USER' TEST_ENV_PW = 'IIASA_CONN_TEST_PW' From 95df140003507bda8aa0eb71d1259d211348dee2 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 1 Jul 2020 22:10:20 +0200 Subject: [PATCH 02/24] update docstrings --- pyam/iiasa.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 34800ce25..38bb38bd7 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -15,7 +15,7 @@ from pyam.logging import deprecation_warning logger = logging.getLogger(__name__) -# quiet this fool +# set requests-logger to WARNING only logging.getLogger('requests').setLevel(logging.WARNING) _BASE_URL = 'https://db1.ene.iiasa.ac.at/EneAuth/config/v1' @@ -98,13 +98,14 @@ def _get_token(creds, base_url): class Connection(object): - """A class to facilitate querying an IIASA scenario explorer database + """A class to facilitate querying an IIASA Scenario Explorer database API Parameters ---------- name : str, optional - A valid database name. For available options, see - valid_connections(). + The name of a database API. + See :meth:`pyam.iiasa.Connection.valid_connections` for a list + of available APIs. creds : str, :class:`pathlib.Path`, list-like, or dict, optional By default, this function will (try to) read user credentials which were set using :meth:`pyam.iiasa.set_config(, )`. @@ -164,11 +165,7 @@ def _connection_map(self): @property @lru_cache() def valid_connections(self): - """ Show a list of valid connection names (application aliases or - names when alias is not available or duplicated) - - :return: list of str - """ + """Return available database API connection names""" return list(self._connection_map.keys()) def connect(self, name): @@ -451,7 +448,7 @@ def query(self, **kwargs): def read_iiasa(name, meta=False, creds=None, base_url=_BASE_URL, **kwargs): - """Read data from an IIASA scenario explorer and return as IamDataFrame + """Query an IIASA Scenario Explorer database and return as IamDataFrame Parameters ---------- From ecc03660c82d4200e4dccebfc30c9d910622beda Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 1 Jul 2020 22:11:19 +0200 Subject: [PATCH 03/24] skip citation message if no ui-url is given --- pyam/iiasa.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 38bb38bd7..d98f22519 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -195,8 +195,9 @@ def connect(self, name): self._base_url = response[idxs['baseUrl']]['value'] # TODO: request the full citation to be added to this metadata instead # of linking to the about page - about = '/'.join([response[idxs['uiUrl']]['value'], '#', 'about']) - logger.info(_CITE_MSG.format(name, about)) + if 'uiUrl' in idxs: + about = '/'.join([response[idxs['uiUrl']]['value'], '#', 'about']) + logger.info(_CITE_MSG.format(name, about)) self._connected = name From fb1a8f5f70093e5e6e0efc1a11db1567f7dab032 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 1 Jul 2020 22:12:16 +0200 Subject: [PATCH 04/24] replace/deprecate `available_metadata()` & `metadata()` for consistency --- pyam/iiasa.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index d98f22519..f8daa7013 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -226,17 +226,24 @@ def scenario_list(self, default=True): _check_response(r, 'Could not get scenario list') return pd.read_json(r.content, orient='records') + @property @lru_cache() - def available_metadata(self): - """List all available meta indicators in the instance""" + def meta_columns(self): + """Return a list of meta indicators in the database instance""" url = '/'.join([self._base_url, 'metadata/types']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r) return pd.read_json(r.content, orient='records')['name'] + def available_metadata(self): + """Deprecated""" + # TODO: deprecate/remove this function in release >=0.8 + deprecation_warning('Use `Connection.meta_columns` instead.') + return self.meta_columns + @lru_cache() - def metadata(self, default=True): + def meta(self, default=True): """All meta categories and indicators of scenarios Parameters @@ -268,6 +275,12 @@ def extract(row): return pd.concat([extract(row) for idx, row in df.iterrows()], sort=False).reset_index() + def metadata(self, default=True): + """Deprecated""" + # TODO: deprecate/remove this function in release >=0.8 + deprecation_warning('Use `Connection.meta()` instead.') + return self.meta + def models(self): """All models in the connected data source""" return pd.Series(self.scenario_list()['model'].unique(), From cfe90ee5ad9cd6d0ba493b78bae8d03bc7c7e05c Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 1 Jul 2020 22:15:23 +0200 Subject: [PATCH 05/24] refactor to `_auth_url` (from `_base_url`) --- pyam/iiasa.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index f8daa7013..a3a53b88f 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -18,7 +18,7 @@ # set requests-logger to WARNING only logging.getLogger('requests').setLevel(logging.WARNING) -_BASE_URL = 'https://db1.ene.iiasa.ac.at/EneAuth/config/v1' +_AUTH_URL = 'https://db1.ene.iiasa.ac.at/EneAuth/config/v1' _CITE_MSG = """ You are connected to the {} scenario explorer hosted by IIASA. If you use this data in any published format, please cite the @@ -120,9 +120,9 @@ class Connection(object): for backwards compatibility. However, this option is NOT RECOMMENDED and will be deprecated in future releases of pyam. """ - def __init__(self, name=None, creds=None, base_url=_BASE_URL): - self._base_url = base_url - self._token, self._user = _get_token(creds, base_url=self._base_url) + def __init__(self, name=None, creds=None, auth_url=_AUTH_URL): + self._auth_url = auth_url + self._token, self._user = _get_token(creds, base_url=self._auth_url) # connect if provided a name self._connected = None @@ -137,7 +137,7 @@ def __init__(self, name=None, creds=None, base_url=_BASE_URL): @property @lru_cache() def _connection_map(self): - url = '/'.join([self._base_url, 'applications']) + url = '/'.join([self._auth_url, 'applications']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r, 'Could not get valid connection list') @@ -185,14 +185,14 @@ def connect(self, name): """ raise ValueError(msg.format(name, valid)) - url = '/'.join([self._base_url, 'applications', name, 'config']) + url = '/'.join([self._auth_url, 'applications', name, 'config']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r, 'Could not get application information') response = r.json() idxs = {x['path']: i for i, x in enumerate(response)} - self._base_url = response[idxs['baseUrl']]['value'] + self._auth_url = response[idxs['baseUrl']]['value'] # TODO: request the full citation to be added to this metadata instead # of linking to the about page if 'uiUrl' in idxs: @@ -220,7 +220,7 @@ def scenario_list(self, default=True): """ default = 'true' if default else 'false' add_url = 'runs?getOnlyDefaultRuns={}' - url = '/'.join([self._base_url, add_url.format(default)]) + url = '/'.join([self._auth_url, add_url.format(default)]) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r, 'Could not get scenario list') @@ -230,7 +230,7 @@ def scenario_list(self, default=True): @lru_cache() def meta_columns(self): """Return a list of meta indicators in the database instance""" - url = '/'.join([self._base_url, 'metadata/types']) + url = '/'.join([self._auth_url, 'metadata/types']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r) @@ -257,7 +257,7 @@ def meta(self, default=True): # up in the future to try to query a subset default = 'true' if default else 'false' add_url = 'runs?getOnlyDefaultRuns={}&includeMetadata=true' - url = '/'.join([self._base_url, add_url.format(default)]) + url = '/'.join([self._auth_url, add_url.format(default)]) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r) @@ -279,7 +279,7 @@ def metadata(self, default=True): """Deprecated""" # TODO: deprecate/remove this function in release >=0.8 deprecation_warning('Use `Connection.meta()` instead.') - return self.meta + return self.meta(default=default) def models(self): """All models in the connected data source""" @@ -294,7 +294,7 @@ def scenarios(self): @lru_cache() def variables(self): """All variables in the connected data source""" - url = '/'.join([self._base_url, 'ts']) + url = '/'.join([self._auth_url, 'ts']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r) @@ -312,7 +312,7 @@ def regions(self, include_synonyms=False): (possibly leading to duplicate region names for regions with more than one synonym) """ - url = '/'.join([self._base_url, 'nodes?hierarchy=%2A']) + url = '/'.join([self._auth_url, 'nodes?hierarchy=%2A']) headers = {'Authorization': 'Bearer {}'.format(self._token)} params = {'includeSynonyms': include_synonyms} r = requests.get(url, headers=headers, params=params) @@ -423,7 +423,7 @@ def query(self, **kwargs): 'Content-Type': 'application/json', } data = json.dumps(self._query_post_data(**kwargs)) - url = '/'.join([self._base_url, 'runs/bulk/ts']) + url = '/'.join([self._auth_url, 'runs/bulk/ts']) logger.debug('Querying timeseries data ' 'from {} with filter {}'.format(url, data)) r = requests.post(url, headers=headers, data=data) @@ -461,7 +461,7 @@ def query(self, **kwargs): return df -def read_iiasa(name, meta=False, creds=None, base_url=_BASE_URL, **kwargs): +def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): """Query an IIASA Scenario Explorer database and return as IamDataFrame Parameters From 81c15131e8fb9ba0b152f6562935e47478e2bbf4 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Thu, 2 Jul 2020 07:06:27 +0200 Subject: [PATCH 06/24] fix attribute markup --- pyam/iiasa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index a3a53b88f..7c7a8d48d 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -104,7 +104,7 @@ class Connection(object): ---------- name : str, optional The name of a database API. - See :meth:`pyam.iiasa.Connection.valid_connections` for a list + See :attr:`pyam.iiasa.Connection.valid_connections` for a list of available APIs. creds : str, :class:`pathlib.Path`, list-like, or dict, optional By default, this function will (try to) read user credentials which From cd4cca90a71f5593c5d14429dd79c25ef9f06e08 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 13:09:52 +0200 Subject: [PATCH 07/24] return nice-names of valid connections, clean up deprecation warning --- pyam/iiasa.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 7c7a8d48d..57e7dcdb7 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -83,9 +83,9 @@ def _get_token(creds, base_url): if plaintextcreds: logger.warning('You provided credentials in plain text. DO NOT save ' 'these in a repository or otherwise post them online') - deprecation_warning('Providing credentials in plain text', - 'Please use `pyam.iiasa.set_config(, )`' - ' to store your credentials in a file!') + deprecation_warning('Please use `pyam.iiasa.set_config(, )`' + ' to store your credentials in a file!', + 'Providing credentials in plain text') # get user token headers = {'Accept': 'application/json', @@ -132,7 +132,7 @@ def __init__(self, name=None, creds=None, auth_url=_AUTH_URL): if self._user: logger.info(f'You are connected as user `{self._user}`') else: - logger.info(f'You are connected as an anonymous user') + logger.info('You are connected as an anonymous user') @property @lru_cache() @@ -183,7 +183,7 @@ def connect(self, name): {} not recognized as a valid connection name. Choose from one of the supported connections for your user: {}. """ - raise ValueError(msg.format(name, valid)) + raise ValueError(msg.format(name, self._connection_map.keys())) url = '/'.join([self._auth_url, 'applications', name, 'config']) headers = {'Authorization': 'Bearer {}'.format(self._token)} From 00e1c5aaf2b3a805884c23a75b6f8e8702c6257f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 13:16:56 +0200 Subject: [PATCH 08/24] redirect all auth-tests to `ixmp-integration` instance --- tests/test_iiasa.py | 58 +++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index a75544975..22a237428 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -1,9 +1,8 @@ import os import copy -import yaml +import logging import pytest import numpy.testing as npt -from requests.exceptions import SSLError from pyam import iiasa from conftest import IIASA_UNAVAILABLE @@ -19,54 +18,51 @@ TEST_ENV_USER, TEST_ENV_PW ) +TEST_API = 'integration-test' +TEST_API_NAME = 'IXSE_INTEGRATION_TEST' -def test_anon_conn(): - conn = iiasa.Connection('IXSE_SR15') - assert conn.current_connection == 'IXSE_SR15' + +def test_unknown_conn(): + # connecting to an unknown API raises an error + pytest.raises(ValueError, iiasa.Connection, 'foo') -def test_anon_conn_warning(): - conn = iiasa.Connection('iamc15') - assert conn.current_connection == 'IXSE_SR15' +def test_anon_conn(): + conn = iiasa.Connection(TEST_API) + assert conn.current_connection == TEST_API_NAME @pytest.mark.skipif(not CONN_ENV_AVAILABLE, reason=CONN_ENV_REASON) -def test_conn_creds_file(tmp_path): - user, pw = os.environ[TEST_ENV_USER], os.environ[TEST_ENV_PW] - path = tmp_path / 'config.yaml' - with open(path, 'w') as f: - yaml.dump({'username': user, 'password': pw}, f) - conn = iiasa.Connection('IXSE_SR15', creds=path) - assert conn.current_connection == 'IXSE_SR15' +def test_conn_creds_config(): + iiasa.set_config(os.environ[TEST_ENV_USER], os.environ[TEST_ENV_PW]) + conn = iiasa.Connection(TEST_API) + assert conn.current_connection == TEST_API_NAME @pytest.mark.skipif(not CONN_ENV_AVAILABLE, reason=CONN_ENV_REASON) def test_conn_creds_tuple(): user, pw = os.environ[TEST_ENV_USER], os.environ[TEST_ENV_PW] - conn = iiasa.Connection('IXSE_SR15', creds=(user, pw)) - assert conn.current_connection == 'IXSE_SR15' - - -def test_conn_bad_creds(): - pytest.raises(RuntimeError, iiasa.Connection, - 'IXSE_SR15', creds=('_foo', '_bar')) - - -def test_anon_conn_tuple_raises(): - pytest.raises(ValueError, iiasa.Connection, 'foo') + conn = iiasa.Connection(TEST_API, creds=(user, pw)) + assert conn.current_connection == TEST_API_NAME @pytest.mark.skipif(not CONN_ENV_AVAILABLE, reason=CONN_ENV_REASON) def test_conn_creds_dict(): user, pw = os.environ[TEST_ENV_USER], os.environ[TEST_ENV_PW] - conn = iiasa.Connection( - 'IXSE_SR15', creds={'username': user, 'password': pw}) - assert conn.current_connection == 'IXSE_SR15' + conn = iiasa.Connection(TEST_API, creds={'username': user, 'password': pw}) + assert conn.current_connection == TEST_API_NAME + + +def test_conn_bad_creds(): + # connecting with invalid credentials raises an error + creds = ('_foo', '_bar') + pytest.raises(RuntimeError, iiasa.Connection, TEST_API, creds=creds) def test_conn_creds_dict_raises(): - pytest.raises(KeyError, iiasa.Connection, - 'IXSE_SR15', creds={'username': 'foo'}) + # connecting with incomplete credentials as dictionary raises an error + creds = {'username': 'foo'} + pytest.raises(KeyError, iiasa.Connection, TEST_API, cres=creds) def test_variables(): From 9d6d70f5961f4816e58af4913f0a3f51944b741a Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 15:01:56 +0200 Subject: [PATCH 09/24] refactor functions and update docstrings --- pyam/iiasa.py | 58 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 57e7dcdb7..1ef575c36 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -165,10 +165,11 @@ def _connection_map(self): @property @lru_cache() def valid_connections(self): - """Return available database API connection names""" + """Return available resources (database API connections)""" return list(self._connection_map.keys()) def connect(self, name): + """Connect to a database API""" if name in self._connection_map: name = self._connection_map[name] @@ -199,25 +200,34 @@ def connect(self, name): about = '/'.join([response[idxs['uiUrl']]['value'], '#', 'about']) logger.info(_CITE_MSG.format(name, about)) + # TODO: use API "nice-name" self._connected = name @property def current_connection(self): + """Currently connected resource (database API connection)""" return self._connected - @lru_cache() - def scenario_list(self, default=True): - """ - Metadata regarding the list of scenarios (e.g., models, scenarios, - run identifier, etc.) in the connected data source. + def index(self, default=True): + """Return the index of models and scenarios in the connected resource Parameters ---------- default : bool, optional - Return *only* the default version of each Scenario. - Any (`model`, `scenario`) without a default version is omitted. - If :obj:`False`, return all versions. + If `True`, return *only* the default version of a model/scenario. + Any model/scenario without a default version is omitted. + If `False`, returns all versions. """ + cols = [] if default else ['version', 'is_default'] + return self._query_index(default)[META_IDX + cols].set_index(META_IDX) + + def scenario_list(self, default=True): + """Deprecated, use :meth:`Connection.index`""" + deprecation_warning('Use `Connection.index()` instead.') + return self._query_index(default) + + @lru_cache() + def _query_index(self, default=True): default = 'true' if default else 'false' add_url = 'runs?getOnlyDefaultRuns={}' url = '/'.join([self._auth_url, add_url.format(default)]) @@ -229,7 +239,7 @@ def scenario_list(self, default=True): @property @lru_cache() def meta_columns(self): - """Return a list of meta indicators in the database instance""" + """Return the list of meta indicators in the connected resource""" url = '/'.join([self._auth_url, 'metadata/types']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) @@ -237,19 +247,19 @@ def meta_columns(self): return pd.read_json(r.content, orient='records')['name'] def available_metadata(self): - """Deprecated""" + """Deprecated, use :attr:`Connection.meta_columns`""" # TODO: deprecate/remove this function in release >=0.8 deprecation_warning('Use `Connection.meta_columns` instead.') return self.meta_columns @lru_cache() def meta(self, default=True): - """All meta categories and indicators of scenarios + """Return meta categories and indicators of scenarios Parameters ---------- default : bool, optional - Return *only* the default version of each Scenario. + Return *only* the default version of each scenario. Any (`model`, `scenario`) without a default version is omitted. If :obj:`False`, return all versions. """ @@ -276,24 +286,24 @@ def extract(row): sort=False).reset_index() def metadata(self, default=True): - """Deprecated""" + """Deprecated, use :meth:`Connection.meta`""" # TODO: deprecate/remove this function in release >=0.8 deprecation_warning('Use `Connection.meta()` instead.') return self.meta(default=default) def models(self): - """All models in the connected data source""" - return pd.Series(self.scenario_list()['model'].unique(), + """List all models in the connected resource""" + return pd.Series(self._query_index()['model'].unique(), name='model') def scenarios(self): - """All scenarios in the connected data source""" - return pd.Series(self.scenario_list()['scenario'].unique(), + """List all scenarios in the connected resource""" + return pd.Series(self._query_index()['scenario'].unique(), name='scenario') @lru_cache() def variables(self): - """All variables in the connected data source""" + """List all variables in the connected resource""" url = '/'.join([self._auth_url, 'ts']) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) @@ -303,7 +313,7 @@ def variables(self): @lru_cache() def regions(self, include_synonyms=False): - """All regions in the connected data source + """List all regions in the connected resource Parameters ---------- @@ -359,7 +369,7 @@ def _match(data, patterns): return data[matches].unique() # get unique run ids - meta = self.scenario_list() + meta = self._query_index() meta = meta[meta.is_default] models = _match(meta['model'], m_pattern) scenarios = _match(meta['scenario'], s_pattern) @@ -397,7 +407,7 @@ def _match(data, patterns): return data def query(self, **kwargs): - """Query the data source with filters + """Query the connected resource for timeseries data (with filters) Available keyword arguments include @@ -462,7 +472,7 @@ def query(self, **kwargs): def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): - """Query an IIASA Scenario Explorer database and return as IamDataFrame + """Query an IIASA Scenario Explorer database API and return as IamDataFrame Parameters ---------- @@ -470,7 +480,7 @@ def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): A valid name of an IIASA scenario explorer instance, see :attr:`pyam.iiasa.Connection.valid_connections` meta : bool or list of strings - If :obj:`True`, include all meta categories & quantitative indicators + If `True`, include all meta categories & quantitative indicators (or subset if list is given). creds : dict Credentials to access scenario explorer instance and From ef3151a8577f223aa9a83b090ef29ead5d6669f6 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 15:02:48 +0200 Subject: [PATCH 10/24] add connection-fixture, switch tests to new instance --- tests/conftest.py | 9 +++++++++ tests/test_iiasa.py | 38 +++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0520e4f06..325830a88 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,9 @@ except SSLError: IIASA_UNAVAILABLE = True +TEST_API = 'integration-test' +TEST_API_NAME = 'IXSE_INTEGRATION_TEST' + here = os.path.dirname(os.path.realpath(__file__)) IMAGE_BASELINE_DIR = os.path.join(here, 'expected_figs') @@ -181,3 +184,9 @@ def plot_df(): def plot_stack_plot_df(): df = IamDataFrame(TEST_STACKPLOT_DF) yield df + + +@pytest.fixture(scope="session") +def conn(): + if not IIASA_UNAVAILABLE: + return iiasa.Connection(TEST_API) \ No newline at end of file diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 22a237428..f5c349f06 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -1,11 +1,13 @@ import os import copy -import logging import pytest +import pandas as pd + import numpy.testing as npt +import pandas.testing as pdt from pyam import iiasa -from conftest import IIASA_UNAVAILABLE +from conftest import IIASA_UNAVAILABLE, TEST_API, TEST_API_NAME if IIASA_UNAVAILABLE: pytest.skip('IIASA database API unavailable', allow_module_level=True) @@ -18,17 +20,13 @@ TEST_ENV_USER, TEST_ENV_PW ) -TEST_API = 'integration-test' -TEST_API_NAME = 'IXSE_INTEGRATION_TEST' - def test_unknown_conn(): # connecting to an unknown API raises an error pytest.raises(ValueError, iiasa.Connection, 'foo') -def test_anon_conn(): - conn = iiasa.Connection(TEST_API) +def test_anon_conn(conn): assert conn.current_connection == TEST_API_NAME @@ -62,27 +60,25 @@ def test_conn_bad_creds(): def test_conn_creds_dict_raises(): # connecting with incomplete credentials as dictionary raises an error creds = {'username': 'foo'} - pytest.raises(KeyError, iiasa.Connection, TEST_API, cres=creds) + pytest.raises(KeyError, iiasa.Connection, TEST_API, creds=creds) -def test_variables(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn.variables().values - assert 'Emissions|CO2' in obs +def test_variables(conn): + # check that connection returns the correct variables + npt.assert_array_equal(conn.variables(), + ['Primary Energy', 'Primary Energy|Coal']) -def test_regions(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn.regions().values - assert 'World' in obs +def test_regions(conn): + # check that connection returns the correct regions + npt.assert_array_equal(conn.regions(), ['World', 'region_a']) -def test_regions_with_synonyms(): - conn = iiasa.Connection('IXSE_SR15') +def test_regions_with_synonyms(conn): obs = conn.regions(include_synonyms=True) - assert 'synonym' in obs.columns - assert (obs[obs.region == 'R5ROWO'] - .synonym == 'Rest of the World (R5)').all() + exp = pd.DataFrame([['World', None], ['region_a', 'ISO_a']], + columns=['region', 'synonym']) + pdt.assert_frame_equal(obs, exp) def test_regions_empty_response(): From 3069d68323398d465664e6ab8b2ae67014a02332 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 21:25:17 +0200 Subject: [PATCH 11/24] rework `index` and `meta` functions --- pyam/iiasa.py | 55 +++++++++++++++++++++++++-------------------- tests/test_iiasa.py | 54 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 33 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 1ef575c36..525c2b050 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -169,7 +169,7 @@ def valid_connections(self): return list(self._connection_map.keys()) def connect(self, name): - """Connect to a database API""" + """Connect to a specific resource (database API)""" if name in self._connection_map: name = self._connection_map[name] @@ -194,8 +194,7 @@ def connect(self, name): idxs = {x['path']: i for i, x in enumerate(response)} self._auth_url = response[idxs['baseUrl']]['value'] - # TODO: request the full citation to be added to this metadata instead - # of linking to the about page + # TODO: proper citation (as metadata) instead of link to the about page if 'uiUrl' in idxs: about = '/'.join([response[idxs['uiUrl']]['value'], '#', 'about']) logger.info(_CITE_MSG.format(name, about)) @@ -218,7 +217,7 @@ def index(self, default=True): Any model/scenario without a default version is omitted. If `False`, returns all versions. """ - cols = [] if default else ['version', 'is_default'] + cols = ['version'] if default else ['version', 'is_default'] return self._query_index(default)[META_IDX + cols].set_index(META_IDX) def scenario_list(self, default=True): @@ -233,7 +232,7 @@ def _query_index(self, default=True): url = '/'.join([self._auth_url, add_url.format(default)]) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) - _check_response(r, 'Could not get scenario list') + _check_response(r, 'Could not retrieve the resource index') return pd.read_json(r.content, orient='records') @property @@ -254,7 +253,7 @@ def available_metadata(self): @lru_cache() def meta(self, default=True): - """Return meta categories and indicators of scenarios + """Return categories and indicators (meta) of scenarios Parameters ---------- @@ -263,27 +262,29 @@ def meta(self, default=True): Any (`model`, `scenario`) without a default version is omitted. If :obj:`False`, return all versions. """ - # at present this reads in all data for all scenarios, it could be sped - # up in the future to try to query a subset - default = 'true' if default else 'false' + # TODO: at present this reads in all data for all scenarios, + # it could be sped up in the future to try to query a subset + _default = 'true' if default else 'false' add_url = 'runs?getOnlyDefaultRuns={}&includeMetadata=true' - url = '/'.join([self._auth_url, add_url.format(default)]) + url = '/'.join([self._auth_url, add_url.format(_default)]) headers = {'Authorization': 'Bearer {}'.format(self._token)} r = requests.get(url, headers=headers) _check_response(r) df = pd.read_json(r.content, orient='records') + cols = ['version'] if default else ['version', 'is_default'] + def extract(row): return ( - pd.concat([row[['model', 'scenario']], + pd.concat([row[META_IDX+cols], pd.Series(row.metadata)]) .to_frame() .T .set_index(['model', 'scenario']) ) - return pd.concat([extract(row) for idx, row in df.iterrows()], - sort=False).reset_index() + return pd.concat([extract(row) for i, row in df.iterrows()], + sort=False) def metadata(self, default=True): """Deprecated, use :meth:`Connection.meta`""" @@ -416,6 +417,10 @@ def query(self, **kwargs): - region - variable + Returns + ------- + IamDataFrame + Examples -------- @@ -444,18 +449,20 @@ def query(self, **kwargs): df = pd.read_json(r.content, orient='records', dtype=dtype) logger.debug('Response size is {0} bytes, ' '{1} records'.format(len(r.content), len(df))) - columns = ['model', 'scenario', 'variable', 'unit', - 'region', 'year', 'value', 'time', 'meta', - 'version'] + columns = ['model', 'scenario', 'variable', 'region', 'unit', + 'year', 'value', 'time', 'version'] # keep only known columns or init empty df - df = pd.DataFrame(data=df, columns=columns) - # replace missing meta (for backward compatibility) - df.fillna({'meta': 0}, inplace=True) - df.fillna({'time': 0}, inplace=True) - df.rename(columns={'time': 'subannual'}, inplace=True) - # check if returned dataframe has subannual disaggregation, drop if not - if pd.Series([i in [-1, 'year'] for i in df.subannual]).all(): - df.drop(columns='subannual', inplace=True) + df = ( + pd.DataFrame(data=df, columns=columns) + # TODO: refactor API to directly return column as 'subannual' + .rename(columns={'time': 'subannual'}) + ) + + # check if timeseries data has subannual disaggregation, drop if not + if 'subannual' in df: + if all([i in [-1, np.nan, 'Year'] for i in df.subannual.unique()]): + df.drop(columns='subannual', inplace=True) + # check if there are multiple version for any model/scenario lst = ( df[META_IDX + ['version']].drop_duplicates() diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index f5c349f06..35d05fb1e 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -2,11 +2,12 @@ import copy import pytest import pandas as pd +import numpy as np import numpy.testing as npt import pandas.testing as pdt -from pyam import iiasa +from pyam import iiasa, META_IDX from conftest import IIASA_UNAVAILABLE, TEST_API, TEST_API_NAME if IIASA_UNAVAILABLE: @@ -20,6 +21,14 @@ TEST_ENV_USER, TEST_ENV_PW ) +META_COLS = ['number', 'string'] +META_DF = pd.DataFrame([ + ['model_a', 'scen_a', 1, True, 1, 'foo'], + ['model_a', 'scen_b', 1, True, 2, np.nan], + ['model_a', 'scen_a', 2, False, 1, 'bar'], + ['model_b', 'scen_a', 1, True, 3, 'baz'] +], columns=META_IDX+['version', 'is_default']+META_COLS).set_index(META_IDX) + def test_unknown_conn(): # connecting to an unknown API raises an error @@ -63,6 +72,7 @@ def test_conn_creds_dict_raises(): pytest.raises(KeyError, iiasa.Connection, TEST_API, creds=creds) + def test_variables(conn): # check that connection returns the correct variables npt.assert_array_equal(conn.variables(), @@ -117,17 +127,43 @@ def test_regions_with_synonyms_response(): .synonym.isin(['Deutschland', 'DE'])).all() -def test_metadata(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn.scenario_list()['model'].values - assert 'MESSAGEix-GLOBIOM 1.0' in obs +def test_meta_columns(conn): + # test that connection returns the correct list of meta indicators + npt.assert_array_equal(conn.meta_columns, META_COLS) + # test for deprecated version of the function + npt.assert_array_equal(conn.available_metadata(), META_COLS) + +@pytest.mark.parametrize("default", [True, False]) +def test_index(conn, default): + # test that connection returns the correct index + if default: + exp = META_DF.loc[META_DF.is_default, ['version']] + else: + exp = META_DF[['version', 'is_default']] + + pdt.assert_frame_equal(conn.index(default=default), exp, check_dtype=False) + + +@pytest.mark.parametrize("default", [True, False]) +def test_meta(conn, default): + # test that connection returns the correct meta dataframe + if default: + exp = META_DF.loc[META_DF.is_default, ['version'] + META_COLS] + else: + exp = META_DF[['version', 'is_default'] + META_COLS] + + pdt.assert_frame_equal(conn.meta(default=default), exp, check_dtype=False) + + # test for deprecated version of the function + pdt.assert_frame_equal(conn.metadata(default=default), exp, + check_dtype=False) -def test_available_indicators(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn.available_metadata() - assert 'carbon price|2050' in list(obs) +def test_query(conn): + # test reading timeseries data + df = conn.query() + print(df) QUERY_DATA_EXP = { "filters": { From 596356ae4051fdeb558938fb32b34416aad4b3cf Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 3 Jul 2020 21:26:36 +0200 Subject: [PATCH 12/24] add a todo --- pyam/iiasa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 525c2b050..01c455305 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -227,6 +227,7 @@ def scenario_list(self, default=True): @lru_cache() def _query_index(self, default=True): + # TODO merge this function with `meta()` default = 'true' if default else 'false' add_url = 'runs?getOnlyDefaultRuns={}' url = '/'.join([self._auth_url, add_url.format(default)]) From 5db4c4306ca9a50a6fc929e0a6cb71ab7ffe8d41 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 13:20:33 +0200 Subject: [PATCH 13/24] fix subannual column with `query`, return IamDataFrame, refactor tests --- pyam/iiasa.py | 26 +++++------ tests/test_iiasa.py | 103 ++++---------------------------------------- 2 files changed, 20 insertions(+), 109 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 01c455305..4a84547a1 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -11,7 +11,7 @@ from collections.abc import Mapping from pyam.core import IamDataFrame -from pyam.utils import META_IDX, islistable, isstr, pattern_match +from pyam.utils import META_IDX, IAMC_IDX, islistable, isstr, pattern_match from pyam.logging import deprecation_warning logger = logging.getLogger(__name__) @@ -352,7 +352,8 @@ def convert_regions_payload(response, include_synonyms): def _query_post_data(self, **kwargs): def _get_kwarg(k): - x = kwargs.pop(k, []) + # TODO refactor API to return all models if model-list is empty + x = kwargs.pop(k, '*' if k == 'model' else []) return [x] if isstr(x) else x m_pattern = _get_kwarg('model') @@ -440,28 +441,22 @@ def query(self, **kwargs): } data = json.dumps(self._query_post_data(**kwargs)) url = '/'.join([self._auth_url, 'runs/bulk/ts']) - logger.debug('Querying timeseries data ' - 'from {} with filter {}'.format(url, data)) + logger.debug(f'Querying timeseries data from {url} with filter {data}') r = requests.post(url, headers=headers, data=data) _check_response(r) # refactor returned json object to be castable to an IamDataFrame dtype = dict(model=str, scenario=str, variable=str, unit=str, region=str, year=int, value=float, version=int) df = pd.read_json(r.content, orient='records', dtype=dtype) - logger.debug('Response size is {0} bytes, ' - '{1} records'.format(len(r.content), len(df))) - columns = ['model', 'scenario', 'variable', 'region', 'unit', - 'year', 'value', 'time', 'version'] + logger.debug(f'Response is {len(r.content)} bytes, {len(df)} records') + cols = IAMC_IDX + ['year', 'value', 'subannual', 'version'] # keep only known columns or init empty df - df = ( - pd.DataFrame(data=df, columns=columns) - # TODO: refactor API to directly return column as 'subannual' - .rename(columns={'time': 'subannual'}) - ) + df = pd.DataFrame(data=df, columns=cols) # check if timeseries data has subannual disaggregation, drop if not if 'subannual' in df: - if all([i in [-1, np.nan, 'Year'] for i in df.subannual.unique()]): + timeslices = df.subannual.dropna().unique() + if all([i in [-1, 'Year'] for i in timeslices]): df.drop(columns='subannual', inplace=True) # check if there are multiple version for any model/scenario @@ -469,6 +464,7 @@ def query(self, **kwargs): df[META_IDX + ['version']].drop_duplicates() .groupby(META_IDX).count().version ) + # checking if there are multiple versions # for every model/scenario combination if len(lst) > 1 and max(lst) > 1: @@ -476,7 +472,7 @@ def query(self, **kwargs): lst[lst > 1].index.to_list())) df.drop(columns='version', inplace=True) - return df + return IamDataFrame(df) def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 35d05fb1e..143383e8b 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -8,6 +8,7 @@ import pandas.testing as pdt from pyam import iiasa, META_IDX +from pyam.testing import assert_iamframe_equal from conftest import IIASA_UNAVAILABLE, TEST_API, TEST_API_NAME if IIASA_UNAVAILABLE: @@ -160,98 +161,12 @@ def test_meta(conn, default): check_dtype=False) -def test_query(conn): +def test_query(conn, test_df_year): # test reading timeseries data - df = conn.query() - print(df) - -QUERY_DATA_EXP = { - "filters": { - "regions": [], - "variables": [], - "runs": [], - "years": [], - "units": [], - "timeslices": [] - } -} - - -def test_query_data_model_scen(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn._query_post_data(model='AIM*', scenario='ADVANCE_2020_Med2C') - exp = copy.deepcopy(QUERY_DATA_EXP) - exp['filters']['runs'] = [2] - assert obs == exp - - -def test_query_data_region(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn._query_post_data(model='AIM*', scenario='ADVANCE_2020_Med2C', - region='*World*') - exp = copy.deepcopy(QUERY_DATA_EXP) - exp['filters']['runs'] = [2] - exp['filters']['regions'] = ['World'] - assert obs == exp - - -def test_query_data_variables(): - conn = iiasa.Connection('IXSE_SR15') - obs = conn._query_post_data(model='AIM*', scenario='ADVANCE_2020_Med2C', - variable='Emissions|CO2*') - exp = copy.deepcopy(QUERY_DATA_EXP) - exp['filters']['runs'] = [2] - exp['filters']['variables'] = [ - 'Emissions|CO2', 'Emissions|CO2|AFOLU', 'Emissions|CO2|Energy', - 'Emissions|CO2|Energy and Industrial Processes', - 'Emissions|CO2|Energy|Demand', 'Emissions|CO2|Energy|Demand|AFOFI', - 'Emissions|CO2|Energy|Demand|Industry', - 'Emissions|CO2|Energy|Demand|Other Sector', - 'Emissions|CO2|Energy|Demand|Residential and Commercial', - 'Emissions|CO2|Energy|Demand|Transportation', - 'Emissions|CO2|Energy|Supply', - 'Emissions|CO2|Energy|Supply|Electricity', - 'Emissions|CO2|Energy|Supply|Gases', - 'Emissions|CO2|Energy|Supply|Heat', - 'Emissions|CO2|Energy|Supply|Liquids', - 'Emissions|CO2|Energy|Supply|Other Sector', - 'Emissions|CO2|Energy|Supply|Solids', - 'Emissions|CO2|Industrial Processes', 'Emissions|CO2|Other' - ] - for k in obs['filters']: - npt.assert_array_equal(obs['filters'][k], exp['filters'][k]) - - -def test_query_IXSE_SR15(): - df = iiasa.read_iiasa('IXSE_SR15', - model='AIM*', - scenario='ADVANCE_2020_Med2C', - variable='Emissions|CO2', - region='World', - ) - assert len(df) == 20 - - -def test_query_IXSE_AR6(): - with pytest.raises(RuntimeError) as excinfo: - variable = 'Emissions|CO2|Energy|Demand|Transportation' - creds = dict(username='mahamba', password='verysecret') - iiasa.read_iiasa('IXSE_AR6', - scenario='ADVANCE_2020_WB2C', - model='AIM/CGE 2.0', - region='World', - variable=variable, - creds=creds) - assert str(excinfo.value).startswith('Login failed for user: mahamba') - - -def test_query_IXSE_SR15_with_metadata(): - df = iiasa.read_iiasa('IXSE_SR15', - model='MESSAGEix*', - variable=['Emissions|CO2', 'Primary Energy|Coal'], - region='World', - meta=['carbon price|2100 (NPV)', 'category'], - ) - assert len(df) == 168 - assert len(df.data) == 168 - assert len(df.meta) == 7 + df = conn.query(model='model_a') + assert_iamframe_equal(df, test_df_year) + + + + + From 4ddc24ff930fd294c690278e15cce277c7909b73 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 13:23:40 +0200 Subject: [PATCH 14/24] add `default` kwarg to `query()` --- pyam/iiasa.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 4a84547a1..67d5a7390 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -409,15 +409,22 @@ def _match(data, patterns): } return data - def query(self, **kwargs): + def query(self, default=True, **kwargs): """Query the connected resource for timeseries data (with filters) - Available keyword arguments include + Parameters + ---------- + default : bool, optional + Return *only* the default version of each scenario. + Any (`model`, `scenario`) without a default version is omitted. + If :obj:`False`, return all versions. + kwargs + Available keyword arguments include - - model - - scenario - - region - - variable + - model + - scenario + - region + - variable Returns ------- @@ -425,16 +432,19 @@ def query(self, **kwargs): Examples -------- - You can read from a :class:`pyam.iiasa.Connection` instance using keyword arguments similar to filtering an :class:`IamDataFrame`: .. code-block:: python - Connection.query(model='MESSAGE', scenario='SSP2*', + Connection.query(model='MESSAGE*', scenario='SSP2*', variable=['Emissions|CO2', 'Primary Energy']) """ + if default is not True: + msg = 'Querying for non-default scenarios is not (yet) supported' + raise ValueError(msg) + headers = { 'Authorization': 'Bearer {}'.format(self._token), 'Content-Type': 'application/json', From 81b43fb156eafd810104c4a63030f75846ff91bb Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 14:14:40 +0200 Subject: [PATCH 15/24] add tests for query with kwargs and subannual data --- pyam/iiasa.py | 1 + tests/test_iiasa.py | 46 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 67d5a7390..6e3ec9a54 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -441,6 +441,7 @@ def query(self, default=True, **kwargs): variable=['Emissions|CO2', 'Primary Energy']) """ + # TODO: API returns non-default versions if default is not True: msg = 'Querying for non-default scenarios is not (yet) supported' raise ValueError(msg) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 143383e8b..4e462eec1 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -7,7 +7,7 @@ import numpy.testing as npt import pandas.testing as pdt -from pyam import iiasa, META_IDX +from pyam import IamDataFrame, iiasa, META_IDX from pyam.testing import assert_iamframe_equal from conftest import IIASA_UNAVAILABLE, TEST_API, TEST_API_NAME @@ -30,6 +30,13 @@ ['model_b', 'scen_a', 1, True, 3, 'baz'] ], columns=META_IDX+['version', 'is_default']+META_COLS).set_index(META_IDX) +MODEL_B_DF = pd.DataFrame([ + ['Primary Energy', 'EJ/yr', 'Summer', 1, 3], + ['Primary Energy', 'EJ/yr', 'Year', 3, 8], + ['Primary Energy|Coal', 'EJ/yr', 'Summer', 0.4, 2], + ['Primary Energy|Coal', 'EJ/yr', 'Year', 0.9, 5] +], columns=['variable', 'unit', 'subannual', 2005, 2010]) + def test_unknown_conn(): # connecting to an unknown API raises an error @@ -160,13 +167,30 @@ def test_meta(conn, default): pdt.assert_frame_equal(conn.metadata(default=default), exp, check_dtype=False) - -def test_query(conn, test_df_year): - # test reading timeseries data - df = conn.query(model='model_a') - assert_iamframe_equal(df, test_df_year) - - - - - +@pytest.mark.parametrize("kwargs", [ + {}, + dict(variable='Primary Energy'), + dict(scenario='scen_a', variable='Primary Energy') +]) +def test_query_year(conn, test_df_year, kwargs): + # test reading timeseries data (`model_a` has only yearly data) + df = conn.query(model='model_a', **kwargs) + assert_iamframe_equal(df, test_df_year.filter(**kwargs)) + + +@pytest.mark.parametrize("kwargs", [ + {}, + dict(variable='Primary Energy'), + dict(scenario='scen_a', variable='Primary Energy') +]) +def test_query_with_subannual(conn, test_pd_df, kwargs): + # test reading timeseries data (including subannual data) + exp = IamDataFrame(test_pd_df, subannual='Year')\ + .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') + df = conn.query(**kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + + +def test_query_non_default(conn): + # querying for non-default scenario data raises an error + pytest.raises(ValueError, conn.query, default=False) From deca4665e578a6ccbadcd4d9175a7383e33fa976 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 21:16:58 +0200 Subject: [PATCH 16/24] make `assert_iamframe_equal` more lenient --- pyam/testing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyam/testing.py b/pyam/testing.py index 941d15a11..cc44a421e 100644 --- a/pyam/testing.py +++ b/pyam/testing.py @@ -8,4 +8,5 @@ def assert_iamframe_equal(a, b, **assert_kwargs): msg = 'IamDataFrame.data are different: \n {}' raise AssertionError(msg.format(diff.head())) - pdt.assert_frame_equal(a.meta, b.meta, **assert_kwargs) + pdt.assert_frame_equal(a.meta, b.meta, check_dtype=False, check_like=True, + **assert_kwargs) From 8e0dbd67b058f98b2505a71f2c7940f967c7690a Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 21:17:37 +0200 Subject: [PATCH 17/24] merge `meta` to query output --- pyam/iiasa.py | 68 ++++++++++++++++++++++++--------------------- tests/test_iiasa.py | 9 +++++- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 6e3ec9a54..474b55a3e 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -409,7 +409,7 @@ def _match(data, patterns): } return data - def query(self, default=True, **kwargs): + def query(self, default=True, meta=True, **kwargs): """Query the connected resource for timeseries data (with filters) Parameters @@ -418,6 +418,9 @@ def query(self, default=True, **kwargs): Return *only* the default version of each scenario. Any (`model`, `scenario`) without a default version is omitted. If :obj:`False`, return all versions. + meta : bool or list, optional + If :obj:`True`, merge all meta columns indicators + (or subset if list is given). kwargs Available keyword arguments include @@ -441,52 +444,64 @@ def query(self, default=True, **kwargs): variable=['Emissions|CO2', 'Primary Energy']) """ - # TODO: API returns non-default versions + # TODO: API returns timeseries data for non-default versions if default is not True: msg = 'Querying for non-default scenarios is not (yet) supported' raise ValueError(msg) + # retrieve data headers = { 'Authorization': 'Bearer {}'.format(self._token), 'Content-Type': 'application/json', } - data = json.dumps(self._query_post_data(**kwargs)) + _args = json.dumps(self._query_post_data(**kwargs)) url = '/'.join([self._auth_url, 'runs/bulk/ts']) - logger.debug(f'Querying timeseries data from {url} with filter {data}') - r = requests.post(url, headers=headers, data=data) + logger.debug(f'Query timeseries data from {url} with data {_args}') + r = requests.post(url, headers=headers, data=_args) _check_response(r) # refactor returned json object to be castable to an IamDataFrame dtype = dict(model=str, scenario=str, variable=str, unit=str, region=str, year=int, value=float, version=int) - df = pd.read_json(r.content, orient='records', dtype=dtype) - logger.debug(f'Response is {len(r.content)} bytes, {len(df)} records') + data = pd.read_json(r.content, orient='records', dtype=dtype) + logger.debug(f'Response: {len(r.content)} bytes, {len(data)} records') cols = IAMC_IDX + ['year', 'value', 'subannual', 'version'] # keep only known columns or init empty df - df = pd.DataFrame(data=df, columns=cols) + data = pd.DataFrame(data=data, columns=cols) # check if timeseries data has subannual disaggregation, drop if not - if 'subannual' in df: - timeslices = df.subannual.dropna().unique() + if 'subannual' in data: + timeslices = data.subannual.dropna().unique() if all([i in [-1, 'Year'] for i in timeslices]): - df.drop(columns='subannual', inplace=True) + data.drop(columns='subannual', inplace=True) # check if there are multiple version for any model/scenario lst = ( - df[META_IDX + ['version']].drop_duplicates() + data[META_IDX + ['version']].drop_duplicates() .groupby(META_IDX).count().version ) # checking if there are multiple versions # for every model/scenario combination + # TODO this is probably not necessary if len(lst) > 1 and max(lst) > 1: raise ValueError('multiple versions for {}'.format( lst[lst > 1].index.to_list())) - df.drop(columns='version', inplace=True) + data.drop(columns='version', inplace=True) + + # cast to IamDataFrame + df = IamDataFrame(data) + + # merge meta categorization and quantitative indications + if meta: + _meta = self.meta().loc[df.meta.index] + for i in _meta.columns if meta is True else meta: + df.set_meta(_meta[i]) return IamDataFrame(df) -def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): +def read_iiasa(name, default=True, meta=False, creds=None, base_url=_AUTH_URL, + **kwargs): """Query an IIASA Scenario Explorer database API and return as IamDataFrame Parameters @@ -494,7 +509,11 @@ def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): name : str A valid name of an IIASA scenario explorer instance, see :attr:`pyam.iiasa.Connection.valid_connections` - meta : bool or list of strings + default : bool, optional + Return *only* the default version of each scenario. + Any (`model`, `scenario`) without a default version is omitted. + If :obj:`False`, return all versions. + meta : bool or list of strings, optional If `True`, include all meta categories & quantitative indicators (or subset if list is given). creds : dict @@ -505,20 +524,5 @@ def read_iiasa(name, meta=False, creds=None, base_url=_AUTH_URL, **kwargs): kwargs Arguments for :meth:`pyam.iiasa.Connection.query` """ - conn = Connection(name, creds, base_url) - # data - df = IamDataFrame(conn.query(**kwargs)) - # meta: categorization and quantitative indications - if meta: - mdf = conn.metadata() - # only data for models/scenarios in df - mdf = mdf[mdf.model.isin(df['model'].unique()) & - mdf.scenario.isin(df['scenario'].unique())] - # get subset of data if meta is a list - if islistable(meta): - mdf = mdf[['model', 'scenario'] + meta] - mdf = mdf.set_index(['model', 'scenario']) - # we have to loop here because `set_meta()` can only take series - for col in mdf: - df.set_meta(mdf[col]) - return df + return Connection(name, creds, base_url)\ + .query(default=default, meta=meta, **kwargs) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 4e462eec1..ee0d197bd 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -174,8 +174,12 @@ def test_meta(conn, default): ]) def test_query_year(conn, test_df_year, kwargs): # test reading timeseries data (`model_a` has only yearly data) + exp = test_df_year.copy() + for i in ['version'] + META_COLS: + exp.set_meta(META_DF.iloc[[0, 1]][i]) + df = conn.query(model='model_a', **kwargs) - assert_iamframe_equal(df, test_df_year.filter(**kwargs)) + assert_iamframe_equal(df, exp.filter(**kwargs)) @pytest.mark.parametrize("kwargs", [ @@ -187,6 +191,9 @@ def test_query_with_subannual(conn, test_pd_df, kwargs): # test reading timeseries data (including subannual data) exp = IamDataFrame(test_pd_df, subannual='Year')\ .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') + for i in ['version'] + META_COLS: + exp.set_meta(META_DF.iloc[[0, 1, 3]][i]) + df = conn.query(**kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) From 51ea304299812f117a3dff00f3e27acd7aaf06c7 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 21:29:05 +0200 Subject: [PATCH 18/24] add tests (and fix issues) when retrieving specific/no meta columns --- pyam/iiasa.py | 4 ++-- tests/test_iiasa.py | 52 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 474b55a3e..662cc118c 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -494,13 +494,13 @@ def query(self, default=True, meta=True, **kwargs): # merge meta categorization and quantitative indications if meta: _meta = self.meta().loc[df.meta.index] - for i in _meta.columns if meta is True else meta: + for i in _meta.columns if meta is True else meta + ['version']: df.set_meta(_meta[i]) return IamDataFrame(df) -def read_iiasa(name, default=True, meta=False, creds=None, base_url=_AUTH_URL, +def read_iiasa(name, default=True, meta=True, creds=None, base_url=_AUTH_URL, **kwargs): """Query an IIASA Scenario Explorer database API and return as IamDataFrame diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index ee0d197bd..4a4ed25b6 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -7,7 +7,7 @@ import numpy.testing as npt import pandas.testing as pdt -from pyam import IamDataFrame, iiasa, META_IDX +from pyam import IamDataFrame, iiasa, read_iiasa, META_IDX from pyam.testing import assert_iamframe_equal from conftest import IIASA_UNAVAILABLE, TEST_API, TEST_API_NAME @@ -178,9 +178,14 @@ def test_query_year(conn, test_df_year, kwargs): for i in ['version'] + META_COLS: exp.set_meta(META_DF.iloc[[0, 1]][i]) + # test method via Connection df = conn.query(model='model_a', **kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) + # test top-level method + df = read_iiasa(TEST_API, model='model_a', **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + @pytest.mark.parametrize("kwargs", [ {}, @@ -194,9 +199,54 @@ def test_query_with_subannual(conn, test_pd_df, kwargs): for i in ['version'] + META_COLS: exp.set_meta(META_DF.iloc[[0, 1, 3]][i]) + # test method via Connection df = conn.query(**kwargs) assert_iamframe_equal(df, exp.filter(**kwargs)) + # test top-level method + df = read_iiasa(TEST_API, **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + + +@pytest.mark.parametrize("kwargs", [ + {}, + dict(variable='Primary Energy'), + dict(scenario='scen_a', variable='Primary Energy') +]) +def test_query_with_meta_arg(conn, test_pd_df, kwargs): + # test reading timeseries data (including subannual data) + exp = IamDataFrame(test_pd_df, subannual='Year')\ + .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') + for i in ['version', 'string']: + exp.set_meta(META_DF.iloc[[0, 1, 3]][i]) + + # test method via Connection + df = conn.query(meta=['string'], **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + + # test top-level method + df = read_iiasa(TEST_API, meta=['string'], **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + + +@pytest.mark.parametrize("kwargs", [ + {}, + dict(variable='Primary Energy'), + dict(scenario='scen_a', variable='Primary Energy') +]) +def test_query_with_meta_false(conn, test_pd_df, kwargs): + # test reading timeseries data (including subannual data) + exp = IamDataFrame(test_pd_df, subannual='Year')\ + .append(MODEL_B_DF, model='model_b', scenario='scen_a', region='World') + + # test method via Connection + df = conn.query(meta=False, **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + + # test top-level method + df = read_iiasa(TEST_API, meta=False, **kwargs) + assert_iamframe_equal(df, exp.filter(**kwargs)) + def test_query_non_default(conn): # querying for non-default scenario data raises an error From 49b53e03506f730382e4fc5acc2a92b30da6fab4 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 21:46:24 +0200 Subject: [PATCH 19/24] add test for valid connections --- tests/test_iiasa.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 4a4ed25b6..8393579d5 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -43,6 +43,11 @@ def test_unknown_conn(): pytest.raises(ValueError, iiasa.Connection, 'foo') +def test_valid_connections(): + # connecting to an unknown API raises an error + assert TEST_API in iiasa.Connection().valid_connections + + def test_anon_conn(conn): assert conn.current_connection == TEST_API_NAME From 220c0c6a8162fe963c233b35edf8b257c771a052 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Sat, 4 Jul 2020 21:47:19 +0200 Subject: [PATCH 20/24] update tutorial notebook for connecting to an IIASA resource --- doc/source/tutorials/iiasa_dbs.ipynb | 57 ++++++++++++++++++---------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/doc/source/tutorials/iiasa_dbs.ipynb b/doc/source/tutorials/iiasa_dbs.ipynb index 232c13433..080c7fdf8 100644 --- a/doc/source/tutorials/iiasa_dbs.ipynb +++ b/doc/source/tutorials/iiasa_dbs.ipynb @@ -10,7 +10,7 @@ "High-profile use cases include the [IAMC 1.5°C Scenario Explorer hosted by IIASA](https://data.ene.iiasa.ac.at/iamc-1.5c-explorer) supporting the *IPCC Special Report on Global Warming of 1.5°C* (SR15) and the Horizon 2020 project [CD-LINKS](https://data.ene.iiasa.ac.at/cd-links).\n", "\n", "IIASA's [modeling platform infrastructure](http://software.ene.iiasa.ac.at/ixmp-server) and the Scenario Explorer UI is not only a great resource on its own, but it also allows the underlying datasets to be directly queried.\n", - "**pyam** takes advantage of this ability to allow you to easily pull data and work with it." + "**pyam** takes advantage of this ability to allow you to easily pull data and work with it in your Python data processing and analysis workflow." ] }, { @@ -26,7 +26,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Accessing an explorer is done via a `Connection` object.\n", + "## Connecting to a data resource (aka the database API of a Scenario Explorer instance)\n", + "\n", + "Accessing a data resource is done via a **Connection** object.\n", "By default, your can connect to all public scenario explorers instances. " ] }, @@ -44,21 +46,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If you have credentials to connect to a non-public or restricted database,\n", - "you can set this (in a separate Python console) using the following command:\n", + "If you have credentials to connect to a non-public or restricted Scenario Explorer instance,\n", + "you can store this information by running the following command in a separate Python console:\n", "\n", "```\n", "import pyam\n", "pyam.iiasa.set_config(, )\n", "```\n", - "When initializing a new `Connection` instance, **pyam** will automatically search for the configuration in a known location." + "When initializing a new **Connection** instance, **pyam** will automatically search for the configuration in a known location." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "In this example, we will be pulling data from the Special Report on 1.5C explorer. This can be done either via the constructor:\n", + "In this example, we will be retrieving data from the *IAMC 1.5°C Scenario Explorer hosted by IIASA*\n", + "([link](https://data.ene.iiasa.ac.at/iamc-1.5c-explorer)),\n", + "which provides the quantiative scenario ensemble underpinning\n", + "the *IPCC Special Report on Global Warming of 1.5C* (SR15).\n", + "\n", + "This can be done either via the constructor:\n", "\n", "```\n", "pyam.iiasa.Connection('iamc15')\n", @@ -76,7 +83,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We also provide some convenience functions to shorten the amount of code you have to write. Under the hood, `read_iiasa()` is just opening an connection to a database and making a query on that data.\n", + "We also provide some convenience functions to shorten the amount of code you have to write. Under the hood, `read_iiasa()` is just opening an connection to a database API and sends a query to the resource.\n", + "\n", "In this tutorial, we will query specific subsets of data in a manner similar to `pyam.IamDataFrame.filter()`." ] }, @@ -99,7 +107,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Here we pulled out all times series data for model(s) that start with 'MESSAGEix' that are in the 'World' region and associated with the two named variables. We also added the \"category\" metadata, which tells us the climate impact categorisation of each scenario as assessed in the IPCC SR15.\n", + "Here we pulled out all times series data for model(s) that start with 'MESSAGEix' that are in the 'World' region and associated with the two named variables. We also added the meta column \"category\", which tells us the climate impact categorisation of each scenario as assessed in the IPCC SR15.\n", "\n", "Let's plot CO2 emissions." ] @@ -143,7 +151,7 @@ "source": [ "## Exploring the data resource\n", "\n", - "If you're interested in what data is actually in the data source, you can use **pyam.iiasa.Connection** to do so." + "If you're interested in what data is available in the data source, you can use **pyam.iiasa.Connection** to do so." ] }, { @@ -159,7 +167,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `conn` object has a number of useful functions for listing what's in the dataset. A few of them are shown below." + "The **Connection** object has a number of useful functions for listing what's available in the data resource.\n", + "These functions follow the conventions of the **IamDataFrame** class (where possible).\n", + "\n", + "A few of them are shown below." ] }, { @@ -202,8 +213,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A number of different kinds of indicators are available for model/scenario combinations.\n", - "We queried the \"category\" metadata in the above example, but there are many more. You can see them with" + "A number of different categorization and quantitative indicators are available for model/scenario combinations.\n", + "These are usually called `meta` indicators in **pyam**.\n", + "\n", + "We queried the meta-indicator \"category\" in the above example, but there are many more.\n", + "You can get a list with the following command:" ] }, { @@ -212,14 +226,14 @@ "metadata": {}, "outputs": [], "source": [ - "conn.available_metadata().head()" + "conn.meta_columns.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can directly query the **Connection**, which will give you a [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)." + "You can directly query the **Connection**, which will return a **pyam.IamDataFrame**..." ] }, { @@ -232,15 +246,14 @@ " model='MESSAGEix*', \n", " variable=['Emissions|CO2', 'Primary Energy|Coal'], \n", " region='World'\n", - ")\n", - "df.head()" + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "And you can easily turn this into a **pyam.IamDataFrame** to continue your analysis." + "...so that you can directly continue with your analysis and visualization workflow using **pyam**!" ] }, { @@ -249,12 +262,18 @@ "metadata": {}, "outputs": [], "source": [ - "df = pyam.IamDataFrame(df)\n", "ax = df.filter(variable='Primary Energy|Coal').line_plot(\n", " color='scenario', \n", " legend=dict(loc='center left', bbox_to_anchor=(1.0, 0.5))\n", ")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -273,7 +292,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.7" } }, "nbformat": 4, From 7201ff2bb62f4af28628759236dcb34ef8393514 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 6 Jul 2020 19:24:37 +0200 Subject: [PATCH 21/24] appease stickler --- pyam/iiasa.py | 4 ++-- tests/conftest.py | 2 +- tests/test_iiasa.py | 12 +++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 662cc118c..a280178e0 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -11,7 +11,7 @@ from collections.abc import Mapping from pyam.core import IamDataFrame -from pyam.utils import META_IDX, IAMC_IDX, islistable, isstr, pattern_match +from pyam.utils import META_IDX, IAMC_IDX, isstr, pattern_match from pyam.logging import deprecation_warning logger = logging.getLogger(__name__) @@ -277,7 +277,7 @@ def meta(self, default=True): def extract(row): return ( - pd.concat([row[META_IDX+cols], + pd.concat([row[META_IDX + cols], pd.Series(row.metadata)]) .to_frame() .T diff --git a/tests/conftest.py b/tests/conftest.py index 325830a88..db833a983 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -189,4 +189,4 @@ def plot_stack_plot_df(): @pytest.fixture(scope="session") def conn(): if not IIASA_UNAVAILABLE: - return iiasa.Connection(TEST_API) \ No newline at end of file + return iiasa.Connection(TEST_API) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 8393579d5..babf4a2bc 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -22,18 +22,19 @@ TEST_ENV_USER, TEST_ENV_PW ) +VERSION_COLS = ['version', 'is_default'] META_COLS = ['number', 'string'] META_DF = pd.DataFrame([ ['model_a', 'scen_a', 1, True, 1, 'foo'], ['model_a', 'scen_b', 1, True, 2, np.nan], ['model_a', 'scen_a', 2, False, 1, 'bar'], ['model_b', 'scen_a', 1, True, 3, 'baz'] -], columns=META_IDX+['version', 'is_default']+META_COLS).set_index(META_IDX) +], columns=META_IDX + VERSION_COLS + META_COLS).set_index(META_IDX) MODEL_B_DF = pd.DataFrame([ ['Primary Energy', 'EJ/yr', 'Summer', 1, 3], ['Primary Energy', 'EJ/yr', 'Year', 3, 8], - ['Primary Energy|Coal', 'EJ/yr', 'Summer', 0.4, 2], + ['Primary Energy|Coal', 'EJ/yr', 'Summer', 0.4, 2], ['Primary Energy|Coal', 'EJ/yr', 'Year', 0.9, 5] ], columns=['variable', 'unit', 'subannual', 2005, 2010]) @@ -85,7 +86,6 @@ def test_conn_creds_dict_raises(): pytest.raises(KeyError, iiasa.Connection, TEST_API, creds=creds) - def test_variables(conn): # check that connection returns the correct variables npt.assert_array_equal(conn.variables(), @@ -147,13 +147,14 @@ def test_meta_columns(conn): # test for deprecated version of the function npt.assert_array_equal(conn.available_metadata(), META_COLS) + @pytest.mark.parametrize("default", [True, False]) def test_index(conn, default): # test that connection returns the correct index if default: exp = META_DF.loc[META_DF.is_default, ['version']] else: - exp = META_DF[['version', 'is_default']] + exp = META_DF[VERSION_COLS] pdt.assert_frame_equal(conn.index(default=default), exp, check_dtype=False) @@ -164,7 +165,7 @@ def test_meta(conn, default): if default: exp = META_DF.loc[META_DF.is_default, ['version'] + META_COLS] else: - exp = META_DF[['version', 'is_default'] + META_COLS] + exp = META_DF[VERSION_COLS + META_COLS] pdt.assert_frame_equal(conn.meta(default=default), exp, check_dtype=False) @@ -172,6 +173,7 @@ def test_meta(conn, default): pdt.assert_frame_equal(conn.metadata(default=default), exp, check_dtype=False) + @pytest.mark.parametrize("kwargs", [ {}, dict(variable='Primary Energy'), From e0e6ad6b42a455bc66ca17b06925a065a8ae3efa Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 6 Jul 2020 19:26:10 +0200 Subject: [PATCH 22/24] add to release notes --- RELEASE_NOTES.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e4f9a2eb3..57a64d1d4 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,6 +2,16 @@ ## API changes +PR [#413](https://github.com/IAMconsortium/pyam/pull/413) changed the +return type of `pyam.read_iiasa()` and `pyam.iiasa.Connection.query()` +to an `IamDataFrame` (instead of a `pandas.DataFrame`) +and loads meta-indicators by default. + +Also, the following functions were deprecated for package consistency: +- `index()` replaces `scenario_list()` for an overview of all scenarios +- `meta_columns` (attribute) replaces `available_metadata()` +- `meta()` replaces `metadata()` + PR [#402](https://github.com/IAMconsortium/pyam/pull/402) changed the default behaviour of `as_pandas()` to include all columns of `meta` in the returned dataframe, or only merge columns given by the renamed argument `meta_cols`. @@ -10,6 +20,7 @@ a utility function `pyam.plotting.mpl_args_to_meta_cols()`. ## Individual Updates +- [#413](https://github.com/IAMconsortium/pyam/pull/413) Refactor IIASA-connection-API and rework all related tests. - [#412](https://github.com/IAMconsortium/pyam/pull/412) Add building the docs to GitHub Actions CI. - [#410](https://github.com/IAMconsortium/pyam/pull/410) Activate tutorial tests on GitHub Actions CI (py3.8). - [#409](https://github.com/IAMconsortium/pyam/pull/409) Remove travis and appveyor CI config. From dfcd025147c9b1133823e1886a83b28a0c46e2f3 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 6 Jul 2020 19:28:27 +0200 Subject: [PATCH 23/24] appease stickler more --- tests/test_iiasa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index babf4a2bc..6ae3ef43c 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -86,6 +86,7 @@ def test_conn_creds_dict_raises(): pytest.raises(KeyError, iiasa.Connection, TEST_API, creds=creds) + def test_variables(conn): # check that connection returns the correct variables npt.assert_array_equal(conn.variables(), From 6253c793bd8e8b1b2b980b5579278935a88bc8a9 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Tue, 7 Jul 2020 07:23:17 +0200 Subject: [PATCH 24/24] fix a typo in the tutorial notebook (per suggestion by @znicholls) Co-authored-by: Zeb Nicholls --- doc/source/tutorials/iiasa_dbs.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tutorials/iiasa_dbs.ipynb b/doc/source/tutorials/iiasa_dbs.ipynb index 080c7fdf8..1e324eb7a 100644 --- a/doc/source/tutorials/iiasa_dbs.ipynb +++ b/doc/source/tutorials/iiasa_dbs.ipynb @@ -83,7 +83,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We also provide some convenience functions to shorten the amount of code you have to write. Under the hood, `read_iiasa()` is just opening an connection to a database API and sends a query to the resource.\n", + "We also provide some convenience functions to shorten the amount of code you have to write. Under the hood, `read_iiasa()` is just opening a connection to a database API and sends a query to the resource.\n", "\n", "In this tutorial, we will query specific subsets of data in a manner similar to `pyam.IamDataFrame.filter()`." ]