From 185e3048c62bffe127419df137e249145d473f08 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Wed, 16 Nov 2022 13:10:15 +0000 Subject: [PATCH 01/22] Initial reorganisation Tests have been organised into ones that immediately work with no icat stack and ones that don't. --- test/conftest.py | 83 ------------- .../__init__.py | 0 test/integration/conftest.py | 85 +++++++++++++ .../datagateway_api}/__init__.py | 0 .../datagateway_api/db/conftest.py | 0 .../endpoints/test_count_with_filters_db.py | 0 .../db/endpoints/test_findone_db.py | 0 .../db/endpoints/test_get_by_id_db.py | 0 .../db/endpoints/test_get_with_filters.py | 0 .../db/endpoints/test_ping_db.py | 0 .../db/endpoints/test_table_endpoints_db.py | 0 .../db/test_database_filter_utilities.py | 0 .../datagateway_api/db/test_entity_helper.py | 0 .../db/test_requires_session_id.py | 0 .../datagateway_api/icat/conftest.py | 8 +- .../endpoints/test_count_with_filters_icat.py | 0 .../icat/endpoints/test_create_icat.py | 4 +- .../icat/endpoints/test_delete_by_id_icat.py | 0 .../icat/endpoints/test_findone_icat.py | 4 +- .../icat/endpoints/test_get_by_id_icat.py | 4 +- .../endpoints/test_get_with_filters_icat.py | 4 +- .../icat/endpoints/test_ping_icat.py | 0 .../endpoints/test_table_endpoints_icat.py | 4 +- .../icat/endpoints/test_update_by_id_icat.py | 4 +- .../endpoints/test_update_multiple_icat.py | 4 +- .../icat/filters/test_distinct_filter.py | 0 .../icat/filters/test_include_filter.py | 0 .../icat/filters/test_limit_filter.py | 0 .../icat/filters/test_order_filter.py | 0 .../icat/filters/test_skip_filter.py | 0 .../icat/filters/test_where_filter.py | 0 .../datagateway_api/icat/test_helpers.py | 0 .../datagateway_api/icat/test_icat_client.py | 0 .../datagateway_api/icat/test_lru_cache.py | 0 .../datagateway_api/icat/test_query.py | 0 .../icat/test_session_handling.py | 0 .../datagateway_api/test_backends.py | 0 .../test_query_filter_factory.py | 0 test/integration/search_api/__init__.py | 0 test/{ => integration}/search_api/conftest.py | 0 .../endpoints/test_count_dataset_files.py | 0 .../endpoints/test_count_endpoint.py | 0 .../endpoints/test_get_dataset_files.py | 0 .../endpoints/test_get_entity_by_pid.py | 0 .../endpoints/test_search_endpoint.py | 0 .../filters/test_search_api_include_filter.py | 0 .../filters/test_search_api_limit_filter.py | 0 .../filters/test_search_api_skip_filter.py | 0 .../filters/test_search_api_where_filter.py | 0 .../search_api/test_error_handling.py | 0 .../search_api/test_nested_where_filters.py | 0 .../search_api/test_search_api_query.py | 0 .../test_search_api_query_filter_factory.py | 0 .../search_api/test_session_handler.py | 0 test/integration/test_config.py | 9 ++ test/{ => integration}/test_endpoint_rules.py | 0 .../test_filter_order_handler.py | 0 .../test_get_filters_from_query.py | 6 +- .../test_get_session_id_from_auth_header.py | 0 test/unit/__init__.py | 0 .../datagateway_api/test_openapi.py | 0 test/unit/search_api/conftest.py | 115 ++++++++++++++++++ test/{ => unit}/search_api/test_models.py | 0 .../search_api/test_panosc_mappings.py | 27 ---- .../test_base_query_filter_factory.py | 0 test/{ => unit}/test_config.py | 9 -- test/{ => unit}/test_date_handler.py | 0 test/{ => unit}/test_exceptions.py | 0 test/{ => unit}/test_get_entity_object.py | 0 test/unit/test_get_filters_from_query.py | 10 ++ test/{ => unit}/test_is_valid_json.py | 0 test/{ => unit}/test_map_distinct_attrs.py | 0 test/{ => unit}/test_queries_records.py | 0 test/{ => unit}/test_query_filter.py | 0 74 files changed, 247 insertions(+), 133 deletions(-) rename test/{datagateway_api => integration}/__init__.py (100%) create mode 100644 test/integration/conftest.py rename test/{search_api => integration/datagateway_api}/__init__.py (100%) rename test/{ => integration}/datagateway_api/db/conftest.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_count_with_filters_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_findone_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_get_by_id_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_get_with_filters.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_ping_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_table_endpoints_db.py (100%) rename test/{ => integration}/datagateway_api/db/test_database_filter_utilities.py (100%) rename test/{ => integration}/datagateway_api/db/test_entity_helper.py (100%) rename test/{ => integration}/datagateway_api/db/test_requires_session_id.py (100%) rename test/{ => integration}/datagateway_api/icat/conftest.py (96%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_count_with_filters_icat.py (100%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_create_icat.py (98%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_delete_by_id_icat.py (100%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_findone_icat.py (92%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_get_by_id_icat.py (95%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_get_with_filters_icat.py (96%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_ping_icat.py (100%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_table_endpoints_icat.py (97%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_update_by_id_icat.py (94%) rename test/{ => integration}/datagateway_api/icat/endpoints/test_update_multiple_icat.py (98%) rename test/{ => integration}/datagateway_api/icat/filters/test_distinct_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/filters/test_include_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/filters/test_limit_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/filters/test_order_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/filters/test_skip_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/filters/test_where_filter.py (100%) rename test/{ => integration}/datagateway_api/icat/test_helpers.py (100%) rename test/{ => integration}/datagateway_api/icat/test_icat_client.py (100%) rename test/{ => integration}/datagateway_api/icat/test_lru_cache.py (100%) rename test/{ => integration}/datagateway_api/icat/test_query.py (100%) rename test/{ => integration}/datagateway_api/icat/test_session_handling.py (100%) rename test/{ => integration}/datagateway_api/test_backends.py (100%) rename test/{ => integration}/datagateway_api/test_query_filter_factory.py (100%) create mode 100644 test/integration/search_api/__init__.py rename test/{ => integration}/search_api/conftest.py (100%) rename test/{ => integration}/search_api/endpoints/test_count_dataset_files.py (100%) rename test/{ => integration}/search_api/endpoints/test_count_endpoint.py (100%) rename test/{ => integration}/search_api/endpoints/test_get_dataset_files.py (100%) rename test/{ => integration}/search_api/endpoints/test_get_entity_by_pid.py (100%) rename test/{ => integration}/search_api/endpoints/test_search_endpoint.py (100%) rename test/{ => integration}/search_api/filters/test_search_api_include_filter.py (100%) rename test/{ => integration}/search_api/filters/test_search_api_limit_filter.py (100%) rename test/{ => integration}/search_api/filters/test_search_api_skip_filter.py (100%) rename test/{ => integration}/search_api/filters/test_search_api_where_filter.py (100%) rename test/{ => integration}/search_api/test_error_handling.py (100%) rename test/{ => integration}/search_api/test_nested_where_filters.py (100%) rename test/{ => integration}/search_api/test_search_api_query.py (100%) rename test/{ => integration}/search_api/test_search_api_query_filter_factory.py (100%) rename test/{ => integration}/search_api/test_session_handler.py (100%) create mode 100644 test/integration/test_config.py rename test/{ => integration}/test_endpoint_rules.py (100%) rename test/{ => integration}/test_filter_order_handler.py (100%) rename test/{ => integration}/test_get_filters_from_query.py (91%) rename test/{ => integration}/test_get_session_id_from_auth_header.py (100%) create mode 100644 test/unit/__init__.py rename test/{ => unit}/datagateway_api/test_openapi.py (100%) create mode 100644 test/unit/search_api/conftest.py rename test/{ => unit}/search_api/test_models.py (100%) rename test/{ => unit}/search_api/test_panosc_mappings.py (90%) rename test/{ => unit}/test_base_query_filter_factory.py (100%) rename test/{ => unit}/test_config.py (91%) rename test/{ => unit}/test_date_handler.py (100%) rename test/{ => unit}/test_exceptions.py (100%) rename test/{ => unit}/test_get_entity_object.py (100%) create mode 100644 test/unit/test_get_filters_from_query.py rename test/{ => unit}/test_is_valid_json.py (100%) rename test/{ => unit}/test_map_distinct_attrs.py (100%) rename test/{ => unit}/test_queries_records.py (100%) rename test/{ => unit}/test_query_filter.py (100%) diff --git a/test/conftest.py b/test/conftest.py index d80c14b1..667a9506 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,35 +1,6 @@ -from datetime import datetime, timedelta -import json -from unittest.mock import mock_open, patch - -from flask import Flask -from icat.client import Client from icat.query import Query import pytest -from datagateway_api.src.api_start_utils import ( - create_api_endpoints, - create_app_infrastructure, -) -from datagateway_api.src.common.config import APIConfig, Config -from datagateway_api.src.datagateway_api.database.helpers import ( - delete_row_by_id, - insert_row_into_table, -) -from datagateway_api.src.datagateway_api.database.models import SESSION - - -@pytest.fixture(scope="package") -def icat_client(): - client = Client( - Config.config.datagateway_api.icat_url, - checkCert=Config.config.datagateway_api.icat_check_cert, - ) - client.login( - Config.config.test_mechanism, Config.config.test_user_credentials.dict(), - ) - return client - @pytest.fixture() def icat_query(icat_client): @@ -46,47 +17,6 @@ def invalid_credentials_header(): return {"Authorization": "Test"} -@pytest.fixture(scope="package") -def flask_test_app(): - """This is used to check the endpoints exist and have the correct HTTP methods""" - test_app = Flask(__name__) - api, spec = create_app_infrastructure(test_app) - create_api_endpoints(test_app, api, spec) - - yield test_app - - -@pytest.fixture(scope="package") -def flask_test_app_db(): - """ - This is in the common conftest file because this test app is also used in - non-backend specific tests - """ - db_app = Flask(__name__) - db_app.config["TESTING"] = True - db_app.config["TEST_BACKEND"] = "db" - - api, spec = create_app_infrastructure(db_app) - create_api_endpoints(db_app, api, spec) - db_app.app_context().push() - - yield db_app.test_client() - - -@pytest.fixture() -def valid_db_credentials_header(): - session = SESSION() - session.id = "Test" - session.expireDateTime = datetime.now() + timedelta(hours=1) - session.username = "Test User" - - insert_row_into_table(SESSION, session) - - yield {"Authorization": f"Bearer {session.id}"} - - delete_row_by_id(SESSION, "Test") - - @pytest.fixture() def test_config_data(): return { @@ -118,16 +48,3 @@ def test_config_data(): "test_user_credentials": {"username": "root", "password": "pw"}, "test_mechanism": "simple", } - - -@pytest.fixture() -def test_config(test_config_data): - with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): - return APIConfig.load("test/path") - - -@pytest.fixture() -def test_config_without_search_api(test_config_data): - del test_config_data["search_api"] - with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): - return APIConfig.load("test/path") diff --git a/test/datagateway_api/__init__.py b/test/integration/__init__.py similarity index 100% rename from test/datagateway_api/__init__.py rename to test/integration/__init__.py diff --git a/test/integration/conftest.py b/test/integration/conftest.py new file mode 100644 index 00000000..c8ac7fee --- /dev/null +++ b/test/integration/conftest.py @@ -0,0 +1,85 @@ +from datetime import datetime, timedelta +import json +from unittest.mock import mock_open, patch + +from flask import Flask +from icat.client import Client +from icat.query import Query +import pytest + +from datagateway_api.src.api_start_utils import ( + create_api_endpoints, + create_app_infrastructure, +) +from datagateway_api.src.common.config import APIConfig, Config +from datagateway_api.src.datagateway_api.database.helpers import ( + delete_row_by_id, + insert_row_into_table, +) +from datagateway_api.src.datagateway_api.database.models import SESSION + + +@pytest.fixture(scope="package") +def icat_client(): + client = Client( + Config.config.datagateway_api.icat_url, + checkCert=Config.config.datagateway_api.icat_check_cert, + ) + client.login( + Config.config.test_mechanism, Config.config.test_user_credentials.dict(), + ) + return client + + +@pytest.fixture(scope="package") +def flask_test_app(): + """This is used to check the endpoints exist and have the correct HTTP methods""" + test_app = Flask(__name__) + api, spec = create_app_infrastructure(test_app) + create_api_endpoints(test_app, api, spec) + + yield test_app + + +@pytest.fixture(scope="package") +def flask_test_app_db(): + """ + This is in the common conftest file because this test app is also used in + non-backend specific tests + """ + db_app = Flask(__name__) + db_app.config["TESTING"] = True + db_app.config["TEST_BACKEND"] = "db" + + api, spec = create_app_infrastructure(db_app) + create_api_endpoints(db_app, api, spec) + db_app.app_context().push() + + yield db_app.test_client() + + +@pytest.fixture() +def valid_db_credentials_header(): + session = SESSION() + session.id = "Test" + session.expireDateTime = datetime.now() + timedelta(hours=1) + session.username = "Test User" + + insert_row_into_table(SESSION, session) + + yield {"Authorization": f"Bearer {session.id}"} + + delete_row_by_id(SESSION, "Test") + + +@pytest.fixture() +def test_config(test_config_data): + with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): + return APIConfig.load("test/path") + + +@pytest.fixture() +def test_config_without_search_api(test_config_data): + del test_config_data["search_api"] + with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): + return APIConfig.load("test/path") diff --git a/test/search_api/__init__.py b/test/integration/datagateway_api/__init__.py similarity index 100% rename from test/search_api/__init__.py rename to test/integration/datagateway_api/__init__.py diff --git a/test/datagateway_api/db/conftest.py b/test/integration/datagateway_api/db/conftest.py similarity index 100% rename from test/datagateway_api/db/conftest.py rename to test/integration/datagateway_api/db/conftest.py diff --git a/test/datagateway_api/db/endpoints/test_count_with_filters_db.py b/test/integration/datagateway_api/db/endpoints/test_count_with_filters_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_count_with_filters_db.py rename to test/integration/datagateway_api/db/endpoints/test_count_with_filters_db.py diff --git a/test/datagateway_api/db/endpoints/test_findone_db.py b/test/integration/datagateway_api/db/endpoints/test_findone_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_findone_db.py rename to test/integration/datagateway_api/db/endpoints/test_findone_db.py diff --git a/test/datagateway_api/db/endpoints/test_get_by_id_db.py b/test/integration/datagateway_api/db/endpoints/test_get_by_id_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_get_by_id_db.py rename to test/integration/datagateway_api/db/endpoints/test_get_by_id_db.py diff --git a/test/datagateway_api/db/endpoints/test_get_with_filters.py b/test/integration/datagateway_api/db/endpoints/test_get_with_filters.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_get_with_filters.py rename to test/integration/datagateway_api/db/endpoints/test_get_with_filters.py diff --git a/test/datagateway_api/db/endpoints/test_ping_db.py b/test/integration/datagateway_api/db/endpoints/test_ping_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_ping_db.py rename to test/integration/datagateway_api/db/endpoints/test_ping_db.py diff --git a/test/datagateway_api/db/endpoints/test_table_endpoints_db.py b/test/integration/datagateway_api/db/endpoints/test_table_endpoints_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_table_endpoints_db.py rename to test/integration/datagateway_api/db/endpoints/test_table_endpoints_db.py diff --git a/test/datagateway_api/db/test_database_filter_utilities.py b/test/integration/datagateway_api/db/test_database_filter_utilities.py similarity index 100% rename from test/datagateway_api/db/test_database_filter_utilities.py rename to test/integration/datagateway_api/db/test_database_filter_utilities.py diff --git a/test/datagateway_api/db/test_entity_helper.py b/test/integration/datagateway_api/db/test_entity_helper.py similarity index 100% rename from test/datagateway_api/db/test_entity_helper.py rename to test/integration/datagateway_api/db/test_entity_helper.py diff --git a/test/datagateway_api/db/test_requires_session_id.py b/test/integration/datagateway_api/db/test_requires_session_id.py similarity index 100% rename from test/datagateway_api/db/test_requires_session_id.py rename to test/integration/datagateway_api/db/test_requires_session_id.py diff --git a/test/datagateway_api/icat/conftest.py b/test/integration/datagateway_api/icat/conftest.py similarity index 96% rename from test/datagateway_api/icat/conftest.py rename to test/integration/datagateway_api/icat/conftest.py index cd0aac73..18ea25e8 100644 --- a/test/datagateway_api/icat/conftest.py +++ b/test/integration/datagateway_api/icat/conftest.py @@ -11,8 +11,12 @@ create_app_infrastructure, ) from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.endpoints.test_create_icat import TestICATCreateData -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.endpoints.test_create_icat import ( + TestICATCreateData, +) +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) @pytest.fixture() diff --git a/test/datagateway_api/icat/endpoints/test_count_with_filters_icat.py b/test/integration/datagateway_api/icat/endpoints/test_count_with_filters_icat.py similarity index 100% rename from test/datagateway_api/icat/endpoints/test_count_with_filters_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_count_with_filters_icat.py diff --git a/test/datagateway_api/icat/endpoints/test_create_icat.py b/test/integration/datagateway_api/icat/endpoints/test_create_icat.py similarity index 98% rename from test/datagateway_api/icat/endpoints/test_create_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_create_icat.py index 646d4a48..3442759c 100644 --- a/test/datagateway_api/icat/endpoints/test_create_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_create_icat.py @@ -1,7 +1,9 @@ import pytest from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestICATCreateData: diff --git a/test/datagateway_api/icat/endpoints/test_delete_by_id_icat.py b/test/integration/datagateway_api/icat/endpoints/test_delete_by_id_icat.py similarity index 100% rename from test/datagateway_api/icat/endpoints/test_delete_by_id_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_delete_by_id_icat.py diff --git a/test/datagateway_api/icat/endpoints/test_findone_icat.py b/test/integration/datagateway_api/icat/endpoints/test_findone_icat.py similarity index 92% rename from test/datagateway_api/icat/endpoints/test_findone_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_findone_icat.py index fd57253b..70097c88 100644 --- a/test/datagateway_api/icat/endpoints/test_findone_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_findone_icat.py @@ -1,5 +1,7 @@ from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestICATFindone: diff --git a/test/datagateway_api/icat/endpoints/test_get_by_id_icat.py b/test/integration/datagateway_api/icat/endpoints/test_get_by_id_icat.py similarity index 95% rename from test/datagateway_api/icat/endpoints/test_get_by_id_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_get_by_id_icat.py index 44cc79d8..cbe16a3b 100644 --- a/test/datagateway_api/icat/endpoints/test_get_by_id_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_get_by_id_icat.py @@ -1,5 +1,7 @@ from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestICATGetByID: diff --git a/test/datagateway_api/icat/endpoints/test_get_with_filters_icat.py b/test/integration/datagateway_api/icat/endpoints/test_get_with_filters_icat.py similarity index 96% rename from test/datagateway_api/icat/endpoints/test_get_with_filters_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_get_with_filters_icat.py index 9d3a9a2d..a868ec1d 100644 --- a/test/datagateway_api/icat/endpoints/test_get_with_filters_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_get_with_filters_icat.py @@ -1,7 +1,9 @@ import pytest from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestICATGetWithFilters: diff --git a/test/datagateway_api/icat/endpoints/test_ping_icat.py b/test/integration/datagateway_api/icat/endpoints/test_ping_icat.py similarity index 100% rename from test/datagateway_api/icat/endpoints/test_ping_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_ping_icat.py diff --git a/test/datagateway_api/icat/endpoints/test_table_endpoints_icat.py b/test/integration/datagateway_api/icat/endpoints/test_table_endpoints_icat.py similarity index 97% rename from test/datagateway_api/icat/endpoints/test_table_endpoints_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_table_endpoints_icat.py index 58a1345b..f6e29ccb 100644 --- a/test/datagateway_api/icat/endpoints/test_table_endpoints_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_table_endpoints_icat.py @@ -1,5 +1,7 @@ from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestICATableEndpoints: diff --git a/test/datagateway_api/icat/endpoints/test_update_by_id_icat.py b/test/integration/datagateway_api/icat/endpoints/test_update_by_id_icat.py similarity index 94% rename from test/datagateway_api/icat/endpoints/test_update_by_id_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_update_by_id_icat.py index a9e52b8f..e9f4af0e 100644 --- a/test/datagateway_api/icat/endpoints/test_update_by_id_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_update_by_id_icat.py @@ -1,5 +1,7 @@ from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestUpdateByID: diff --git a/test/datagateway_api/icat/endpoints/test_update_multiple_icat.py b/test/integration/datagateway_api/icat/endpoints/test_update_multiple_icat.py similarity index 98% rename from test/datagateway_api/icat/endpoints/test_update_multiple_icat.py rename to test/integration/datagateway_api/icat/endpoints/test_update_multiple_icat.py index 6c7de86b..41f92ef8 100644 --- a/test/datagateway_api/icat/endpoints/test_update_multiple_icat.py +++ b/test/integration/datagateway_api/icat/endpoints/test_update_multiple_icat.py @@ -1,7 +1,9 @@ import pytest from datagateway_api.src.common.config import Config -from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion +from test.integration.datagateway_api.icat.test_query import ( + prepare_icat_data_for_assertion, +) class TestUpdateMultipleEntities: diff --git a/test/datagateway_api/icat/filters/test_distinct_filter.py b/test/integration/datagateway_api/icat/filters/test_distinct_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_distinct_filter.py rename to test/integration/datagateway_api/icat/filters/test_distinct_filter.py diff --git a/test/datagateway_api/icat/filters/test_include_filter.py b/test/integration/datagateway_api/icat/filters/test_include_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_include_filter.py rename to test/integration/datagateway_api/icat/filters/test_include_filter.py diff --git a/test/datagateway_api/icat/filters/test_limit_filter.py b/test/integration/datagateway_api/icat/filters/test_limit_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_limit_filter.py rename to test/integration/datagateway_api/icat/filters/test_limit_filter.py diff --git a/test/datagateway_api/icat/filters/test_order_filter.py b/test/integration/datagateway_api/icat/filters/test_order_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_order_filter.py rename to test/integration/datagateway_api/icat/filters/test_order_filter.py diff --git a/test/datagateway_api/icat/filters/test_skip_filter.py b/test/integration/datagateway_api/icat/filters/test_skip_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_skip_filter.py rename to test/integration/datagateway_api/icat/filters/test_skip_filter.py diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/integration/datagateway_api/icat/filters/test_where_filter.py similarity index 100% rename from test/datagateway_api/icat/filters/test_where_filter.py rename to test/integration/datagateway_api/icat/filters/test_where_filter.py diff --git a/test/datagateway_api/icat/test_helpers.py b/test/integration/datagateway_api/icat/test_helpers.py similarity index 100% rename from test/datagateway_api/icat/test_helpers.py rename to test/integration/datagateway_api/icat/test_helpers.py diff --git a/test/datagateway_api/icat/test_icat_client.py b/test/integration/datagateway_api/icat/test_icat_client.py similarity index 100% rename from test/datagateway_api/icat/test_icat_client.py rename to test/integration/datagateway_api/icat/test_icat_client.py diff --git a/test/datagateway_api/icat/test_lru_cache.py b/test/integration/datagateway_api/icat/test_lru_cache.py similarity index 100% rename from test/datagateway_api/icat/test_lru_cache.py rename to test/integration/datagateway_api/icat/test_lru_cache.py diff --git a/test/datagateway_api/icat/test_query.py b/test/integration/datagateway_api/icat/test_query.py similarity index 100% rename from test/datagateway_api/icat/test_query.py rename to test/integration/datagateway_api/icat/test_query.py diff --git a/test/datagateway_api/icat/test_session_handling.py b/test/integration/datagateway_api/icat/test_session_handling.py similarity index 100% rename from test/datagateway_api/icat/test_session_handling.py rename to test/integration/datagateway_api/icat/test_session_handling.py diff --git a/test/datagateway_api/test_backends.py b/test/integration/datagateway_api/test_backends.py similarity index 100% rename from test/datagateway_api/test_backends.py rename to test/integration/datagateway_api/test_backends.py diff --git a/test/datagateway_api/test_query_filter_factory.py b/test/integration/datagateway_api/test_query_filter_factory.py similarity index 100% rename from test/datagateway_api/test_query_filter_factory.py rename to test/integration/datagateway_api/test_query_filter_factory.py diff --git a/test/integration/search_api/__init__.py b/test/integration/search_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/search_api/conftest.py b/test/integration/search_api/conftest.py similarity index 100% rename from test/search_api/conftest.py rename to test/integration/search_api/conftest.py diff --git a/test/search_api/endpoints/test_count_dataset_files.py b/test/integration/search_api/endpoints/test_count_dataset_files.py similarity index 100% rename from test/search_api/endpoints/test_count_dataset_files.py rename to test/integration/search_api/endpoints/test_count_dataset_files.py diff --git a/test/search_api/endpoints/test_count_endpoint.py b/test/integration/search_api/endpoints/test_count_endpoint.py similarity index 100% rename from test/search_api/endpoints/test_count_endpoint.py rename to test/integration/search_api/endpoints/test_count_endpoint.py diff --git a/test/search_api/endpoints/test_get_dataset_files.py b/test/integration/search_api/endpoints/test_get_dataset_files.py similarity index 100% rename from test/search_api/endpoints/test_get_dataset_files.py rename to test/integration/search_api/endpoints/test_get_dataset_files.py diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/integration/search_api/endpoints/test_get_entity_by_pid.py similarity index 100% rename from test/search_api/endpoints/test_get_entity_by_pid.py rename to test/integration/search_api/endpoints/test_get_entity_by_pid.py diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/integration/search_api/endpoints/test_search_endpoint.py similarity index 100% rename from test/search_api/endpoints/test_search_endpoint.py rename to test/integration/search_api/endpoints/test_search_endpoint.py diff --git a/test/search_api/filters/test_search_api_include_filter.py b/test/integration/search_api/filters/test_search_api_include_filter.py similarity index 100% rename from test/search_api/filters/test_search_api_include_filter.py rename to test/integration/search_api/filters/test_search_api_include_filter.py diff --git a/test/search_api/filters/test_search_api_limit_filter.py b/test/integration/search_api/filters/test_search_api_limit_filter.py similarity index 100% rename from test/search_api/filters/test_search_api_limit_filter.py rename to test/integration/search_api/filters/test_search_api_limit_filter.py diff --git a/test/search_api/filters/test_search_api_skip_filter.py b/test/integration/search_api/filters/test_search_api_skip_filter.py similarity index 100% rename from test/search_api/filters/test_search_api_skip_filter.py rename to test/integration/search_api/filters/test_search_api_skip_filter.py diff --git a/test/search_api/filters/test_search_api_where_filter.py b/test/integration/search_api/filters/test_search_api_where_filter.py similarity index 100% rename from test/search_api/filters/test_search_api_where_filter.py rename to test/integration/search_api/filters/test_search_api_where_filter.py diff --git a/test/search_api/test_error_handling.py b/test/integration/search_api/test_error_handling.py similarity index 100% rename from test/search_api/test_error_handling.py rename to test/integration/search_api/test_error_handling.py diff --git a/test/search_api/test_nested_where_filters.py b/test/integration/search_api/test_nested_where_filters.py similarity index 100% rename from test/search_api/test_nested_where_filters.py rename to test/integration/search_api/test_nested_where_filters.py diff --git a/test/search_api/test_search_api_query.py b/test/integration/search_api/test_search_api_query.py similarity index 100% rename from test/search_api/test_search_api_query.py rename to test/integration/search_api/test_search_api_query.py diff --git a/test/search_api/test_search_api_query_filter_factory.py b/test/integration/search_api/test_search_api_query_filter_factory.py similarity index 100% rename from test/search_api/test_search_api_query_filter_factory.py rename to test/integration/search_api/test_search_api_query_filter_factory.py diff --git a/test/search_api/test_session_handler.py b/test/integration/search_api/test_session_handler.py similarity index 100% rename from test/search_api/test_session_handler.py rename to test/integration/search_api/test_session_handler.py diff --git a/test/integration/test_config.py b/test/integration/test_config.py new file mode 100644 index 00000000..122c2f8a --- /dev/null +++ b/test/integration/test_config.py @@ -0,0 +1,9 @@ +class TestAPIConfig: + def test_load_with_valid_config_data(self, test_config): + backend_type = test_config.datagateway_api.backend + assert backend_type == "db" + + def test_set_backend_type(self, test_config): + test_config.datagateway_api.set_backend_type("backend_name_changed") + + assert test_config.datagateway_api.backend == "backend_name_changed" diff --git a/test/test_endpoint_rules.py b/test/integration/test_endpoint_rules.py similarity index 100% rename from test/test_endpoint_rules.py rename to test/integration/test_endpoint_rules.py diff --git a/test/test_filter_order_handler.py b/test/integration/test_filter_order_handler.py similarity index 100% rename from test/test_filter_order_handler.py rename to test/integration/test_filter_order_handler.py diff --git a/test/test_get_filters_from_query.py b/test/integration/test_get_filters_from_query.py similarity index 91% rename from test/test_get_filters_from_query.py rename to test/integration/test_get_filters_from_query.py index 4e232db0..02f98d47 100644 --- a/test/test_get_filters_from_query.py +++ b/test/integration/test_get_filters_from_query.py @@ -1,6 +1,6 @@ import pytest -from datagateway_api.src.common.exceptions import ApiError, FilterError +from datagateway_api.src.common.exceptions import FilterError from datagateway_api.src.common.helpers import get_filters_from_query_string from datagateway_api.src.datagateway_api.database.filters import ( DatabaseDistinctFieldFilter, @@ -60,7 +60,3 @@ def test_valid_search_api_filter(self, flask_test_app_db): filters = get_filters_from_query_string("search_api", "Dataset") assert len(filters) == 2 - - def test_invalid_api_type(self): - with pytest.raises(ApiError): - get_filters_from_query_string("unknown_api") diff --git a/test/test_get_session_id_from_auth_header.py b/test/integration/test_get_session_id_from_auth_header.py similarity index 100% rename from test/test_get_session_id_from_auth_header.py rename to test/integration/test_get_session_id_from_auth_header.py diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/datagateway_api/test_openapi.py b/test/unit/datagateway_api/test_openapi.py similarity index 100% rename from test/datagateway_api/test_openapi.py rename to test/unit/datagateway_api/test_openapi.py diff --git a/test/unit/search_api/conftest.py b/test/unit/search_api/conftest.py new file mode 100644 index 00000000..d5b46c85 --- /dev/null +++ b/test/unit/search_api/conftest.py @@ -0,0 +1,115 @@ +import json +from unittest.mock import mock_open, patch + + +import pytest +from datagateway_api.src.search_api.panosc_mappings import PaNOSCMappings + + +@pytest.fixture() +def test_search_api_mappings_data(): + return { + "Affiliation": { + "base_icat_entity": "Affiliation", + "id": "id", + "name": "name", + "address": "fullReference", + "city": "", + "country": "", + "members": {"Member": "user.user.investigationUsers"}, + }, + "Dataset": { + "base_icat_entity": "Dataset", + "pid": ["doi", "id"], + "title": "name", + "isPublic": "", + "creationDate": "createTime", + "size": "", + "documents": {"Document": "investigation"}, + "techniques": {"Technique": "datasetTechniques.technique"}, + "instrument": {"Instrument": "datasetInstruments.instrument"}, + "files": {"File": "datafiles"}, + "parameters": {"Parameter": "parameters"}, + "samples": {"Sample": "sample"}, + }, + "Document": { + "base_icat_entity": "Investigation", + "pid": ["doi", "id"], + "isPublic": "", + "type": "type.name", + "title": "name", + "summary": "summary", + "doi": "doi", + "startDate": "startDate", + "endDate": "endDate", + "releaseDate": "releaseDate", + "license": "", + "keywords": "keywords.name", + "datasets": {"Dataset": "datasets"}, + "members": {"Member": "investigationUsers"}, + "parameters": {"Parameter": "parameters"}, + }, + "File": { + "base_icat_entity": "Datafile", + "id": "id", + "name": "name", + "path": "location", + "size": "fileSize", + "dataset": {"Dataset": "dataset"}, + }, + "Instrument": { + "base_icat_entity": "Instrument", + "pid": ["pid", "id"], + "name": "name", + "facility": "facility.name", + "datasets": {"Dataset": "datasetInstruments.dataset"}, + }, + "Member": { + "base_icat_entity": "InvestigationUser", + "id": "id", + "role": "role", + "document": {"Document": "investigation"}, + "person": {"Person": "user"}, + "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"}, + }, + "Parameter": { + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], + "id": "id", + "name": "type.name", + "value": ["numericValue", "stringValue", "dateTimeValue"], + "unit": "type.units", + "dataset": {"Dataset": "dataset"}, + "document": {"Document": "investigation"}, + }, + "Person": { + "base_icat_entity": "User", + "id": "id", + "fullName": "fullName", + "orcid": "orcidId", + "researcherId": "", + "firstName": "givenName", + "lastName": "familyName", + "members": {"Member": "investigationUsers"}, + }, + "Sample": { + "base_icat_entity": "Sample", + "name": "name", + "pid": ["pid", "id"], + "description": "parameters.type.description", + "datasets": {"Dataset": "datasets"}, + }, + "Technique": { + "base_icat_entity": "Technique", + "pid": ["pid", "id"], + "name": "name", + "datasets": {"Dataset": "datasetTechniques.dataset"}, + }, + } + + +@pytest.fixture() +def test_panosc_mappings(test_search_api_mappings_data): + with patch( + "builtins.open", mock_open(read_data=json.dumps(test_search_api_mappings_data)), + ): + return PaNOSCMappings("test/path") diff --git a/test/search_api/test_models.py b/test/unit/search_api/test_models.py similarity index 100% rename from test/search_api/test_models.py rename to test/unit/search_api/test_models.py diff --git a/test/search_api/test_panosc_mappings.py b/test/unit/search_api/test_panosc_mappings.py similarity index 90% rename from test/search_api/test_panosc_mappings.py rename to test/unit/search_api/test_panosc_mappings.py index 9871618d..5040865e 100644 --- a/test/search_api/test_panosc_mappings.py +++ b/test/unit/search_api/test_panosc_mappings.py @@ -1,5 +1,3 @@ -from unittest.mock import patch - import pytest from datagateway_api.src.common.exceptions import FilterError, SearchAPIError @@ -11,31 +9,6 @@ def test_valid_load_mappings(self, test_panosc_mappings): test_mappings = PaNOSCMappings() assert test_mappings.mappings == test_panosc_mappings.mappings - @pytest.mark.parametrize( - "search_api_config_flag", - [ - pytest.param(True, id="Search API config present"), - pytest.param(False, id="No search API config"), - ], - ) - def test_invalid_load_mappings( - self, test_config, test_config_without_search_api, search_api_config_flag, - ): - if search_api_config_flag: - current_test_config = test_config - else: - current_test_config = test_config_without_search_api - - with patch( - "datagateway_api.src.common.config.Config.config", current_test_config, - ): - if search_api_config_flag: - with pytest.raises(SystemExit): - PaNOSCMappings("bad/path") - else: - # Shouldn't SysExit if a user isn't using the search API - PaNOSCMappings("bad/path") - @pytest.mark.parametrize( "panosc_entity_name, field_name, expected_panosc_entity_name" ", expected_icat_field_name", diff --git a/test/test_base_query_filter_factory.py b/test/unit/test_base_query_filter_factory.py similarity index 100% rename from test/test_base_query_filter_factory.py rename to test/unit/test_base_query_filter_factory.py diff --git a/test/test_config.py b/test/unit/test_config.py similarity index 91% rename from test/test_config.py rename to test/unit/test_config.py index c22e07b8..f9646bf8 100644 --- a/test/test_config.py +++ b/test/unit/test_config.py @@ -7,10 +7,6 @@ class TestAPIConfig: - def test_load_with_valid_config_data(self, test_config): - backend_type = test_config.datagateway_api.backend - assert backend_type == "db" - def test_load_with_no_config_data(self): with patch("builtins.open", mock_open(read_data="{}")): with pytest.raises(SystemExit): @@ -61,11 +57,6 @@ def test_load_with_same_api_extensions(self, test_config_data): with pytest.raises(SystemExit): APIConfig.load("test/path") - def test_set_backend_type(self, test_config): - test_config.datagateway_api.set_backend_type("backend_name_changed") - - assert test_config.datagateway_api.backend == "backend_name_changed" - @pytest.mark.parametrize( "input_extension, expected_extension", [ diff --git a/test/test_date_handler.py b/test/unit/test_date_handler.py similarity index 100% rename from test/test_date_handler.py rename to test/unit/test_date_handler.py diff --git a/test/test_exceptions.py b/test/unit/test_exceptions.py similarity index 100% rename from test/test_exceptions.py rename to test/unit/test_exceptions.py diff --git a/test/test_get_entity_object.py b/test/unit/test_get_entity_object.py similarity index 100% rename from test/test_get_entity_object.py rename to test/unit/test_get_entity_object.py diff --git a/test/unit/test_get_filters_from_query.py b/test/unit/test_get_filters_from_query.py new file mode 100644 index 00000000..09327d30 --- /dev/null +++ b/test/unit/test_get_filters_from_query.py @@ -0,0 +1,10 @@ +import pytest + +from datagateway_api.src.common.exceptions import ApiError +from datagateway_api.src.common.helpers import get_filters_from_query_string + + +class TestGetFiltersFromQueryString: + def test_invalid_api_type(self): + with pytest.raises(ApiError): + get_filters_from_query_string("unknown_api") diff --git a/test/test_is_valid_json.py b/test/unit/test_is_valid_json.py similarity index 100% rename from test/test_is_valid_json.py rename to test/unit/test_is_valid_json.py diff --git a/test/test_map_distinct_attrs.py b/test/unit/test_map_distinct_attrs.py similarity index 100% rename from test/test_map_distinct_attrs.py rename to test/unit/test_map_distinct_attrs.py diff --git a/test/test_queries_records.py b/test/unit/test_queries_records.py similarity index 100% rename from test/test_queries_records.py rename to test/unit/test_queries_records.py diff --git a/test/test_query_filter.py b/test/unit/test_query_filter.py similarity index 100% rename from test/test_query_filter.py rename to test/unit/test_query_filter.py From aa39a81c7a1ebb5d7c9c3f39ed7f79a7693f78e0 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Wed, 16 Nov 2022 14:16:15 +0000 Subject: [PATCH 02/22] Fix linting --- test/integration/conftest.py | 1 - test/unit/search_api/conftest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index c8ac7fee..a8acf1c6 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -4,7 +4,6 @@ from flask import Flask from icat.client import Client -from icat.query import Query import pytest from datagateway_api.src.api_start_utils import ( diff --git a/test/unit/search_api/conftest.py b/test/unit/search_api/conftest.py index d5b46c85..536ecbc4 100644 --- a/test/unit/search_api/conftest.py +++ b/test/unit/search_api/conftest.py @@ -1,8 +1,8 @@ import json from unittest.mock import mock_open, patch - import pytest + from datagateway_api.src.search_api.panosc_mappings import PaNOSCMappings From 7ffc4b8fa7af07f741d340face9dcc178ee5f95c Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 22 Nov 2022 06:33:38 +0000 Subject: [PATCH 03/22] reorganise new DB tests --- .../datagateway_api/db/conftest.py | 4 +- .../datagateway_api/db/endpoints/conftest.py | 50 +++++++++++++++++++ .../db/endpoints/test_create_db.py | 0 .../db/endpoints/test_delete_by_id_db.py | 0 .../db/endpoints/test_update_by_id_db.py | 0 .../db/endpoints/test_update_multiple_db.py | 0 .../db/test_database_filter_utilities.py | 2 +- 7 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 test/integration/datagateway_api/db/endpoints/conftest.py rename test/{ => integration}/datagateway_api/db/endpoints/test_create_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_delete_by_id_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_update_by_id_db.py (100%) rename test/{ => integration}/datagateway_api/db/endpoints/test_update_multiple_db.py (100%) diff --git a/test/integration/datagateway_api/db/conftest.py b/test/integration/datagateway_api/db/conftest.py index 154d1dd0..33e1e0f2 100644 --- a/test/integration/datagateway_api/db/conftest.py +++ b/test/integration/datagateway_api/db/conftest.py @@ -16,7 +16,9 @@ INVESTIGATION, INVESTIGATIONINSTRUMENT, ) -from test.datagateway_api.db.endpoints.test_create_db import TestDBCreateData +from test.integration.datagateway_api.db.endpoints.test_create_db import ( + TestDBCreateData, +) def set_meta_attributes(entity): diff --git a/test/integration/datagateway_api/db/endpoints/conftest.py b/test/integration/datagateway_api/db/endpoints/conftest.py new file mode 100644 index 00000000..667a9506 --- /dev/null +++ b/test/integration/datagateway_api/db/endpoints/conftest.py @@ -0,0 +1,50 @@ +from icat.query import Query +import pytest + + +@pytest.fixture() +def icat_query(icat_client): + return Query(icat_client, "Investigation") + + +@pytest.fixture() +def bad_credentials_header(): + return {"Authorization": "Bearer Invalid"} + + +@pytest.fixture() +def invalid_credentials_header(): + return {"Authorization": "Test"} + + +@pytest.fixture() +def test_config_data(): + return { + "datagateway_api": { + "extension": "/datagateway-api", + "backend": "db", + "client_cache_size": 5, + "client_pool_init_size": 2, + "client_pool_max_size": 5, + "db_url": "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb", + "icat_url": "https://localhost:8181", + "icat_check_cert": False, + }, + "search_api": { + "extension": "/search-api", + "icat_url": "https://localhost.testdomain:8181", + "icat_check_cert": True, + "mechanism": "anon", + "username": "", + "password": "", + }, + "flask_reloader": False, + "log_level": "WARN", + "log_location": "/home/runner/work/datagateway-api/datagateway-api/logs.log", + "debug_mode": False, + "generate_swagger": False, + "host": "127.0.0.1", + "port": "5000", + "test_user_credentials": {"username": "root", "password": "pw"}, + "test_mechanism": "simple", + } diff --git a/test/datagateway_api/db/endpoints/test_create_db.py b/test/integration/datagateway_api/db/endpoints/test_create_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_create_db.py rename to test/integration/datagateway_api/db/endpoints/test_create_db.py diff --git a/test/datagateway_api/db/endpoints/test_delete_by_id_db.py b/test/integration/datagateway_api/db/endpoints/test_delete_by_id_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_delete_by_id_db.py rename to test/integration/datagateway_api/db/endpoints/test_delete_by_id_db.py diff --git a/test/datagateway_api/db/endpoints/test_update_by_id_db.py b/test/integration/datagateway_api/db/endpoints/test_update_by_id_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_update_by_id_db.py rename to test/integration/datagateway_api/db/endpoints/test_update_by_id_db.py diff --git a/test/datagateway_api/db/endpoints/test_update_multiple_db.py b/test/integration/datagateway_api/db/endpoints/test_update_multiple_db.py similarity index 100% rename from test/datagateway_api/db/endpoints/test_update_multiple_db.py rename to test/integration/datagateway_api/db/endpoints/test_update_multiple_db.py diff --git a/test/integration/datagateway_api/db/test_database_filter_utilities.py b/test/integration/datagateway_api/db/test_database_filter_utilities.py index cdc5aee1..a8a5ff4e 100644 --- a/test/integration/datagateway_api/db/test_database_filter_utilities.py +++ b/test/integration/datagateway_api/db/test_database_filter_utilities.py @@ -7,7 +7,7 @@ DatabaseWhereFilter, ) from datagateway_api.src.datagateway_api.database.helpers import ReadQuery -from test.datagateway_api.db.endpoints.test_create_db import ( +from test.integration.datagateway_api.db.endpoints.test_create_db import ( prepare_db_data_for_assertion, ) From 4ebeb7753ef7a6416e001ad7ef63862b9198f7a7 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Thu, 1 Dec 2022 09:34:56 +0000 Subject: [PATCH 04/22] Add panosc mapping to unit tests --- test/conftest.py | 18 +++++++++++++ test/unit/search_api/test_panosc_mappings.py | 27 ++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 667a9506..c105ee4e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,11 @@ +import json +from unittest.mock import mock_open, patch + from icat.query import Query import pytest +from datagateway_api.src.common.config import APIConfig + @pytest.fixture() def icat_query(icat_client): @@ -48,3 +53,16 @@ def test_config_data(): "test_user_credentials": {"username": "root", "password": "pw"}, "test_mechanism": "simple", } + + +@pytest.fixture() +def test_config(test_config_data): + with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): + return APIConfig.load("test/path") + + +@pytest.fixture() +def test_config_without_search_api(test_config_data): + del test_config_data["search_api"] + with patch("builtins.open", mock_open(read_data=json.dumps(test_config_data))): + return APIConfig.load("test/path") diff --git a/test/unit/search_api/test_panosc_mappings.py b/test/unit/search_api/test_panosc_mappings.py index 5040865e..9871618d 100644 --- a/test/unit/search_api/test_panosc_mappings.py +++ b/test/unit/search_api/test_panosc_mappings.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + import pytest from datagateway_api.src.common.exceptions import FilterError, SearchAPIError @@ -9,6 +11,31 @@ def test_valid_load_mappings(self, test_panosc_mappings): test_mappings = PaNOSCMappings() assert test_mappings.mappings == test_panosc_mappings.mappings + @pytest.mark.parametrize( + "search_api_config_flag", + [ + pytest.param(True, id="Search API config present"), + pytest.param(False, id="No search API config"), + ], + ) + def test_invalid_load_mappings( + self, test_config, test_config_without_search_api, search_api_config_flag, + ): + if search_api_config_flag: + current_test_config = test_config + else: + current_test_config = test_config_without_search_api + + with patch( + "datagateway_api.src.common.config.Config.config", current_test_config, + ): + if search_api_config_flag: + with pytest.raises(SystemExit): + PaNOSCMappings("bad/path") + else: + # Shouldn't SysExit if a user isn't using the search API + PaNOSCMappings("bad/path") + @pytest.mark.parametrize( "panosc_entity_name, field_name, expected_panosc_entity_name" ", expected_icat_field_name", From c0595bac1e4e5aaa59529281a4472349a084c819 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Mon, 12 Dec 2022 16:16:32 +0000 Subject: [PATCH 05/22] Split unit and integration test runs --- .github/workflows/ci-build.yml | 58 +++++++++++++++++++--------------- noxfile.py | 24 ++++++++++++-- poetry.lock | 2 +- pyproject.toml | 2 +- 4 files changed, 56 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 048e15cd..fc58071c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -29,8 +29,36 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 + # Install Nox, Poetry and API's dependencies + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.9 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + + # This command is a workaround for getting Poetry working with Python 3.10. An + # fix is made in Poetry 1.2.0a2 but there is currently no official release for + # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the + # experimental installer is a workaround for Poetry 1.1.x + # See https://github.com/python-poetry/poetry/issues/4210 for more details + - name: Disable Poetry's experimental new installer + run: poetry config experimental.new-installer false + + - name: Install dependencies + run: poetry install + + # Run Unit tests + - name: Run Nox unit tests session + run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml + # ICAT Ansible clone and install dependencies - name: Checkout icat-ansible + if: success() uses: actions/checkout@v2 with: repository: icatproject-contrib/icat-ansible @@ -94,36 +122,14 @@ jobs: - name: Create search_api_mapping.json run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - # Install Nox, Poetry and API's dependencies - - name: Install Nox - run: pip install nox==2020.8.22 - - name: Install Poetry - run: pip install poetry==1.1.9 - - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 - - name: Uninstall setuptools - run: poetry run pip uninstall -y setuptools - - name: Install older setuptools - run: poetry run pip install 'setuptools<58.0.0' - - # This command is a workaround for getting Poetry working with Python 3.10. An - # fix is made in Poetry 1.2.0a2 but there is currently no official release for - # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the - # experimental installer is a workaround for Poetry 1.1.x - # See https://github.com/python-poetry/poetry/issues/4210 for more details - - name: Disable Poetry's experimental new installer - run: poetry config experimental.new-installer false - - - name: Install dependencies - run: poetry install - - name: Add dummy data to icatdb run: | poetry run python -m util.icat_db_generator - # Run Nox tests session, saves and uploads a coverage report to codecov - - name: Run Nox tests session - run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml + # Run Nox integration tests session, saves and uploads a coverage report to codecov + - name: Run Nox Integration tests session + if: success() + run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml - name: Upload code coverage report if: matrix.python-version == '3.6' uses: codecov/codecov-action@v1 diff --git a/noxfile.py b/noxfile.py index 1ec9df25..2f749e89 100644 --- a/noxfile.py +++ b/noxfile.py @@ -95,7 +95,7 @@ def safety(session): @nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"], reuse_venv=True) -def tests(session): +def unit_tests(session): args = session.posargs # Installing setuptools that will work with `2to3` which is used when building # `python-icat` < 1.0. 58.0.0 removes support of this tool during builds: @@ -111,4 +111,24 @@ def tests(session): # sign is specified for a package session.run("pip", "install", "setuptools<58.0.0") session.run("poetry", "install", external=True) - session.run("pytest", *args) + session.run("pytest", "test/unit", *args) + + +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"], reuse_venv=True) +def integration_tests(session): + args = session.posargs + # Installing setuptools that will work with `2to3` which is used when building + # `python-icat` < 1.0. 58.0.0 removes support of this tool during builds: + # https://setuptools.pypa.io/en/latest/history.html#v58-0-0 + # Ideally this would be done within `pyproject.toml` but specifying `setuptools` as + # a dependency requires Poetry 1.2: + # https://github.com/python-poetry/poetry/issues/4511#issuecomment-922420457 + # Currently, only a pre-release exists for Poetry 1.2. Testing on the pre-release + # version didn't fix the `2to3` issue when building Python ICAT, perhaps because + # Python ICAT isn't built on the downgraded version for some reason? + session.run("pip", "uninstall", "-y", "setuptools") + # Not using `poetry run` as it errors on Windows OS when a version with the '<' + # sign is specified for a package + session.run("pip", "install", "setuptools<58.0.0") + session.run("poetry", "install", external=True) + session.run("pytest", "test/integration", *args) diff --git a/poetry.lock b/poetry.lock index b39ed41f..a1e4d2d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1279,7 +1279,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4.0" -content-hash = "61ac95d09ddde72cadab31194a33d1fd79b68e3a771725db0f9ea627cc9f842c" +content-hash = "eeb4ff9346bd1d90ab3b7b6ea3cd2bdb14d1569a49c0ce5c2cc297c3b919b769" [metadata.files] aniso8601 = [] diff --git a/pyproject.toml b/pyproject.toml index 03f80450..13f8388c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" PyYAML = "5.4" -python-icat = "0.21.0" +python-icat = "^0.21.0" suds-community = "^0.8.4" py-object-pool = "^1.1" cachetools = "^4.2.1" From ebcb7a6408c99cb2c1f1d6d134c9a736befd3cc2 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Mon, 12 Dec 2022 16:29:02 +0000 Subject: [PATCH 06/22] Checkout api earlier --- .github/workflows/ci-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index fc58071c..93893d55 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -29,6 +29,9 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + # Install Nox, Poetry and API's dependencies - name: Install Nox run: pip install nox==2020.8.22 @@ -105,9 +108,6 @@ jobs: run: | cd /home/runner/install/icat.server/ && ./setup -vv install - - name: Checkout DataGateway API - uses: actions/checkout@v2 - # Prep for using the API for tests - name: Create log file run: touch logs.log From 4e275f680f88e3cf2bf6ffe5196cf34577eb6a47 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Mon, 12 Dec 2022 16:35:43 +0000 Subject: [PATCH 07/22] Create config earlier and dependency updates --- .github/workflows/ci-build.yml | 28 ++++++++++++++-------------- poetry.lock | 26 +++++++++++++------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 93893d55..580c77b1 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -55,6 +55,20 @@ jobs: - name: Install dependencies run: poetry install + # Prep for using the API for tests + - name: Create log file + run: touch logs.log + - name: Configure log file location + run: echo "`yq \ + '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example + - name: Configure datagateway extension + run: echo "`yq \ + '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example + - name: Create config.yaml + run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml + - name: Create search_api_mapping.json + run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json + # Run Unit tests - name: Run Nox unit tests session run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml @@ -108,20 +122,6 @@ jobs: run: | cd /home/runner/install/icat.server/ && ./setup -vv install - # Prep for using the API for tests - - name: Create log file - run: touch logs.log - - name: Configure log file location - run: echo "`yq \ - '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example - - name: Configure datagateway extension - run: echo "`yq \ - '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example - - name: Create config.yaml - run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml - - name: Create search_api_mapping.json - run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - name: Add dummy data to icatdb run: | poetry run python -m util.icat_db_generator diff --git a/poetry.lock b/poetry.lock index a1e4d2d7..e7fa354e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -112,7 +112,7 @@ python-versions = "~=3.5" [[package]] name = "certifi" -version = "2022.9.24" +version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -199,7 +199,7 @@ packaging = ">=20.4" [[package]] name = "cryptography" -version = "38.0.2" +version = "38.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -374,7 +374,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "flake8-import-order" -version = "0.18.1" +version = "0.18.2" description = "Flake8 and pylama plugin that checks the ordering of import statements." category = "dev" optional = false @@ -497,7 +497,7 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\" [[package]] name = "greenlet" -version = "2.0.0" +version = "2.0.1" description = "Lightweight in-process concurrent programming" category = "main" optional = false @@ -505,7 +505,7 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.extras] docs = ["sphinx", "docutils (<0.18)"] -test = ["objgraph", "faulthandler"] +test = ["objgraph", "psutil", "faulthandler"] [[package]] name = "icdiff" @@ -695,14 +695,14 @@ testing = ["mock", "pytest", "pytest-rerunfailures"] [[package]] name = "pkginfo" -version = "1.8.3" +version = "1.9.2" description = "Query metadatdata from sdists / bdists / installed packages." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.6" [package.extras] -testing = ["nose", "coverage"] +testing = ["pytest", "pytest-cov"] [[package]] name = "pluggy" @@ -1044,7 +1044,7 @@ python-versions = ">=3.5" [[package]] name = "safety" -version = "2.3.1" +version = "2.3.5" description = "Checks installed dependencies for known vulnerabilities and licenses." category = "dev" optional = false @@ -1054,7 +1054,7 @@ python-versions = "*" Click = ">=8.0.2" dataclasses = {version = "0.8", markers = "python_version == \"3.6\""} dparse = ">=0.6.2" -packaging = ">=21.0" +packaging = ">=21.0,<22.0" requests = "*" "ruamel.yaml" = ">=0.17.21" @@ -1100,7 +1100,7 @@ python-versions = ">=3.6" [[package]] name = "sqlalchemy" -version = "1.4.42" +version = "1.4.45" description = "Database Abstraction Library" category = "main" optional = false @@ -1231,11 +1231,11 @@ python-versions = ">=3.6" [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.13" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] From da66facce1abbd0379c8671385f9593ee07b1d03 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 13 Dec 2022 13:40:48 +0000 Subject: [PATCH 08/22] Ignore sqlalchemy vulnerability --- noxfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 2f749e89..1466d59a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -73,7 +73,8 @@ def safety(session): external=True, ) # Ignore 50916 as the latest version of pydantic and - # Ignore 51457 as the latest version of pytest does not support + # Ignore 51457 as the latest version of pytest and + # Ignore 51668 as the latest version of SQLAchemy as they do not support # python 3.6 which is still used in production session.run( "safety", @@ -84,6 +85,8 @@ def safety(session): "50916", "--ignore", "51457", + "--ignore", + "51668", ) try: From b3d0fc5693673b0cbfb8bd47de535188af767d4f Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 13 Dec 2022 15:34:27 +0000 Subject: [PATCH 09/22] Use latest codecov action --- .github/workflows/ci-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 580c77b1..fb8c3f7f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -132,7 +132,9 @@ jobs: run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml - name: Upload code coverage report if: matrix.python-version == '3.6' - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 + with: + verbose: true linting: runs-on: ubuntu-20.04 From a6f5d06742e090aa7f5beed8702a78715cf761e3 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 13 Dec 2022 16:24:26 +0000 Subject: [PATCH 10/22] Multiple codecov uploads --- .github/workflows/ci-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index fb8c3f7f..0df28b3a 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -72,6 +72,9 @@ jobs: # Run Unit tests - name: Run Nox unit tests session run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml + - name: Upload unit test code coverage report + if: matrix.python-version == '3.6' + uses: codecov/codecov-action@v3 # ICAT Ansible clone and install dependencies - name: Checkout icat-ansible @@ -130,11 +133,9 @@ jobs: - name: Run Nox Integration tests session if: success() run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml - - name: Upload code coverage report + - name: Upload integration test code coverage report if: matrix.python-version == '3.6' uses: codecov/codecov-action@v3 - with: - verbose: true linting: runs-on: ubuntu-20.04 From ebd60b6e751cb546a981ef388fe45fd750d686b9 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Thu, 15 Dec 2022 11:07:19 +0000 Subject: [PATCH 11/22] update to v1 --- poetry.lock | 23 ++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index e7fa354e..52555996 100644 --- a/poetry.lock +++ b/poetry.lock @@ -643,7 +643,7 @@ python-versions = "*" name = "packaging" version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -810,7 +810,7 @@ ed25519 = ["PyNaCl (>=1.4.0)"] name = "pyparsing" version = "3.0.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -897,11 +897,15 @@ yaml = ["PyYaml (>=5.2)"] [[package]] name = "python-icat" -version = "0.21.0" +version = "1.0.0b1" description = "Python interface to ICAT and IDS" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.4" + +[package.dependencies] +packaging = "*" +suds = "*" [[package]] name = "python-semantic-release" @@ -1143,6 +1147,14 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} pbr = ">=2.0.0,<2.1.0 || >2.1.0" +[[package]] +name = "suds" +version = "1.1.2" +description = "Lightweight SOAP client (community fork)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "suds-community" version = "0.8.5" @@ -1279,7 +1291,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4.0" -content-hash = "eeb4ff9346bd1d90ab3b7b6ea3cd2bdb14d1569a49c0ce5c2cc297c3b919b769" +content-hash = "0d8e9bf57e14289e06b845712960decf753eec105ba46edc8d5d9914cee36e54" [metadata.files] aniso8601 = [] @@ -1377,6 +1389,7 @@ six = [] smmap = [] sqlalchemy = [] stevedore = [] +suds = [] suds-community = [] text-unidecode = [] toml = [] diff --git a/pyproject.toml b/pyproject.toml index 13f8388c..46d37af0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" PyYAML = "5.4" -python-icat = "^0.21.0" +python-icat = "1.0.0b1" suds-community = "^0.8.4" py-object-pool = "^1.1" cachetools = "^4.2.1" From 86b9a11e96224335bfe0c366e329eded46ec900b Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Thu, 15 Dec 2022 11:12:33 +0000 Subject: [PATCH 12/22] assert correct values for v1 --- test/integration/datagateway_api/icat/test_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/datagateway_api/icat/test_helpers.py b/test/integration/datagateway_api/icat/test_helpers.py index 49dd9370..515a9d61 100644 --- a/test/integration/datagateway_api/icat/test_helpers.py +++ b/test/integration/datagateway_api/icat/test_helpers.py @@ -18,11 +18,11 @@ class TestICATHelpers: [ pytest.param("User", "user", id="singular single word entity name"), pytest.param( - "PublicStep", "publicStep", id="singular two word entity name", + "PublicStep", "publicstep", id="singular two word entity name", ), pytest.param( "PermissibleStringValue", - "permissibleStringValue", + "permissiblestringvalue", id="singular multi-word entity name", ), ], From e7b8794a16ac0f9beed62c33d9b3456e1506ea1d Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Thu, 15 Dec 2022 14:30:59 +0000 Subject: [PATCH 13/22] remove get_icat_entity_name_as_camel_case --- .../src/datagateway_api/icat/helpers.py | 36 +------------------ .../datagateway_api/icat/test_helpers.py | 31 +--------------- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/helpers.py b/datagateway_api/src/datagateway_api/icat/helpers.py index 55b79c77..cd20b387 100644 --- a/datagateway_api/src/datagateway_api/icat/helpers.py +++ b/datagateway_api/src/datagateway_api/icat/helpers.py @@ -145,40 +145,6 @@ def refresh_client_session(client): client.refresh() -def get_icat_entity_name_as_camel_case(client, entity_name): - """ - From the entity name, this function returns a camelCase version of its input - - Due to the case sensitivity of Python ICAT, a camelCase version of the entity name - is required for creating ICAT entities in ICAT (e.g. `client.new("parameterType")`). - - :param client: ICAT client containing an authenticated user - :type client: :class:`icat.client.Client` - :param entity_name: Entity name to fetch a camelCase version of - :type entity_name: :class:`str` - :return: Entity name (of type string) in the correct casing ready to be passed into - Python ICAT - :raises BadRequestError: If the entity cannot be found - """ - - entity_names = getTypeMap(client).keys() - lowercase_entity_name = entity_name.lower() - python_icat_entity_name = None - - for entity_name in entity_names: - lowercase_name = entity_name.lower() - if lowercase_name == lowercase_entity_name: - python_icat_entity_name = entity_name - - # Raise a 400 if a valid entity cannot be found - if python_icat_entity_name is None: - raise BadRequestError( - f"Bad request made, cannot find {entity_name} entity within Python ICAT", - ) - - return python_icat_entity_name - - def update_attributes(old_entity, new_entity): """ Updates the attribute(s) of a given object which is a record of an entity from @@ -507,7 +473,7 @@ def create_entities(client, entity_type, data): data = [data] for result in data: - new_entity = client.new(get_icat_entity_name_as_camel_case(client, entity_type)) + new_entity = client.new(entity_type.lower()) for attribute_name, value in result.items(): log.debug("Preparing data for %s", attribute_name) diff --git a/test/integration/datagateway_api/icat/test_helpers.py b/test/integration/datagateway_api/icat/test_helpers.py index 515a9d61..654d6aa2 100644 --- a/test/integration/datagateway_api/icat/test_helpers.py +++ b/test/integration/datagateway_api/icat/test_helpers.py @@ -4,41 +4,12 @@ import pytest from datagateway_api.src.common.exceptions import BadRequestError, PythonICATError -from datagateway_api.src.datagateway_api.icat.helpers import ( - get_icat_entity_name_as_camel_case, - push_data_updates_to_icat, -) +from datagateway_api.src.datagateway_api.icat.helpers import push_data_updates_to_icat class TestICATHelpers: """Testing the helper functions which aren't covered in the endpoint tests""" - @pytest.mark.parametrize( - "input_entity_name, expected_entity_name", - [ - pytest.param("User", "user", id="singular single word entity name"), - pytest.param( - "PublicStep", "publicstep", id="singular two word entity name", - ), - pytest.param( - "PermissibleStringValue", - "permissiblestringvalue", - id="singular multi-word entity name", - ), - ], - ) - def test_valid_get_icat_entity_name_as_camel_case( - self, icat_client, input_entity_name, expected_entity_name, - ): - camel_case_entity_name = get_icat_entity_name_as_camel_case( - icat_client, input_entity_name, - ) - assert camel_case_entity_name == expected_entity_name - - def test_invalid_get_icat_entity_name_as_camel_case(self, icat_client): - with pytest.raises(BadRequestError): - get_icat_entity_name_as_camel_case(icat_client, "UnknownEntityName") - def test_invalid_update_pushes(self, icat_client): with patch( "icat.entity.Entity.update", From 24c3bd78c34c4b842cc472867df631dba5ca702a Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 3 Jan 2023 13:55:16 +0000 Subject: [PATCH 14/22] Fix linting --- datagateway_api/src/datagateway_api/icat/helpers.py | 1 - test/integration/datagateway_api/icat/test_helpers.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datagateway_api/src/datagateway_api/icat/helpers.py b/datagateway_api/src/datagateway_api/icat/helpers.py index cd20b387..61f4980c 100644 --- a/datagateway_api/src/datagateway_api/icat/helpers.py +++ b/datagateway_api/src/datagateway_api/icat/helpers.py @@ -4,7 +4,6 @@ from cachetools import cached from dateutil.tz import tzlocal -from icat.entities import getTypeMap from icat.exception import ( ICATInternalError, ICATNoObjectError, diff --git a/test/integration/datagateway_api/icat/test_helpers.py b/test/integration/datagateway_api/icat/test_helpers.py index 654d6aa2..dd8c0401 100644 --- a/test/integration/datagateway_api/icat/test_helpers.py +++ b/test/integration/datagateway_api/icat/test_helpers.py @@ -3,7 +3,7 @@ from icat.exception import ICATInternalError import pytest -from datagateway_api.src.common.exceptions import BadRequestError, PythonICATError +from datagateway_api.src.common.exceptions import PythonICATError from datagateway_api.src.datagateway_api.icat.helpers import push_data_updates_to_icat From a0e952744d415144d20f27846d7b2418f6c8ef2f Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Wed, 4 Jan 2023 13:10:05 +0000 Subject: [PATCH 15/22] Update python icat to full release --- poetry.lock | 34 ++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/poetry.lock b/poetry.lock index 52555996..3193c2be 100644 --- a/poetry.lock +++ b/poetry.lock @@ -43,17 +43,19 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] +tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] [[package]] name = "bandit" @@ -199,7 +201,7 @@ packaging = ">=20.4" [[package]] name = "cryptography" -version = "38.0.4" +version = "39.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -209,9 +211,9 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +pep8test = ["black", "ruff"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] @@ -509,7 +511,7 @@ test = ["objgraph", "psutil", "faulthandler"] [[package]] name = "icdiff" -version = "2.0.5" +version = "2.0.6" description = "improved colored diff" category = "dev" optional = false @@ -695,7 +697,7 @@ testing = ["mock", "pytest", "pytest-rerunfailures"] [[package]] name = "pkginfo" -version = "1.9.2" +version = "1.9.3" description = "Query metadatdata from sdists / bdists / installed packages." category = "dev" optional = false @@ -785,7 +787,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.13.0" +version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false @@ -897,7 +899,7 @@ yaml = ["PyYaml (>=5.2)"] [[package]] name = "python-icat" -version = "1.0.0b1" +version = "1.0.0" description = "Python interface to ICAT and IDS" category = "main" optional = false @@ -937,7 +939,7 @@ test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest [[package]] name = "pytz" -version = "2022.6" +version = "2022.7" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -1104,7 +1106,7 @@ python-versions = ">=3.6" [[package]] name = "sqlalchemy" -version = "1.4.45" +version = "1.4.46" description = "Database Abstraction Library" category = "main" optional = false @@ -1291,7 +1293,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4.0" -content-hash = "0d8e9bf57e14289e06b845712960decf753eec105ba46edc8d5d9914cee36e54" +content-hash = "dd0974f8d4bd51d89357dbf61ba18a53866907c44cb5ea90a234b75db65a71a8" [metadata.files] aniso8601 = [] diff --git a/pyproject.toml b/pyproject.toml index 46d37af0..fe0b84a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" PyYAML = "5.4" -python-icat = "1.0.0b1" +python-icat = "1.0.0" suds-community = "^0.8.4" py-object-pool = "^1.1" cachetools = "^4.2.1" From aa4350c3b9c8e95cfdb3cdb5465f5dba958f8373 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Wed, 4 Jan 2023 14:30:03 +0000 Subject: [PATCH 16/22] Update readme for new test runners. --- README.md | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2489aa83..d75c283b 100644 --- a/README.md +++ b/README.md @@ -205,9 +205,12 @@ Currently, the following Nox sessions have been created: - `safety` - this uses [safety](https://github.com/pyupio/safety) to check the dependencies (pulled directly from Poetry) for any known vulnerabilities. This session gives the output in a full ASCII style report. -- `tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the - automated tests in `test/`, tests for the database and ICAT backends, and non-backend +- `unit_tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the + automated tests in `test/unit`, tests for the database and ICAT backends, and non-backend specific tests. More details about the tests themselves [here](#running-tests). +- `integration_tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the + automated tests in `test/unit`, tests for the database and ICAT backends, and non-backend + specific tests. Requires an ICAT backend. More details about the tests themselves [here](#running-tests). Each Nox session builds an environment using the repo's dependencies (defined using Poetry) using `install_with_constraints()`. This stores the dependencies in a @@ -434,9 +437,14 @@ for the Swagger interface to be up to date when using the Python ICAT backend. # Running Tests -To run the tests use `nox -s tests`. The repository contains a variety of tests, to test -the functionality of the API works as intended. The tests are split into 3 main -sections: non-backend specific (testing features such as the date handler), ICAT backend +There are two seperate test runners provided. The integration tests, and the unit tests. +The unit test do not require an ICAT stack to be setup to run. The integration tests do +require an ICAT stack. In order to cover all the code you will need to run both tests. + +To run the unit test use `nox -s unit_tests`, and to run the integration tests use `nox -s integration_tests` +The repository contains a variety of tests, to test the functionality of the API works as intended, for convenience +and quicker action runs these are additionally split into the unit and integration tests. +The tests are split into 3 main sections: non-backend specific (testing features such as the date handler), ICAT backend tests (containing tests for backend specific components, including tests for the different types of endpoints) and Database Backend tests (like the ICAT backend tests, but covering only the most used aspects of the API). @@ -450,14 +458,16 @@ Python 3.6, 3.7, 3.8, 3.9 and 3.10. For most cases, running the tests in a singl version will be sufficient: ```bash -nox -p 3.6 -s tests +nox -p 3.6 -s unit_tests +nox -p 3.6 -s integration ``` This repository also utilises [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/) to check how much of the codebase is covered by the tests in `test/`: ```bash -nox -p 3.6 -s tests -- --cov-report term --cov=./datagateway_api +nox -p 3.6 -s unit_tests -- --cov-report term --cov=./datagateway_api +nox -p 3.6 -s integration_tests -- --cov-report term --cov=./datagateway_api ``` With `pytest`, you can output the duration for each test, useful for showing the slower @@ -466,21 +476,24 @@ into setup, call and teardown to more easily understand where the tests are bein down: ```bash -nox -p 3.6 -s tests -- --durations=0 +nox -p 3.6 -s unit_tests -- --durations=0 +nox -p 3.6 -s integration_tests -- --durations=0 ``` -To test a specific test class (or even a specific test function), use a double colon to -denote a each level down beyond the filename: +To test a specific test class (or even a specific test function), you will +need to use pytest itself through poetry. If you want to change the python +version use `poetry env use 3.6` which will generate a virtual env with that +version. ```bash # Test a specific file -nox -p 3.6 -s tests -- test/icat/test_query.py +poetry run pytest test/integration/datagateway_api/icat/test_query.py # Test a specific test class -nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery +poetry run pytest test/integration/datagateway_api/icat/test_query.py::TestICATQuery # Test a specific test function -nox -p 3.6 -s tests -- test/icat/test_query.py::TestICATQuery::test_valid_query_exeuction +poetry run pytest test/integration/datagateway_api/icat/test_query.py::TestICATQuery::test_valid_query_exeuction ``` # Project Structure From 985aa02bf097a916a4305a293ffb5aa8cfe6ca97 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 10 Jan 2023 10:34:04 +0000 Subject: [PATCH 17/22] Remove poetry install and setuptools install --- .github/workflows/ci-build.yml | 35 ---------------------------------- 1 file changed, 35 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 0df28b3a..3ceaaf1f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -38,23 +38,6 @@ jobs: - name: Install Poetry run: pip install poetry==1.1.9 - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 - - name: Uninstall setuptools - run: poetry run pip uninstall -y setuptools - - name: Install older setuptools - run: poetry run pip install 'setuptools<58.0.0' - - # This command is a workaround for getting Poetry working with Python 3.10. An - # fix is made in Poetry 1.2.0a2 but there is currently no official release for - # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the - # experimental installer is a workaround for Poetry 1.1.x - # See https://github.com/python-poetry/poetry/issues/4210 for more details - - name: Disable Poetry's experimental new installer - run: poetry config experimental.new-installer false - - - name: Install dependencies - run: poetry install - # Prep for using the API for tests - name: Create log file run: touch logs.log @@ -264,15 +247,6 @@ jobs: - name: Install Poetry run: pip install poetry==1.1.9 - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 - - name: Uninstall setuptools - run: poetry run pip uninstall -y setuptools - - name: Install older setuptools - run: poetry run pip install 'setuptools<58.0.0' - - - name: Install dependencies - run: poetry install - - name: Add dummy data to icatdb run: poetry run python -m util.icat_db_generator - name: Get SQL dump of dummy data @@ -316,15 +290,6 @@ jobs: - name: Create search_api_mapping.json run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 - - name: Uninstall setuptools - run: poetry run pip uninstall -y setuptools - - name: Install older setuptools - run: poetry run pip install 'setuptools<58.0.0' - - - name: Install dependencies - run: poetry install - - name: Add dummy data to icatdb run: poetry run python -m util.icat_db_generator - name: Get SQL dump of dummy data from main's generator script From f9eb154626be953b63ecf433587eafa77689463a Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 10 Jan 2023 10:39:14 +0000 Subject: [PATCH 18/22] Readd fix for python 3.10 --- .github/workflows/ci-build.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 3ceaaf1f..d6920dac 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -38,6 +38,15 @@ jobs: - name: Install Poetry run: pip install poetry==1.1.9 + # This command is a workaround for getting Poetry working with Python 3.10. An + # fix is made in Poetry 1.2.0a2 but there is currently no official release for + # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the + # experimental installer is a workaround for Poetry 1.1.x + # See https://github.com/python-poetry/poetry/issues/4210 for more details + - name: Disable Poetry's experimental new installer + if: ${{ matrix.python-version == '3.10'}} + run: poetry config experimental.new-installer false + # Prep for using the API for tests - name: Create log file run: touch logs.log From 5f11ba2542ff8a11a717e42a82653b3589d98e45 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 10 Jan 2023 11:04:35 +0000 Subject: [PATCH 19/22] Poetry install needed for generator --- .github/workflows/ci-build.yml | 32 +++++++++++++++++++++++++++++++- noxfile.py | 5 ++++- poetry.lock | 32 +++++++++++++++++--------------- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index d6920dac..a2604e7c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -44,7 +44,7 @@ jobs: # experimental installer is a workaround for Poetry 1.1.x # See https://github.com/python-poetry/poetry/issues/4210 for more details - name: Disable Poetry's experimental new installer - if: ${{ matrix.python-version == '3.10'}} + if: ${{ matrix.python-version == '3.10' }} run: poetry config experimental.new-installer false # Prep for using the API for tests @@ -117,6 +117,18 @@ jobs: run: | cd /home/runner/install/icat.server/ && ./setup -vv install + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + + - name: Install dependencies + run: poetry install + - name: Add dummy data to icatdb run: | poetry run python -m util.icat_db_generator @@ -256,6 +268,15 @@ jobs: - name: Install Poetry run: pip install poetry==1.1.9 + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + + - name: Install dependencies + run: poetry install + - name: Add dummy data to icatdb run: poetry run python -m util.icat_db_generator - name: Get SQL dump of dummy data @@ -299,6 +320,15 @@ jobs: - name: Create search_api_mapping.json run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + + - name: Install dependencies + run: poetry install + - name: Add dummy data to icatdb run: poetry run python -m util.icat_db_generator - name: Get SQL dump of dummy data from main's generator script diff --git a/noxfile.py b/noxfile.py index 1466d59a..4d297aea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -74,7 +74,8 @@ def safety(session): ) # Ignore 50916 as the latest version of pydantic and # Ignore 51457 as the latest version of pytest and - # Ignore 51668 as the latest version of SQLAchemy as they do not support + # Ignore 51668 as the latest version of SQLAchemy and + # Ignore 52355 as the latest version of gitpython as they do not support # python 3.6 which is still used in production session.run( "safety", @@ -87,6 +88,8 @@ def safety(session): "51457", "--ignore", "51668", + "--ignore", + "52322", ) try: diff --git a/poetry.lock b/poetry.lock index e7fa354e..8b569922 100644 --- a/poetry.lock +++ b/poetry.lock @@ -43,17 +43,19 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] +tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] [[package]] name = "bandit" @@ -199,7 +201,7 @@ packaging = ">=20.4" [[package]] name = "cryptography" -version = "38.0.4" +version = "39.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -209,9 +211,9 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +pep8test = ["black", "ruff"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] @@ -509,7 +511,7 @@ test = ["objgraph", "psutil", "faulthandler"] [[package]] name = "icdiff" -version = "2.0.5" +version = "2.0.6" description = "improved colored diff" category = "dev" optional = false @@ -695,8 +697,8 @@ testing = ["mock", "pytest", "pytest-rerunfailures"] [[package]] name = "pkginfo" -version = "1.9.2" -description = "Query metadatdata from sdists / bdists / installed packages." +version = "1.9.6" +description = "Query metadata from sdists / bdists / installed packages." category = "dev" optional = false python-versions = ">=3.6" @@ -785,7 +787,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.13.0" +version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false @@ -933,7 +935,7 @@ test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest [[package]] name = "pytz" -version = "2022.6" +version = "2022.7" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -1100,7 +1102,7 @@ python-versions = ">=3.6" [[package]] name = "sqlalchemy" -version = "1.4.45" +version = "1.4.46" description = "Database Abstraction Library" category = "main" optional = false From c2e3f77949a4e6fb211236301c229f4771451b6f Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 10 Jan 2023 11:26:38 +0000 Subject: [PATCH 20/22] Remove second checkout --- .github/workflows/ci-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index a2604e7c..92eef07f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -117,9 +117,6 @@ jobs: run: | cd /home/runner/install/icat.server/ && ./setup -vv install - - name: Checkout DataGateway API - uses: actions/checkout@v2 - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 - name: Uninstall setuptools run: poetry run pip uninstall -y setuptools From 35780bbb066662b2751c5312c14a6b125539082f Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Tue, 10 Jan 2023 12:35:12 +0000 Subject: [PATCH 21/22] Add prep for api for integration tests --- .github/workflows/ci-build.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 92eef07f..dac2fe1c 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -117,7 +117,23 @@ jobs: run: | cd /home/runner/install/icat.server/ && ./setup -vv install - # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + # Prep for using the API for tests + - name: Create log file + run: touch logs.log + - name: Configure log file location + run: echo "`yq \ + '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example + - name: Configure datagateway extension + run: echo "`yq \ + '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example + - name: Create config.yaml + run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml + - name: Create search_api_mapping.json + run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json + - name: Uninstall setuptools run: poetry run pip uninstall -y setuptools - name: Install older setuptools From 619d3e4fe05eeca04f31bc9891e864ac05335dc8 Mon Sep 17 00:00:00 2001 From: Reillyhewitson Date: Wed, 1 Feb 2023 16:27:57 +0000 Subject: [PATCH 22/22] Update for safety --- noxfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 4d297aea..20f3b158 100644 --- a/noxfile.py +++ b/noxfile.py @@ -75,7 +75,8 @@ def safety(session): # Ignore 50916 as the latest version of pydantic and # Ignore 51457 as the latest version of pytest and # Ignore 51668 as the latest version of SQLAchemy and - # Ignore 52355 as the latest version of gitpython as they do not support + # Ignore 52355 and 52518 as the latest version of gitpython + # as they do not support # python 3.6 which is still used in production session.run( "safety", @@ -90,6 +91,8 @@ def safety(session): "51668", "--ignore", "52322", + "--ignore", + "52518", ) try: