From f67ad95f484363e46fb226402bd5c526e9eb2ec6 Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 14 Jan 2021 13:03:00 +0000 Subject: [PATCH 01/18] Fix session handling for db backend using Flask-SQLAlchemy --- datagateway_api/common/database/helpers.py | 14 ++- .../common/database/session_manager.py | 2 + datagateway_api/src/main.py | 9 +- poetry.lock | 98 +++++++++++-------- pyproject.toml | 1 + 5 files changed, 77 insertions(+), 47 deletions(-) diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index 55dbb40f..0d2474c7 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -17,7 +17,7 @@ INVESTIGATIONINSTRUMENT, SESSION, ) -from datagateway_api.common.database.session_manager import session_manager +from datagateway_api.common.database.session_manager import db from datagateway_api.common.exceptions import ( AuthenticationError, BadRequestError, @@ -41,12 +41,11 @@ def requires_session_id(method): @wraps(method) def wrapper_requires_session(*args, **kwargs): log.info(" Authenticating consumer") - session = session_manager.get_icat_db_session() + session = db.session() query = session.query(SESSION).filter(SESSION.ID == args[1]).first() if query is not None: log.info(" Closing DB session") session.close() - session.close() log.info(" Consumer authenticated") return method(*args, **kwargs) else: @@ -66,7 +65,7 @@ class Query(ABC): @abstractmethod def __init__(self, table): - self.session = session_manager.get_icat_db_session() + self.session = db.session self.table = table self.base_query = self.session.query(table) @@ -86,7 +85,12 @@ def commit_changes(self): Commits all changes to the database and closes the session """ log.info(" Committing changes to %s", self.table) - self.session.commit() + try: + self.session.commit() + except Exception as e: + log.error(f"Error whilst committing changes to {self.table}, rolling back") + self.session.rollback() + raise e class CountQuery(Query): diff --git a/datagateway_api/common/database/session_manager.py b/datagateway_api/common/database/session_manager.py index 5353fd26..386b601b 100644 --- a/datagateway_api/common/database/session_manager.py +++ b/datagateway_api/common/database/session_manager.py @@ -2,6 +2,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import QueuePool +from flask_sqlalchemy import SQLAlchemy from datagateway_api.common.constants import Constants engine = create_engine( @@ -24,3 +25,4 @@ def get_icat_db_session(self): session_manager = SessionManager() +db = SQLAlchemy() diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 22eb13df..35debd92 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -7,10 +7,13 @@ from flask_cors import CORS from flask_restful import Api from flask_swagger_ui import get_swaggerui_blueprint +from flask_sqlalchemy import SQLAlchemy from datagateway_api.common.backends import create_backend from datagateway_api.common.config import config from datagateway_api.common.logger_setup import setup_logger +from datagateway_api.common.database.session_manager import db +from datagateway_api.common.constants import Constants from datagateway_api.src.resources.entities.entity_endpoint import ( get_count_endpoint, get_endpoint, @@ -63,6 +66,8 @@ def create_app_infrastructure(flask_app): CORS(flask_app) flask_app.url_map.strict_slashes = False api = CustomErrorHandledApi(flask_app) + app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL + db.init_app(app) initialise_spec(spec) @@ -168,10 +173,10 @@ def specs(): resp.mimetype = "application/json" return resp +api, spec = create_app_infrastructure(app) +create_api_endpoints(app, api, spec) if __name__ == "__main__": - api, spec = create_app_infrastructure(app) - create_api_endpoints(app, api, spec) openapi_config(spec) app.run( host=config.get_host(), port=config.get_port(), debug=config.is_debug_mode(), diff --git a/poetry.lock b/poetry.lock index 037b6641..0ec30e5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,12 +15,12 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] +tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] docs = ["marshmallow (>=2.19.2)", "pyyaml (==5.3)", "sphinx (==2.4.1)", "sphinx-issues (==1.2.0)", "sphinx-rtd-theme (==0.4.3)"] lint = ["flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)"] -tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] -validation = ["prance[osv] (>=0.11)"] +dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] yaml = ["PyYAML (>=3.10)"] +validation = ["prance[osv] (>=0.11)"] [[package]] name = "appdirs" @@ -47,10 +47,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] [[package]] name = "bandit" @@ -61,11 +61,11 @@ optional = false python-versions = "*" [package.dependencies] -colorama = {version = ">=0.3.9", markers = "platform_system == \"Windows\""} -GitPython = ">=1.0.1" +stevedore = ">=1.20.0" PyYAML = ">=3.13" six = ">=1.10.0" -stevedore = ">=1.20.0" +colorama = {version = ">=0.3.9", markers = "platform_system == \"Windows\""} +GitPython = ">=1.0.1" [[package]] name = "black" @@ -76,13 +76,13 @@ optional = false python-versions = ">=3.6" [package.dependencies] -appdirs = "*" -attrs = ">=18.1.0" -click = ">=6.5" -pathspec = ">=0.6,<1" regex = "*" -toml = ">=0.9.4" typed-ast = ">=1.4.0" +toml = ">=0.9.4" +attrs = ">=18.1.0" +pathspec = ">=0.6,<1" +click = ">=6.5" +appdirs = "*" [package.extras] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] @@ -186,9 +186,9 @@ python-versions = "*" [package.dependencies] bandit = "*" -flake8 = "*" flake8-polyfill = "*" pycodestyle = "*" +flake8 = "*" [[package]] name = "flake8-black" @@ -259,8 +259,8 @@ optional = false python-versions = ">=3.5" [package.dependencies] -flake8 = ">=3.0,<3.2.0 || >3.2.0,<4" importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +flake8 = ">=3.0,<3.2.0 || >3.2.0,<4" [[package]] name = "flake8-import-order" @@ -301,15 +301,15 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -click = ">=5.1" itsdangerous = ">=0.24" Jinja2 = ">=2.10.1" +click = ">=5.1" Werkzeug = ">=0.15" [package.extras] -dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] -docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] dotenv = ["python-dotenv"] +docs = ["sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] +dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinxcontrib-log-cabinet", "sphinx-issues"] [[package]] name = "flask-cors" @@ -332,14 +332,26 @@ optional = false python-versions = "*" [package.dependencies] -aniso8601 = ">=0.82" Flask = ">=0.8" +aniso8601 = ">=0.82" pytz = "*" six = ">=1.3.0" [package.extras] docs = ["sphinx"] +[[package]] +name = "flask-sqlalchemy" +version = "2.4.4" +description = "Adds SQLAlchemy support to your Flask application." +category = "main" +optional = false +python-versions = ">= 2.7, != 3.0.*, != 3.1.*, != 3.2.*, != 3.3.*" + +[package.dependencies] +Flask = ">=0.10" +SQLAlchemy = ">=0.8.0" + [[package]] name = "flask-swagger-ui" version = "3.25.0" @@ -498,12 +510,12 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] -click = ">=7" six = "*" +click = ">=7" [package.extras] -coverage = ["pytest-cov"] testing = ["mock", "pytest", "pytest-rerunfailures"] +coverage = ["pytest-cov"] [[package]] name = "pluggy" @@ -579,19 +591,19 @@ optional = false python-versions = ">=3.5" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" py = ">=1.8.2" +iniconfig = "*" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +colorama = {version = "*", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" toml = "*" +pluggy = ">=0.12,<1.0" +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +checkqa_mypy = ["mypy (==0.780)"] [[package]] name = "pytest-cov" @@ -602,8 +614,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -coverage = ">=4.4" pytest = ">=4.6" +coverage = ">=4.4" [package.extras] testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] @@ -673,9 +685,9 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] +idna = ">=2.5,<3" certifi = ">=2017.4.17" chardet = ">=3.0.2,<4" -idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" [package.extras] @@ -691,10 +703,10 @@ optional = false python-versions = ">=3.5" [package.dependencies] -Click = ">=6.0" -dparse = ">=0.5.1" packaging = "*" requests = "*" +Click = ">=6.0" +dparse = ">=0.5.1" [[package]] name = "six" @@ -721,16 +733,16 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -mssql = ["pyodbc"] -mssql_pymssql = ["pymssql"] -mssql_pyodbc = ["pyodbc"] -mysql = ["mysqlclient"] -oracle = ["cx-oracle"] -postgresql = ["psycopg2"] postgresql_pg8000 = ["pg8000"] -postgresql_psycopg2binary = ["psycopg2-binary"] +postgresql = ["psycopg2"] postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql"] +mssql_pymssql = ["pymssql"] +postgresql_psycopg2binary = ["psycopg2-binary"] +mysql = ["mysqlclient"] +oracle = ["cx-oracle"] +mssql = ["pyodbc"] +mssql_pyodbc = ["pyodbc"] [[package]] name = "stevedore" @@ -798,8 +810,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] watchdog = ["watchdog"] +dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] [[package]] name = "zipp" @@ -816,7 +828,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "8ce6140731b2c2e9d2ba4f591ee85ca6d805851acba95169562ba1311a252ac5" +content-hash = "270d9adcbf9a704468e4e264bf85ca1c27fe66122d402e24a1ff7193636894ce" [metadata.files] aniso8601 = [ @@ -960,6 +972,10 @@ flask-restful = [ {file = "Flask-RESTful-0.3.7.tar.gz", hash = "sha256:f8240ec12349afe8df1db168ea7c336c4e5b0271a36982bff7394f93275f2ca9"}, {file = "Flask_RESTful-0.3.7-py2.py3-none-any.whl", hash = "sha256:ecd620c5cc29f663627f99e04f17d1f16d095c83dc1d618426e2ad68b03092f8"}, ] +flask-sqlalchemy = [ + {file = "Flask-SQLAlchemy-2.4.4.tar.gz", hash = "sha256:bfc7150eaf809b1c283879302f04c42791136060c6eeb12c0c6674fb1291fae5"}, + {file = "Flask_SQLAlchemy-2.4.4-py2.py3-none-any.whl", hash = "sha256:05b31d2034dd3f2a685cbbae4cfc4ed906b2a733cff7964ada450fd5e462b84e"}, +] flask-swagger-ui = [ {file = "flask-swagger-ui-3.25.0.tar.gz", hash = "sha256:42d098997e06b04f992609c4945cc990738b269c153d8388fc59a91a5dfcee9e"}, ] @@ -1114,6 +1130,8 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ diff --git a/pyproject.toml b/pyproject.toml index 5fea5a14..1891a50d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ flask-swagger-ui = "3.25.0" PyYAML = "5.3.1" python-icat = "0.17.0" suds-community = "^0.8.4" +Flask-SQLAlchemy = "^2.4.4" [tool.poetry.dev-dependencies] pip-tools = "5.3.1" From 71a65b3d06bd727a5d0e3d613daa610f6f2d730b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 22 Jan 2021 11:46:56 +0000 Subject: [PATCH 02/18] #201: Add teardown for POST entity endpoint tests - This replaces the data deletion loop that was after the assert statement, which wouldn't run if the an AssertionError was raised, resulting in undeleted test data --- test/icat/conftest.py | 35 +++++++++++++++++++++++++ test/icat/endpoints/test_create_icat.py | 29 +++++++------------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/test/icat/conftest.py b/test/icat/conftest.py index 05efd649..48198dad 100644 --- a/test/icat/conftest.py +++ b/test/icat/conftest.py @@ -12,6 +12,7 @@ create_api_endpoints, create_app_infrastructure, ) +from test.icat.endpoints.test_create_icat import TestICATCreateData from test.icat.test_query import prepare_icat_data_for_assertion @@ -154,3 +155,37 @@ def final_facilitycycle_id(flask_test_app_icat, valid_icat_credentials_header): headers=valid_icat_credentials_header, ) return final_facilitycycle_result.json["id"] + + +@pytest.fixture() +def remove_test_created_investigation_data( + flask_test_app_icat, valid_icat_credentials_header, +): + """ + This is used to delete the data created inside `test_valid` test functions in + TestICATCreateData + + This is done by fetching the data which has been created in + those functions (by using the investigation name prefix, as defined in the test + class), extracting the IDs from the results, and iterating over those to perform + DELETE by ID requests + """ + + yield + + created_test_data = flask_test_app_icat.get( + '/investigations?where={"name":{"like":' + f'"{TestICATCreateData.investigation_name_prefix}"' + "}}", + headers=valid_icat_credentials_header, + ) + + investigation_ids = [] + for investigation in created_test_data.json: + investigation_ids.append(investigation["id"]) + + for investigation_id in investigation_ids: + flask_test_app_icat.delete( + f"/investigations/{investigation_id}", + headers=valid_icat_credentials_header, + ) diff --git a/test/icat/endpoints/test_create_icat.py b/test/icat/endpoints/test_create_icat.py index 6c41f961..c20aeaba 100644 --- a/test/icat/endpoints/test_create_icat.py +++ b/test/icat/endpoints/test_create_icat.py @@ -1,13 +1,18 @@ +import pytest + from test.icat.test_query import prepare_icat_data_for_assertion class TestICATCreateData: + investigation_name_prefix = "Test Data for API Testing, Data Creation" + + @pytest.mark.usefixtures("remove_test_created_investigation_data") def test_valid_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): create_investigations_json = [ { - "name": f"Test Data for API Testing, Data Creation {i}", + "name": f"{self.investigation_name_prefix} {i}", "title": "Test data for the Python ICAT Backend on DataGateway API", "summary": "Test data for DataGateway API testing", "releaseDate": "2020-03-03 08:00:08+00:00", @@ -27,13 +32,9 @@ def test_valid_create_data( json=create_investigations_json, ) - test_data_ids = [] - for investigation_request, investigation_response in zip( - create_investigations_json, test_response.json, - ): + for investigation_request in create_investigations_json: investigation_request.pop("facility") investigation_request.pop("type") - test_data_ids.append(investigation_response["id"]) response_json = prepare_icat_data_for_assertion( test_response.json, remove_id=True, @@ -41,20 +42,14 @@ def test_valid_create_data( assert create_investigations_json == response_json - # Delete the entities created by this test - for investigation_id in test_data_ids: - flask_test_app_icat.delete( - f"/investigations/{investigation_id}", - headers=valid_icat_credentials_header, - ) - + @pytest.mark.usefixtures("remove_test_created_investigation_data") def test_valid_boundary_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): """Create a single investigation, as opposed to multiple""" create_investigation_json = { - "name": "Test Data for API Testing, Data Creation 0", + "name": f"{self.investigation_name_prefix} 0", "title": "Test data for the Python ICAT Backend on the API", "summary": "Test data for DataGateway API testing", "releaseDate": "2020-03-03 08:00:08+00:00", @@ -74,7 +69,6 @@ def test_valid_boundary_create_data( create_investigation_json.pop("facility") create_investigation_json.pop("type") - created_test_data_id = test_response.json[0]["id"] response_json = prepare_icat_data_for_assertion( test_response.json, remove_id=True, @@ -82,11 +76,6 @@ def test_valid_boundary_create_data( assert [create_investigation_json] == response_json - flask_test_app_icat.delete( - f"/investigations/{created_test_data_id}", - headers=valid_icat_credentials_header, - ) - def test_invalid_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): From f3498a8dd32e6fb54116060c96046664ec27491a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 2 Feb 2021 16:05:56 +0000 Subject: [PATCH 03/18] #203: Test codecov path fixing --- .github/workflows/ci-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 356c523d..a1fdfced 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -83,12 +83,13 @@ jobs: # Run Nox tests session, saves and uploads a coverage report to codecov - name: Run Nox tests session - run: nox -s tests -f datagateway-api/noxfile.py -- --cov=datagateway_api --cov-report=xml + run: cd datagateway-api && nox -s tests -- --cov=datagateway_api --cov-report=xml - name: Upload code coverage report if: matrix.python-version == '3.6' uses: codecov/codecov-action@v1 with: directory: ./datagateway-api + gcov_root_dir: ./datagateway-api linting: runs-on: ubuntu-16.04 From 1d6404038acfde129adacf0f9c63d5c6f43894d7 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 2 Feb 2021 17:26:12 +0000 Subject: [PATCH 04/18] #203: Stop API being cloned into a separate directory - This should help fix path issues with codecov --- .github/workflows/ci-build.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index a1fdfced..8606184f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -57,17 +57,15 @@ jobs: - name: Checkout DataGateway API uses: actions/checkout@v2 - with: - path: datagateway-api # Prep for using the API for tests - name: Create log file run: touch logs.log - name: Configure log file location run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \ - '.log_location=$REPO_DIR' datagateway-api/config.json.example`" > datagateway-api/config.json.example + '.log_location=$REPO_DIR' config.json.example`" > config.json.example - name: Create config.json - run: cp datagateway-api/config.json.example datagateway-api/config.json + run: cp config.json.example config.json # Install Nox, Poetry and API's dependencies - name: Install Nox @@ -75,21 +73,18 @@ jobs: - name: Install Poetry run: pip install poetry==1.1.4 - name: Install dependencies - run: cd datagateway-api/ && poetry install + run: poetry install - name: Add dummy data to icatdb run: | - cd datagateway-api && poetry run python -m util.icat_db_generator -s 4 -y 3 + poetry run python -m util.icat_db_generator -s 4 -y 3 # Run Nox tests session, saves and uploads a coverage report to codecov - name: Run Nox tests session - run: cd datagateway-api && nox -s tests -- --cov=datagateway_api --cov-report=xml + run: nox -s tests -- --cov=datagateway_api --cov-report=xml - name: Upload code coverage report if: matrix.python-version == '3.6' uses: codecov/codecov-action@v1 - with: - directory: ./datagateway-api - gcov_root_dir: ./datagateway-api linting: runs-on: ubuntu-16.04 From f6a48531d2d53db8570a34141e1383cd50bee886 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 2 Feb 2021 18:18:59 +0000 Subject: [PATCH 05/18] #203: Apply path fixing solution to other workflow jobs - This commit has no effect on codecov path fixing, but it makes the entire workflow consistent in how the API is cloned onto the Actions runner --- .github/workflows/ci-build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 8606184f..de7cfeab 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -97,8 +97,6 @@ jobs: architecture: x64 - name: Checkout DataGateway API uses: actions/checkout@v2 - with: - path: datagateway-api - name: Install Nox run: pip install nox==2020.8.22 @@ -106,7 +104,7 @@ jobs: run: pip install poetry==1.1.4 - name: Run Nox lint session - run: nox -s lint -f datagateway-api/noxfile.py + run: nox -s lint formatting: runs-on: ubuntu-16.04 @@ -119,8 +117,6 @@ jobs: architecture: x64 - name: Checkout DataGateway API uses: actions/checkout@v2 - with: - path: datagateway-api - name: Install Nox run: pip install nox==2020.8.22 @@ -128,7 +124,7 @@ jobs: run: pip install poetry==1.1.4 - name: Run Nox black session - run: nox -s black -f datagateway-api/noxfile.py + run: nox -s black safety: runs-on: ubuntu-16.04 @@ -141,8 +137,6 @@ jobs: architecture: x64 - name: Checkout DataGateway API uses: actions/checkout@v2 - with: - path: datagateway-api - name: Install Nox run: pip install nox==2020.8.22 @@ -150,4 +144,4 @@ jobs: run: pip install poetry==1.1.4 - name: Run Nox safety session - run: nox -s safety -f datagateway-api/noxfile.py + run: nox -s safety From 38dc28e445c78e36a1bd2b1e701a6cd2a1360eaa Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 11:32:19 +0000 Subject: [PATCH 06/18] Move session init code to more sensible places --- datagateway_api/common/database/helpers.py | 5 ++-- .../common/database/session_manager.py | 28 ------------------- datagateway_api/src/api_start_utils.py | 12 ++++++++ poetry.lock | 3 +- util/icat_db_generator.py | 4 +-- 5 files changed, 18 insertions(+), 34 deletions(-) delete mode 100644 datagateway_api/common/database/session_manager.py diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index 0d2474c7..c57d38ff 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -4,6 +4,7 @@ import logging from sqlalchemy.orm import aliased +from flask_sqlalchemy import SQLAlchemy from datagateway_api.common.database.filters import ( DatabaseIncludeFilter as IncludeFilter, @@ -17,7 +18,6 @@ INVESTIGATIONINSTRUMENT, SESSION, ) -from datagateway_api.common.database.session_manager import db from datagateway_api.common.exceptions import ( AuthenticationError, BadRequestError, @@ -28,6 +28,7 @@ log = logging.getLogger() +db = SQLAlchemy() def requires_session_id(method): """ @@ -41,7 +42,7 @@ def requires_session_id(method): @wraps(method) def wrapper_requires_session(*args, **kwargs): log.info(" Authenticating consumer") - session = db.session() + session = db.session query = session.query(SESSION).filter(SESSION.ID == args[1]).first() if query is not None: log.info(" Closing DB session") diff --git a/datagateway_api/common/database/session_manager.py b/datagateway_api/common/database/session_manager.py deleted file mode 100644 index 386b601b..00000000 --- a/datagateway_api/common/database/session_manager.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.pool import QueuePool - -from flask_sqlalchemy import SQLAlchemy -from datagateway_api.common.constants import Constants - -engine = create_engine( - Constants.DATABASE_URL, poolclass=QueuePool, pool_size=100, max_overflow=0, -) -session_factory = sessionmaker(engine) - - -class SessionManager(object): - def __init__(self): - self.Session = scoped_session(session_factory) - - def get_icat_db_session(self): - """ - Gets a new session in the scoped_session collection - :return: A new session - """ - - return self.Session() - - -session_manager = SessionManager() -db = SQLAlchemy() diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index 036f596f..3e8274da 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -9,6 +9,8 @@ from datagateway_api.common.backends import create_backend from datagateway_api.common.config import config +from datagateway_api.common.constants import Constants +from datagateway_api.common.database.helpers import db from datagateway_api.src.resources.entities.entity_endpoint import ( get_count_endpoint, get_endpoint, @@ -58,6 +60,16 @@ def create_app_infrastructure(flask_app): flask_app.url_map.strict_slashes = False api = CustomErrorHandledApi(flask_app) + try: + backend_type = flask_app.config["TEST_BACKEND"] + config.set_backend_type(backend_type) + except KeyError: + backend_type = config.get_backend_type() + + if backend_type == "db": + flask_app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL + db.init_app(flask_app) + initialise_spec(spec) return (api, spec) diff --git a/poetry.lock b/poetry.lock index 4a95e699..5f320163 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,7 +605,6 @@ python-versions = ">=3.5" [package.dependencies] packaging = "*" pathlib2 = {version = ">=2.2.0", markers = "python_version < \"3.6\""} -pluggy = ">=0.12,<1.0" py = ">=1.8.2" iniconfig = "*" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -842,7 +841,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.5" -content-hash = "db66f1e49ef4c5f9929c189d886b7c6b7d4306044a31e0eed7993d3ab6e9baab" +content-hash = "3cb90bdb013041464c10d254b247ae5431b025b945b22394e1822bfae30be9b4" [metadata.files] aniso8601 = [ diff --git a/util/icat_db_generator.py b/util/icat_db_generator.py index 4e53007d..fc3b76e6 100644 --- a/util/icat_db_generator.py +++ b/util/icat_db_generator.py @@ -7,7 +7,7 @@ from faker import Faker from datagateway_api.common.database import models -from datagateway_api.common.database.session_manager import session_manager +from datagateway_api.common.database.helper import db parser = argparse.ArgumentParser() parser.add_argument( @@ -33,7 +33,7 @@ faker.seed(SEED) seed(a=SEED) -session = session_manager.get_icat_db_session() +session = db.session def post_entity(entity): From 9d3ddc6419ffc20640f2cb2d68338285f5312a25 Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 11:45:17 +0000 Subject: [PATCH 07/18] Fix styling issues --- datagateway_api/common/database/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datagateway_api/common/database/helpers.py b/datagateway_api/common/database/helpers.py index c57d38ff..bf19a65d 100644 --- a/datagateway_api/common/database/helpers.py +++ b/datagateway_api/common/database/helpers.py @@ -3,8 +3,8 @@ from functools import wraps import logging -from sqlalchemy.orm import aliased from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.orm import aliased from datagateway_api.common.database.filters import ( DatabaseIncludeFilter as IncludeFilter, @@ -30,6 +30,7 @@ db = SQLAlchemy() + def requires_session_id(method): """ Decorator for database backend methods that makes sure a valid session_id is @@ -89,7 +90,7 @@ def commit_changes(self): try: self.session.commit() except Exception as e: - log.error(f"Error whilst committing changes to {self.table}, rolling back") + log.error("Error whilst committing changes to %s, rolling back", self.table) self.session.rollback() raise e From 5f1c2b2d0335c24896ff6659a9041099c2aa0809 Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 11:48:21 +0000 Subject: [PATCH 08/18] Fix import typo --- util/icat_db_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/icat_db_generator.py b/util/icat_db_generator.py index fc3b76e6..8a93eff0 100644 --- a/util/icat_db_generator.py +++ b/util/icat_db_generator.py @@ -7,7 +7,7 @@ from faker import Faker from datagateway_api.common.database import models -from datagateway_api.common.database.helper import db +from datagateway_api.common.database.helpers import db parser = argparse.ArgumentParser() parser.add_argument( From 825c2dd37a476e3498d9e66b2f4f3e38aa0f028b Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 11:58:44 +0000 Subject: [PATCH 09/18] Fix SQLALCHEMY_TRACK_MODIFICATIONS warning --- datagateway_api/src/api_start_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index 3e8274da..5228cfbb 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -68,6 +68,7 @@ def create_app_infrastructure(flask_app): if backend_type == "db": flask_app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL + flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False db.init_app(flask_app) initialise_spec(spec) From 582817aae28e72ec278f161216a65a6c1c24836a Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 11:58:59 +0000 Subject: [PATCH 10/18] Fix tests by manually pushing app context --- test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/conftest.py b/test/conftest.py index 9d29d01e..3c7c3b16 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -46,6 +46,7 @@ def flask_test_app_db(): api, spec = create_app_infrastructure(db_app) create_api_endpoints(db_app, api, spec) + db_app.app_context().push() yield db_app.test_client() From 9e8bed2e17c86f4934eb7680aa00e036b422ef88 Mon Sep 17 00:00:00 2001 From: Louise Davies Date: Thu, 4 Feb 2021 13:06:33 +0000 Subject: [PATCH 11/18] Use old session management for icat_db_generator script --- datagateway_api/src/api_start_utils.py | 2 +- util/icat_db_generator.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index 5228cfbb..f2313e50 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -68,7 +68,7 @@ def create_app_infrastructure(flask_app): if backend_type == "db": flask_app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL - flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False db.init_app(flask_app) initialise_spec(spec) diff --git a/util/icat_db_generator.py b/util/icat_db_generator.py index 8a93eff0..ff6e4c76 100644 --- a/util/icat_db_generator.py +++ b/util/icat_db_generator.py @@ -5,9 +5,12 @@ from random import choice, randrange, seed from faker import Faker +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.pool import QueuePool +from datagateway_api.common.constants import Constants from datagateway_api.common.database import models -from datagateway_api.common.database.helpers import db parser = argparse.ArgumentParser() parser.add_argument( @@ -33,7 +36,12 @@ faker.seed(SEED) seed(a=SEED) -session = db.session + +engine = create_engine( + Constants.DATABASE_URL, poolclass=QueuePool, pool_size=100, max_overflow=0, +) +session_factory = sessionmaker(engine) +session = scoped_session(session_factory)() def post_entity(entity): From 6b8e1e94fafa6d850598cd52d1353cb92302c7f4 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 16 Feb 2021 15:57:44 +0000 Subject: [PATCH 12/18] Move config.json to datagateway_api directory --- .github/workflows/ci-build.yml | 4 ++-- README.md | 2 +- datagateway_api/common/config.py | 2 +- config.json.example => datagateway_api/config.json.example | 0 test/test_config.py | 4 +++- 5 files changed, 7 insertions(+), 5 deletions(-) rename config.json.example => datagateway_api/config.json.example (100%) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index de7cfeab..b90d9cce 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -63,9 +63,9 @@ jobs: run: touch logs.log - name: Configure log file location run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \ - '.log_location=$REPO_DIR' config.json.example`" > config.json.example + '.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example - name: Create config.json - run: cp config.json.example config.json + run: cp datagateway_api/config.json.example datagateway_api/config.json # Install Nox, Poetry and API's dependencies - name: Install Nox diff --git a/README.md b/README.md index 2fc4f4a8..2eb0e403 100644 --- a/README.md +++ b/README.md @@ -427,8 +427,8 @@ illustrated below: ├── .pre-commit-config.yaml ├── LICENSE ├── README.md -├── config.json.example ├── datagateway_api +│ ├── config.json.example │ ├── common │ │ ├── backend.py │ │ ├── backends.py diff --git a/datagateway_api/common/config.py b/datagateway_api/common/config.py index 9b5090a4..28e4d8d1 100644 --- a/datagateway_api/common/config.py +++ b/datagateway_api/common/config.py @@ -10,7 +10,7 @@ class Config(object): - def __init__(self, path=Path(__file__).parent.parent.parent / "config.json"): + def __init__(self, path=Path(__file__).parent.parent / "config.json"): self.path = path with open(self.path) as target: self.config = json.load(target) diff --git a/config.json.example b/datagateway_api/config.json.example similarity index 100% rename from config.json.example rename to datagateway_api/config.json.example diff --git a/test/test_config.py b/test/test_config.py index 4d5da769..108f4cdc 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -8,7 +8,9 @@ @pytest.fixture() def valid_config(): - return Config(path=Path(__file__).parent.parent / "config.json.example") + return Config( + path=Path(__file__).parent.parent / "datagateway_api" / "config.json.example" + ) @pytest.fixture() From 8afe25a1337185ca6f00f8f41507bdd2716afb76 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 16 Feb 2021 16:25:03 +0000 Subject: [PATCH 13/18] Add wsgi.py --- README.md | 1 + datagateway_api/wsgi.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 datagateway_api/wsgi.py diff --git a/README.md b/README.md index 2eb0e403..db51e24e 100644 --- a/README.md +++ b/README.md @@ -429,6 +429,7 @@ illustrated below: ├── README.md ├── datagateway_api │ ├── config.json.example +│ ├── wsgi.py │ ├── common │ │ ├── backend.py │ │ ├── backends.py diff --git a/datagateway_api/wsgi.py b/datagateway_api/wsgi.py new file mode 100644 index 00000000..a31e1f21 --- /dev/null +++ b/datagateway_api/wsgi.py @@ -0,0 +1,6 @@ +import logging +import sys + +logging.basicConfig(stream=sys.stderr) + +from datagateway_api.src.main import app as application From 668d0e9fe26d0ba21444c1c932d92b143191ec73 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Tue, 16 Feb 2021 16:13:25 +0000 Subject: [PATCH 14/18] Add requests as a dependency --- poetry.lock | 43 ++++++++++++++++++++++++++++++++----------- pyproject.toml | 1 + 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index b9d46a1d..b54a51aa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -91,7 +91,7 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] name = "certifi" version = "2020.11.8" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -99,7 +99,7 @@ python-versions = "*" name = "chardet" version = "3.0.4" description = "Universal encoding detector for Python 2 and 3" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -385,7 +385,7 @@ python-versions = "*" name = "idna" version = "2.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -679,17 +679,17 @@ python-versions = "*" [[package]] name = "requests" -version = "2.24.0" +version = "2.25.1" description = "Python HTTP for Humans." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<4" +chardet = ">=3.0.2,<5" idna = ">=2.5,<3" -urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" +urllib3 = ">=1.21.1,<1.27" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] @@ -793,7 +793,7 @@ python-versions = "*" name = "urllib3" version = "1.25.11" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" @@ -829,7 +829,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.5" -content-hash = "db66f1e49ef4c5f9929c189d886b7c6b7d4306044a31e0eed7993d3ab6e9baab" +content-hash = "4517e06de24c84bb2c343d4f94823821ba1dc09478b4cf7b315341f624712fb0" [metadata.files] aniso8601 = [ @@ -1026,20 +1026,39 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp35-cp35m-win32.whl", hash = "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1"}, {file = "MarkupSafe-1.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d"}, {file = "MarkupSafe-1.1.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5"}, {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473"}, {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7"}, {file = "MarkupSafe-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66"}, {file = "MarkupSafe-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"}, {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"}, {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"}, {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b"}, {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"}, {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-win32.whl", hash = "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39"}, + {file = "MarkupSafe-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] mccabe = [ @@ -1131,6 +1150,8 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ @@ -1179,8 +1200,8 @@ regex = [ {file = "regex-2020.10.28.tar.gz", hash = "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b"}, ] requests = [ - {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, - {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, + {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, + {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] safety = [ {file = "safety-1.9.0-py2.py3-none-any.whl", hash = "sha256:86c1c4a031fe35bd624fce143fbe642a0234d29f7cbf7a9aa269f244a955b087"}, diff --git a/pyproject.toml b/pyproject.toml index 1323ded7..4b73b234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ flask-swagger-ui = "3.25.0" PyYAML = "5.3.1" python-icat = "0.17.0" suds-community = "^0.8.4" +requests = "^2.25.1" [tool.poetry.dev-dependencies] pip-tools = "5.3.1" From 2b87c760f98cbe85f0afc3da6fc73d5f0538eb0c Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 17 Feb 2021 09:38:54 +0000 Subject: [PATCH 15/18] Add trailing comma to appease linter --- test/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_config.py b/test/test_config.py index 108f4cdc..248764cb 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -9,7 +9,7 @@ @pytest.fixture() def valid_config(): return Config( - path=Path(__file__).parent.parent / "datagateway_api" / "config.json.example" + path=Path(__file__).parent.parent / "datagateway_api" / "config.json.example", ) From 8f3ffc696e18b3cec655475b36051071dc4d1bd9 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 17 Feb 2021 09:47:11 +0000 Subject: [PATCH 16/18] Ignore linter errors in wsgi.py --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index eef539f2..58cc2602 100644 --- a/.flake8 +++ b/.flake8 @@ -6,5 +6,5 @@ max-complexity = 17 max-line-length = 80 application-import-names = datagateway_api,test,util import-order-style = google -per-file-ignores = test/*:S101,util/icat_db_generator.py:S311 +per-file-ignores = test/*:S101,util/icat_db_generator.py:S311,datagateway_api/wsgi.py:E402,F401 enable-extensions=G From 1572c0cf9a118cbffc26b6006fdff28e148afda9 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 24 Feb 2021 11:54:02 +0000 Subject: [PATCH 17/18] Add missing python-dateutil dependency --- poetry.lock | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0ffd69c4..e03f1899 100644 --- a/poetry.lock +++ b/poetry.lock @@ -650,7 +650,7 @@ pytest = "*" name = "python-dateutil" version = "2.8.1" description = "Extensions to the standard Python datetime module" -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" @@ -841,7 +841,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.5" -content-hash = "2494cbdf616bbd9d2e75af3026faf972a06a8eb2ee81f353036bbae62beaa784" +content-hash = "77aef4a2b4ca9adc854dd74da351a5914a7807046dd280e4e957c6a8e3860b83" [metadata.files] aniso8601 = [ diff --git a/pyproject.toml b/pyproject.toml index 7aa761f7..8c52cca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ python-icat = "0.17.0" suds-community = "^0.8.4" Flask-SQLAlchemy = "^2.4.4" requests = "^2.25.1" +python-dateutil = "^2.8.1" [tool.poetry.dev-dependencies] pip-tools = "5.3.1" From 611c227879bd149a4976dcb1a5c3440098b0b9d1 Mon Sep 17 00:00:00 2001 From: Alan Kyffin Date: Wed, 24 Feb 2021 16:23:46 +0000 Subject: [PATCH 18/18] Don't write to openapi.yaml when generate_swagger set to false --- datagateway_api/src/api_start_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index f2313e50..e6443431 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -160,9 +160,9 @@ def openapi_config(spec): for endpoint_name in sorted(entity_data.keys()): entity_data.move_to_end(endpoint_name) - openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" - with open(openapi_spec_path, "w") as f: - f.write(spec.to_yaml()) + openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" + with open(openapi_spec_path, "w") as f: + f.write(spec.to_yaml()) def create_openapi_endpoint(app, api_spec):