Merge branch 'master' into bugfix/client-cache-#209

.flake8

@@ -9,5 +9,6 @@ import-order-style = google
 per-file-ignores =
   test/*: S101
   util/icat_db_generator.py: S311
+  datagateway_api/wsgi.py:E402,F401
   datagateway_api/common/database/models.py: N815,A003
 enable-extensions=G

.github/workflows/ci-build.yml

@@ -57,38 +57,34 @@ jobs:
 
     - name: Checkout DataGateway API
       uses: actions/checkout@v2
-      with:
-        path: datagateway-api
 
     # Prep for using the API for tests
     - name: Create log file
       run: touch logs.log
     - name: Configure log file location
       run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \
-        '.log_location=$REPO_DIR' datagateway-api/config.json.example`" > datagateway-api/config.json.example
+        '.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example
     - name: Create config.json
-      run: cp datagateway-api/config.json.example datagateway-api/config.json
+      run: cp datagateway_api/config.json.example datagateway_api/config.json
 
     # Install Nox, Poetry and API's dependencies
     - name: Install Nox
       run: pip install nox==2020.8.22
     - name: Install Poetry
       run: pip install poetry==1.1.4
     - name: Install dependencies
-      run: cd datagateway-api/ && poetry install
+      run: poetry install
 
     - name: Add dummy data to icatdb
       run: |
-        cd datagateway-api && poetry run python -m util.icat_db_generator -s 4 -y 3
+        poetry run python -m util.icat_db_generator -s 4 -y 3
 
     # Run Nox tests session, saves and uploads a coverage report to codecov
     - name: Run Nox tests session
-      run: nox -s tests -f datagateway-api/noxfile.py -- --cov=datagateway_api --cov-report=xml
+      run: nox -s tests -- --cov=datagateway_api --cov-report=xml
     - name: Upload code coverage report
       if: matrix.python-version == '3.6'
       uses: codecov/codecov-action@v1
-      with:
-        directory: ./datagateway-api
 
   linting:
     runs-on: ubuntu-16.04
@@ -101,16 +97,14 @@ jobs:
         architecture: x64
     - name: Checkout DataGateway API
       uses: actions/checkout@v2
-      with:
-        path: datagateway-api
 
     - name: Install Nox
       run: pip install nox==2020.8.22
     - name: Install Poetry
       run: pip install poetry==1.1.4
 
     - name: Run Nox lint session
-      run: nox -s lint -f datagateway-api/noxfile.py
+      run: nox -s lint
 
   formatting:
     runs-on: ubuntu-16.04
@@ -123,16 +117,14 @@ jobs:
         architecture: x64
     - name: Checkout DataGateway API
       uses: actions/checkout@v2
-      with:
-        path: datagateway-api
 
     - name: Install Nox
       run: pip install nox==2020.8.22
     - name: Install Poetry
       run: pip install poetry==1.1.4
 
     - name: Run Nox black session
-      run: nox -s black -f datagateway-api/noxfile.py
+      run: nox -s black
 
   safety:
     runs-on: ubuntu-16.04
@@ -145,13 +137,11 @@ jobs:
         architecture: x64
     - name: Checkout DataGateway API
       uses: actions/checkout@v2
-      with:
-        path: datagateway-api
 
     - name: Install Nox
       run: pip install nox==2020.8.22
     - name: Install Poetry
       run: pip install poetry==1.1.4
 
     - name: Run Nox safety session
-      run: nox -s safety -f datagateway-api/noxfile.py
+      run: nox -s safety

README.md

@@ -427,8 +427,9 @@ illustrated below:
 ├── .pre-commit-config.yaml
 ├── LICENSE
 ├── README.md
-├── config.json.example
 ├── datagateway_api
+│   ├── config.json.example
+│   ├── wsgi.py
 │   ├── common
 │   │   ├── backend.py
 │   │   ├── backends.py

datagateway_api/common/config.py

@@ -10,7 +10,7 @@
 
 
 class Config(object):
-    def __init__(self, path=Path(__file__).parent.parent.parent / "config.json"):
+    def __init__(self, path=Path(__file__).parent.parent / "config.json"):
         self.path = path
         with open(self.path) as target:
             self.config = json.load(target)

datagateway_api/common/database/helpers.py

@@ -3,6 +3,7 @@
 from functools import wraps
 import logging
 
+from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy.orm import aliased
 
 from datagateway_api.common.database.filters import (
@@ -17,7 +18,6 @@
     INVESTIGATIONINSTRUMENT,
     SESSION,
 )
-from datagateway_api.common.database.session_manager import session_manager
 from datagateway_api.common.exceptions import (
     AuthenticationError,
     BadRequestError,
@@ -28,6 +28,8 @@
 
 log = logging.getLogger()
 
+db = SQLAlchemy()
+
 
 def requires_session_id(method):
     """
@@ -41,12 +43,11 @@ def requires_session_id(method):
     @wraps(method)
     def wrapper_requires_session(*args, **kwargs):
         log.info(" Authenticating consumer")
-        session = session_manager.get_icat_db_session()
+        session = db.session
         query = session.query(SESSION).filter(SESSION.id == args[1]).first()
         if query is not None:
             log.info(" Closing DB session")
             session.close()
-            session.close()
             log.info(" Consumer authenticated")
             return method(*args, **kwargs)
         else:
@@ -66,7 +67,7 @@ class Query(ABC):
 
     @abstractmethod
     def __init__(self, table):
-        self.session = session_manager.get_icat_db_session()
+        self.session = db.session
         self.table = table
         self.base_query = self.session.query(table)
 
@@ -86,7 +87,12 @@ def commit_changes(self):
         Commits all changes to the database and closes the session
         """
         log.info(" Committing changes to %s", self.table)
-        self.session.commit()
+        try:
+            self.session.commit()
+        except Exception as e:
+            log.error("Error whilst committing changes to %s, rolling back", self.table)
+            self.session.rollback()
+            raise e
 
 
 class CountQuery(Query):

datagateway_api/common/database/models.py

@@ -148,7 +148,15 @@ def _nest_string_include(self, dictionary, include):
 
     def get_related_entity(self, entity):
         """
-        Given a string for the related entity name, return the related entity
+        Given a string for the related entity name, return the related entity. This
+        function is used to retrieve a related entity as a result of an include filter.
+
+        Some related entities have a plural name (so end with an 's') so this is
+        checked. If `entity` contains a name which is a singular, `entity` is
+        uppercased (table names are defined in upper case, as seen in `__tablename__` in
+        the entity classes lower down in this file). Both related entity name types
+        (singular and plural) are used to retrieve the actual related entity.
+
         :param entity: String - The name of the entity
         :return: The entity
         """

datagateway_api/common/icat/query.py

@@ -264,8 +264,15 @@ def map_distinct_attributes_to_entity_names(self, distinct_fields, included_fiel
                 # range of a list with a single element
                 split_fields.insert(0, "base")
 
-            # If a key doesn't exist in the dictionary, create it and assign an empty
-            # list to it
+            # Check that only an entity name, and attribute name exist
+            # Code within loop is used for when `split_fields` =
+            # ['dataset', 'investigation', 'name'] for example
+            while len(split_fields) > 2:
+                # If a key doesn't exist in the dictionary, create it and assign an
+                # empty list to it
+                distinct_field_dict.setdefault(split_fields[0], [])
+                split_fields.pop(0)
+
             distinct_field_dict.setdefault(split_fields[0], [])
             distinct_field_dict[split_fields[0]].append(split_fields[-1])
 
@@ -303,10 +310,12 @@ def prepare_distinct_fields(self, entity_name, distinct_fields):
         :return: A copy of `distinct_fields`, with the data from the entity name put
             into the base portion of the dictionary
         """
-        # Reset base fields
-        distinct_fields["base"] = []
+        log.debug("Entity Name: %s, Distinct Fields: %s", entity_name, distinct_fields)
 
         distinct_fields_copy = distinct_fields.copy()
+
+        # Reset base fields
+        distinct_fields_copy["base"] = []
         if entity_name in distinct_fields_copy.keys():
             distinct_fields_copy["base"] = distinct_fields_copy[entity_name]
 

datagateway_api/src/api_start_utils.py

@@ -9,6 +9,8 @@
 
 from datagateway_api.common.backends import create_backend
 from datagateway_api.common.config import config
+from datagateway_api.common.constants import Constants
+from datagateway_api.common.database.helpers import db
 from datagateway_api.common.icat.icat_client_pool import create_client_pool
 from datagateway_api.src.resources.entities.entity_endpoint import (
     get_count_endpoint,
@@ -59,6 +61,17 @@ def create_app_infrastructure(flask_app):
     flask_app.url_map.strict_slashes = False
     api = CustomErrorHandledApi(flask_app)
 
+    try:
+        backend_type = flask_app.config["TEST_BACKEND"]
+        config.set_backend_type(backend_type)
+    except KeyError:
+        backend_type = config.get_backend_type()
+
+    if backend_type == "db":
+        flask_app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL
+        flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+        db.init_app(flask_app)
+
     initialise_spec(spec)
 
     return (api, spec)
@@ -157,9 +170,9 @@ def openapi_config(spec):
             for endpoint_name in sorted(entity_data.keys()):
                 entity_data.move_to_end(endpoint_name)
 
-    openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml"
-    with open(openapi_spec_path, "w") as f:
-        f.write(spec.to_yaml())
+        openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml"
+        with open(openapi_spec_path, "w") as f:
+            f.write(spec.to_yaml())
 
 
 def create_openapi_endpoint(app, api_spec):

datagateway_api/src/resources/entities/entity_map.py

@@ -76,6 +76,7 @@ def create_entity_models():
                 or relationship_class.direction.name == "ONETOMANY"
             ):
                 entity_underscore_strip = relationship_name.strip("_")
+                # Checking for plurals on a related ICAT entity
                 if entity_underscore_strip[-1] == "s":
                     pascal_case = (
                         entity_underscore_strip[0].upper() + entity_underscore_strip[1:]

datagateway_api/wsgi.py

@@ -0,0 +1,6 @@
+import logging
+import sys
+
+logging.basicConfig(stream=sys.stderr)
+
+from datagateway_api.src.main import app as application