Merge branch 'master' into 87_improve_readme

README.md

@@ -20,7 +20,7 @@ ICAT API to interface with the Data Gateway
 
 
 ## Requirements
-All requirements can be installed with `pip install -r requirements.txt`
+All requirements can be installed with `pip install -r requirements.txt`, and all development requirements can be installed with `pip install -r dev-requirements.txt`
 
 The required python libraries:  
    - [SQLAlchemy](https://www.sqlalchemy.org/)    
@@ -33,8 +33,14 @@ The required python libraries:
 ## Setup and running the API   
 The database connection needs to be set up first. This is set in config.json, an example config file called `config.json.example` is provided.
 
+Ideally the API would be run with:  
+`python -m src.main`
+However it can be run with the flask run command as shown below:
+
+
+**Warning: the host, port and debug config options will not be respected when the API is run this way**
 
-To run the API from the command line, the enviroment variable `FLASK_APP` should be set to `src/main.py`. Once this is 
+To use `flask run`, the enviroment variable `FLASK_APP` should be set to `src/main.py`. Once this is 
 set the API can be run with `flask run` while inside the root directory of the project. The `flask run` command gets installed with flask.   
 
 Examples shown:  
@@ -84,7 +90,8 @@ This is illustrated below.
     ├── src
     │   ├── resources
     │   │   ├── entities
-    │   │   │   └── <entity>_endpoints.py
+    │   │   │   ├── entity_endpoint.py
+    │   │   │   └── entity_map.py
     │   │   └── non_entities
     │   │       └── <non_entity>_endpoints.py
     │   ├── swagger
@@ -114,11 +121,11 @@ Example:
 This means that the http methods defined in the `DatafilesWithID` class are mapped to  `/datafiles/<int:id>`   
 
 #### Endpoints:  
-The logic for each endpoint are within `/src/resources`. They are split into entities 
-and non entities. Each endpoint has its own file within these folders e.g. the datafile endpoint
-is in `/src/resources/entities/datafiles_endpoints.py`. Inside of this file there is a class for
-each type of endpoint e.g. `/datafiles/count`. Each class is then imported to `/main.py` and added
-as a resource.
+The logic for each endpoint are within `/src/resources`. They are split into entities, non_entities and 
+table_endpoints. The entities package contains `entities_map` which maps entity names to their sqlalchemy
+model. The `entity_endpoint` module contains the function that is used to generate endpoints at start up.
+`table_endpoints` contains the endpoint classes that are table specific. Finally, non_entities contains the
+session endpoint.
 
 
 #### Mapped classes:

common/database_helpers.py

@@ -6,6 +6,7 @@
 from sqlalchemy.orm import aliased
 
 from common.exceptions import MissingRecordError, BadFilterError, BadRequestError, MultipleIncludeError
+from common.models import db_models
 from common.models.db_models import INVESTIGATIONUSER, INVESTIGATION, INSTRUMENT, FACILITYCYCLE, \
     INVESTIGATIONINSTRUMENT, FACILITY
 from common.session_manager import session_manager
@@ -147,21 +148,50 @@ class WhereFilter(QueryFilter):
 
     def __init__(self, field, value, operation):
         self.field = field
+        self.included_field = None
+        self.included_included_field = None
+        self._set_filter_fields()
         self.value = value
         self.operation = operation
 
+    def _set_filter_fields(self):
+        if self.field.count(".") == 1:
+            self.included_field = self.field.split(".")[1]
+            self.field = self.field.split(".")[0]
+
+        if self.field.count(".") == 2:
+            self.included_included_field = self.field.split(".")[2]
+            self.included_field = self.field.split(".")[1]
+            self.field = self.field.split(".")[0]
+
+
     def apply_filter(self, query):
+        try:
+            field = getattr(query.table, self.field)
+        except AttributeError:
+            raise BadFilterError(f"Bad  WhereFilter requested")
+
+        if self.included_included_field:
+            included_table = getattr(db_models, self.field)
+            included_included_table = getattr(db_models, self.included_field)
+            query.base_query = query.base_query.join(included_table).join(included_included_table)
+            field = getattr(included_included_table, self.included_included_field)
+
+        elif self.included_field:
+            included_table = getattr(db_models, self.field)
+            query.base_query = query.base_query.join(included_table)
+            field = getattr(included_table, self.included_field)
 
         if self.operation == "eq":
-            query.base_query = query.base_query.filter(getattr(query.table, self.field) == self.value)
+            query.base_query = query.base_query.filter(field == self.value)
         elif self.operation == "like":
-            query.base_query = query.base_query.filter(getattr(query.table, self.field).like(f"%{self.value}%"))
+            query.base_query = query.base_query.filter(field.like(f"%{self.value}%"))
         elif self.operation == "lte":
-            query.base_query = query.base_query.filter(getattr(query.table, self.field) <= self.value)
+            query.base_query = query.base_query.filter(field <= self.value)
         elif self.operation == "gte":
-            query.base_query = query.base_query.filter(getattr(query.table, self.field) >= self.value)
+            query.base_query = query.base_query.filter(field >= self.value)
         elif self.operation == "in":
-            query.base_query = query.base_query.filter(getattr(query.table, self.field).in_(self.value))
+            query.base_query = query.base_query.filter(field.in_(self.value))
         else:
             raise BadFilterError(f" Bad operation given to where filter. operation: {self.operation}")
 

common/helpers.py

@@ -21,7 +21,7 @@ def requires_session_id(method):
     :param method: The method for the endpoint
     :returns a 403, "Forbidden" if a valid session_id is not provided with the request
     """
-    log.info("")
+
 
     @wraps(method)
     def wrapper_requires_session(*args, **kwargs):

common/models/db_models.py

@@ -1,6 +1,8 @@
 import enum
 
 from datetime import datetime
+from decimal import Decimal
+
 from sqlalchemy import Index, Column, BigInteger, String, DateTime, ForeignKey, Integer, Float, FetchedValue, \
     TypeDecorator, Boolean
 from sqlalchemy.ext.declarative import declarative_base
@@ -50,9 +52,22 @@ def to_dict(self):
         dictionary = {}
         for column in self.__table__.columns:
             attribute = getattr(self, column.name)
-            dictionary[column.name] = str(attribute) if isinstance(attribute, datetime) else attribute
+            dictionary[column.name] = self._make_serializable(attribute)
         return dictionary
 
+    def _make_serializable(self, field):
+        """
+        Given a field, convert to a JSON serializable type
+        :param field: The field to be converted
+        :return: The converted field
+        """
+        if isinstance(field, datetime):
+            return str(field)
+        elif isinstance(field, Decimal):
+            return float(field)
+        else:
+            return field
+
     def to_nested_dict(self, includes):
         """
         Given related models return a nested dictionary with the child or parent rows nested.

dev-requirements.in

@@ -0,0 +1,2 @@
+Faker == 2.0.2
+pyyaml == 5.1.2

dev-requirements.txt

@@ -0,0 +1,11 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile '.\dev-requirements.in'
+#
+faker==2.0.2
+python-dateutil==2.8.0    # via faker
+pyyaml==5.1.2
+six==1.12.0               # via faker, python-dateutil
+text-unidecode==1.3       # via faker

requirements.in

@@ -1,6 +1,4 @@
 flask_restful == 0.3.7
 sqlalchemy == 1.3.8
 pymysql == 0.9.3
-pyyaml == 5.1.2
 flask-cors == 3.0.8
-Faker == 2.0.2

requirements.txt

@@ -2,22 +2,18 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile
+#    pip-compile '.\requirements.in'
 #
 aniso8601==8.0.0          # via flask-restful
 click==7.0                # via flask
-faker==2.0.2
 flask-cors==3.0.8
 flask==1.1.1              # via flask-cors, flask-restful
 flask_restful==0.3.7
 itsdangerous==1.1.0       # via flask
 jinja2==2.10.1            # via flask
 markupsafe==1.1.1         # via jinja2
 pymysql==0.9.3
-python-dateutil==2.8.0    # via faker
 pytz==2019.2              # via flask-restful
-pyyaml==5.1.2
-six==1.12.0               # via faker, flask-cors, flask-restful, python-dateutil
+six==1.12.0               # via flask-cors, flask-restful
 sqlalchemy==1.3.8
-text-unidecode==1.3       # via faker
 werkzeug==0.16.0          # via flask