diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c405c1b2..c661d0f5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,13 +3,13 @@ This PR will close #{issue number} ## Description Enter a description of the changes here -## Testing instructions +## Testing Instructions Add a set up instructions describing how the reviewer should test the code - [ ] Review code -- [ ] Check Travis build +- [ ] Check GitHub Actions build - [ ] Review changes to test coverage - [ ] {more steps here} -## Agile board tracking -connect to #{issue number} +## Agile Board Tracking +Connect to #{issue number} diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 00000000..356c523d --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,157 @@ +name: CI +on: + workflow_dispatch: + pull_request: + push: + branches: + - master +jobs: + tests: + runs-on: ubuntu-16.04 + strategy: + fail-fast: false + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9'] + name: Python ${{ matrix.python-version }} Build & Tests + steps: + - name: Add apt repo + run: sudo add-apt-repository universe + + # Setup Java & Python + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 8 + java-package: jdk + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + # ICAT Ansible clone and install dependencies + - name: Checkout icat-ansible + uses: actions/checkout@v2 + with: + repository: icatproject-contrib/icat-ansible + ref: change-payara-setup-script-url + path: icat-ansible + - name: Install Ansible + run: pip install -r icat-ansible/requirements.txt + + # Prep for running the playbook + - name: Create hosts file + run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts + - name: Prepare vault pass + run: echo -e "icattravispw" > icat-ansible/vault_pass.txt + - name: Move vault to directory it'll get detected by Ansible + run: mv icat-ansible/vault.yml icat-ansible/group_vars/all + - name: Replace default payara user with Actions user + run: | + sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml + + # Create local instance of ICAT + - name: Run ICAT Ansible Playbook + run: | + ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv + + - name: Checkout DataGateway API + uses: actions/checkout@v2 + with: + path: datagateway-api + + # Prep for using the API for tests + - name: Create log file + run: touch logs.log + - name: Configure log file location + run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \ + '.log_location=$REPO_DIR' datagateway-api/config.json.example`" > datagateway-api/config.json.example + - name: Create config.json + run: cp datagateway-api/config.json.example datagateway-api/config.json + + # Install Nox, Poetry and API's dependencies + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.4 + - name: Install dependencies + run: cd datagateway-api/ && poetry install + + - name: Add dummy data to icatdb + run: | + cd datagateway-api && poetry run python -m util.icat_db_generator -s 4 -y 3 + + # Run Nox tests session, saves and uploads a coverage report to codecov + - name: Run Nox tests session + run: nox -s tests -f datagateway-api/noxfile.py -- --cov=datagateway_api --cov-report=xml + - name: Upload code coverage report + if: matrix.python-version == '3.6' + uses: codecov/codecov-action@v1 + with: + directory: ./datagateway-api + + linting: + runs-on: ubuntu-16.04 + name: Linting + steps: + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.6' + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + with: + path: datagateway-api + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.4 + + - name: Run Nox lint session + run: nox -s lint -f datagateway-api/noxfile.py + + formatting: + runs-on: ubuntu-16.04 + name: Code Formatting + steps: + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.6' + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + with: + path: datagateway-api + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.4 + + - name: Run Nox black session + run: nox -s black -f datagateway-api/noxfile.py + + safety: + runs-on: ubuntu-16.04 + name: Dependency Safety + steps: + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.6' + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + with: + path: datagateway-api + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.4 + + - name: Run Nox safety session + run: nox -s safety -f datagateway-api/noxfile.py diff --git a/.gitignore b/.gitignore index bfcfa809..51b7483a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ config.json .nox/ .python-version .coverage +coverage.xml diff --git a/README.md b/README.md index f60871e9..2fc4f4a8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +[![Build Status](https://github.com/ral-facilities/datagateway-api/workflows/CI/badge.svg?branch=master)](https://github.com/ral-facilities/datagateway-api/actions?query=workflow%3A%22CI%22) +[![Codecov](https://codecov.io/gh/ral-facilities/datagateway-api/branch/master/graph/badge.svg)](https://codecov.io/gh/ral-facilities/datagateway-api) + + + # DataGateway API This is a Flask-based API that fetches data from an ICAT instance, to interface with [DataGateway](https://github.com/ral-facilities/datagateway). This API uses two ways @@ -90,6 +95,7 @@ version needs to be downloaded and built individually: pyenv install 3.6.8 pyenv install 3.7.7 pyenv install 3.8.2 +pyenv install 3.9.0 ``` To verify the installation commands worked: @@ -98,6 +104,7 @@ To verify the installation commands worked: python3.6 --version python3.7 --version python3.8 --version +python3.9 --version ``` These Python versions need to be made available to local version of the repository. They @@ -106,7 +113,7 @@ following command will create a `.python-version` file inside the repo (this fil currently listed in `.gitignore`): ```bash -pyenv local 3.6.8 3.7.7 3.8.2 +pyenv local 3.6.8 3.7.7 3.8.2 3.9.0 ``` diff --git a/config.json.example b/config.json.example index 8aab7943..d44534ec 100644 --- a/config.json.example +++ b/config.json.example @@ -1,10 +1,10 @@ { "backend": "db", - "DB_URL": "mysql+pymysql://root:rootpw@localhost:13306/icatdb", - "ICAT_URL": "https://localhost.localdomain:8181", + "DB_URL": "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb", + "ICAT_URL": "https://localhost:8181", "icat_check_cert": false, "log_level": "WARN", - "log_location": "/home/user1/datagateway-api/logs.log", + "log_location": "/home/runner/work/datagateway-api/datagateway-api/logs.log", "debug_mode": false, "generate_swagger": false, "host": "127.0.0.1", diff --git a/datagateway_api/common/constants.py b/datagateway_api/common/constants.py index ce1ae9ac..f3b63956 100644 --- a/datagateway_api/common/constants.py +++ b/datagateway_api/common/constants.py @@ -5,7 +5,6 @@ class Constants: DATABASE_URL = config.get_db_url() - ACCEPTED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" PYTHON_ICAT_DISTNCT_CONDITION = "!= null" ICAT_PROPERTIES = config.get_icat_properties() TEST_MOD_CREATE_DATETIME = datetime(2000, 1, 1) diff --git a/datagateway_api/common/database/models.py b/datagateway_api/common/database/models.py index 1ce06e7c..5fe71d22 100644 --- a/datagateway_api/common/database/models.py +++ b/datagateway_api/common/database/models.py @@ -1,3 +1,4 @@ +from abc import ABC from datetime import datetime from decimal import Decimal import enum @@ -52,7 +53,7 @@ def copy(self, **kwargs): return EnumAsInteger(self.enum_type) -class EntityHelper(object): +class EntityHelper(ABC): """ EntityHelper class that contains methods to be shared across all entities """ @@ -186,7 +187,19 @@ def update_from_dict(self, dictionary): return self.to_dict() -class APPLICATION(Base, EntityHelper): +class EntityMeta(type(Base), type(EntityHelper)): + """ + This class is used as a way of ensuring classes that inherit from both `Base` and + `EntityHelper` don't have metaclass conflicts. `EntityHelper`'s metaclass is + `ABCMeta`, but this isn't the case for `Base`. Further explanation regarding this + issue (and how this class fixes the problem) can be found here: + https://stackoverflow.com/a/28727066/8752408 + """ + + pass + + +class APPLICATION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "APPLICATION" __table_args__ = (Index("UNQ_APPLICATION_0", "FACILITY_ID", "NAME", "VERSION"),) @@ -206,7 +219,7 @@ class APPLICATION(Base, EntityHelper): ) -class FACILITY(Base, EntityHelper): +class FACILITY(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "FACILITY" ID = Column(BigInteger, primary_key=True) @@ -221,7 +234,7 @@ class FACILITY(Base, EntityHelper): URL = Column(String(255)) -class DATACOLLECTION(Base, EntityHelper): +class DATACOLLECTION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATACOLLECTION" ID = Column(BigInteger, primary_key=True) @@ -232,7 +245,7 @@ class DATACOLLECTION(Base, EntityHelper): MOD_TIME = Column(DateTime, nullable=False) -class DATACOLLECTIONDATAFILE(Base, EntityHelper): +class DATACOLLECTIONDATAFILE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATACOLLECTIONDATAFILE" __table_args__ = ( Index("UNQ_DATACOLLECTIONDATAFILE_0", "DATACOLLECTION_ID", "DATAFILE_ID"), @@ -258,7 +271,7 @@ class DATACOLLECTIONDATAFILE(Base, EntityHelper): ) -class DATACOLLECTIONDATASET(Base, EntityHelper): +class DATACOLLECTIONDATASET(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATACOLLECTIONDATASET" __table_args__ = ( Index("UNQ_DATACOLLECTIONDATASET_0", "DATACOLLECTION_ID", "DATASET_ID"), @@ -284,7 +297,7 @@ class DATACOLLECTIONDATASET(Base, EntityHelper): ) -class DATACOLLECTIONPARAMETER(Base, EntityHelper): +class DATACOLLECTIONPARAMETER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATACOLLECTIONPARAMETER" __table_args__ = ( Index( @@ -320,7 +333,7 @@ class DATACOLLECTIONPARAMETER(Base, EntityHelper): ) -class DATAFILE(Base, EntityHelper): +class DATAFILE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATAFILE" __table_args__ = (Index("UNQ_DATAFILE_0", "DATASET_ID", "NAME"),) @@ -350,7 +363,7 @@ class DATAFILE(Base, EntityHelper): ) -class DATAFILEFORMAT(Base, EntityHelper): +class DATAFILEFORMAT(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATAFILEFORMAT" __table_args__ = (Index("UNQ_DATAFILEFORMAT_0", "FACILITY_ID", "NAME", "VERSION"),) @@ -372,7 +385,7 @@ class DATAFILEFORMAT(Base, EntityHelper): ) -class DATAFILEPARAMETER(Base, EntityHelper): +class DATAFILEPARAMETER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATAFILEPARAMETER" __table_args__ = ( Index("UNQ_DATAFILEPARAMETER_0", "DATAFILE_ID", "PARAMETER_TYPE_ID"), @@ -406,7 +419,7 @@ class DATAFILEPARAMETER(Base, EntityHelper): ) -class DATASET(Base, EntityHelper): +class DATASET(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATASET" __table_args__ = (Index("UNQ_DATASET_0", "INVESTIGATION_ID", "NAME"),) @@ -441,7 +454,7 @@ class DATASET(Base, EntityHelper): ) -class DATASETPARAMETER(Base, EntityHelper): +class DATASETPARAMETER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATASETPARAMETER" __table_args__ = ( Index("UNQ_DATASETPARAMETER_0", "DATASET_ID", "PARAMETER_TYPE_ID"), @@ -475,7 +488,7 @@ class DATASETPARAMETER(Base, EntityHelper): ) -class DATASETTYPE(Base, EntityHelper): +class DATASETTYPE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "DATASETTYPE" __table_args__ = (Index("UNQ_DATASETTYPE_0", "FACILITY_ID", "NAME"),) @@ -495,7 +508,7 @@ class DATASETTYPE(Base, EntityHelper): ) -class FACILITYCYCLE(Base, EntityHelper): +class FACILITYCYCLE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "FACILITYCYCLE" __table_args__ = (Index("UNQ_FACILITYCYCLE_0", "FACILITY_ID", "NAME"),) @@ -517,7 +530,7 @@ class FACILITYCYCLE(Base, EntityHelper): ) -class GROUPING(Base, EntityHelper): +class GROUPING(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "GROUPING" ID = Column(BigInteger, primary_key=True) @@ -528,7 +541,7 @@ class GROUPING(Base, EntityHelper): NAME = Column(String(255), nullable=False, unique=True) -class INSTRUMENT(Base, EntityHelper): +class INSTRUMENT(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INSTRUMENT" __table_args__ = (Index("UNQ_INSTRUMENT_0", "FACILITY_ID", "NAME"),) @@ -551,7 +564,7 @@ class INSTRUMENT(Base, EntityHelper): ) -class INSTRUMENTSCIENTIST(Base, EntityHelper): +class INSTRUMENTSCIENTIST(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INSTRUMENTSCIENTIST" __table_args__ = (Index("UNQ_INSTRUMENTSCIENTIST_0", "USER_ID", "INSTRUMENT_ID"),) @@ -575,7 +588,7 @@ class INSTRUMENTSCIENTIST(Base, EntityHelper): ) -class INVESTIGATION(Base, EntityHelper): +class INVESTIGATION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATION" __table_args__ = (Index("UNQ_INVESTIGATION_0", "FACILITY_ID", "NAME", "VISIT_ID"),) @@ -607,7 +620,7 @@ class INVESTIGATION(Base, EntityHelper): ) -class INVESTIGATIONGROUP(Base, EntityHelper): +class INVESTIGATIONGROUP(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATIONGROUP" __table_args__ = ( Index("UNQ_INVESTIGATIONGROUP_0", "GROUP_ID", "INVESTIGATION_ID", "ROLE"), @@ -636,7 +649,7 @@ class INVESTIGATIONGROUP(Base, EntityHelper): ) -class INVESTIGATIONINSTRUMENT(Base, EntityHelper): +class INVESTIGATIONINSTRUMENT(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATIONINSTRUMENT" __table_args__ = ( Index("UNQ_INVESTIGATIONINSTRUMENT_0", "INVESTIGATION_ID", "INSTRUMENT_ID"), @@ -662,7 +675,7 @@ class INVESTIGATIONINSTRUMENT(Base, EntityHelper): ) -class INVESTIGATIONPARAMETER(Base, EntityHelper): +class INVESTIGATIONPARAMETER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATIONPARAMETER" __table_args__ = ( Index("UNQ_INVESTIGATIONPARAMETER_0", "INVESTIGATION_ID", "PARAMETER_TYPE_ID"), @@ -696,7 +709,7 @@ class INVESTIGATIONPARAMETER(Base, EntityHelper): ) -class INVESTIGATIONTYPE(Base, EntityHelper): +class INVESTIGATIONTYPE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATIONTYPE" __table_args__ = (Index("UNQ_INVESTIGATIONTYPE_0", "NAME", "FACILITY_ID"),) @@ -716,7 +729,7 @@ class INVESTIGATIONTYPE(Base, EntityHelper): ) -class INVESTIGATIONUSER(Base, EntityHelper): +class INVESTIGATIONUSER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "INVESTIGATIONUSER" __table_args__ = ( Index("UNQ_INVESTIGATIONUSER_0", "USER_ID", "INVESTIGATION_ID", "ROLE"), @@ -745,7 +758,7 @@ class INVESTIGATIONUSER(Base, EntityHelper): ) -class JOB(Base, EntityHelper): +class JOB(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "JOB" ID = Column(BigInteger, primary_key=True) @@ -770,7 +783,7 @@ class JOB(Base, EntityHelper): ) -class KEYWORD(Base, EntityHelper): +class KEYWORD(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "KEYWORD" __table_args__ = (Index("UNQ_KEYWORD_0", "NAME", "INVESTIGATION_ID"),) @@ -791,7 +804,7 @@ class KEYWORD(Base, EntityHelper): ) -class PARAMETERTYPE(Base, EntityHelper): +class PARAMETERTYPE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "PARAMETERTYPE" __table_args__ = (Index("UNQ_PARAMETERTYPE_0", "FACILITY_ID", "NAME", "UNITS"),) @@ -828,7 +841,7 @@ class ValueTypeEnum(enum.Enum): ) -class PERMISSIBLESTRINGVALUE(Base, EntityHelper): +class PERMISSIBLESTRINGVALUE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "PERMISSIBLESTRINGVALUE" __table_args__ = ( Index("UNQ_PERMISSIBLESTRINGVALUE_0", "VALUE", "PARAMETERTYPE_ID"), @@ -851,7 +864,7 @@ class PERMISSIBLESTRINGVALUE(Base, EntityHelper): ) -class PUBLICATION(Base, EntityHelper): +class PUBLICATION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "PUBLICATION" ID = Column(BigInteger, primary_key=True) @@ -875,7 +888,7 @@ class PUBLICATION(Base, EntityHelper): ) -class PUBLICSTEP(Base, EntityHelper): +class PUBLICSTEP(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "PUBLICSTEP" __table_args__ = (Index("UNQ_PUBLICSTEP_0", "ORIGIN", "FIELD"),) @@ -888,7 +901,7 @@ class PUBLICSTEP(Base, EntityHelper): ORIGIN = Column(String(32), nullable=False) -class RELATEDDATAFILE(Base, EntityHelper): +class RELATEDDATAFILE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "RELATEDDATAFILE" __table_args__ = ( Index("UNQ_RELATEDDATAFILE_0", "SOURCE_DATAFILE_ID", "DEST_DATAFILE_ID"), @@ -910,7 +923,7 @@ class RELATEDDATAFILE(Base, EntityHelper): ) -class RULE(Base, EntityHelper): +class RULE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "RULE_" ID = Column(BigInteger, primary_key=True) @@ -937,7 +950,7 @@ class RULE(Base, EntityHelper): ) -class SAMPLE(Base, EntityHelper): +class SAMPLE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "SAMPLE" __table_args__ = (Index("UNQ_SAMPLE_0", "INVESTIGATION_ID", "NAME"),) @@ -962,7 +975,7 @@ class SAMPLE(Base, EntityHelper): ) -class SAMPLEPARAMETER(Base, EntityHelper): +class SAMPLEPARAMETER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "SAMPLEPARAMETER" __table_args__ = (Index("UNQ_SAMPLEPARAMETER_0", "SAMPLE_ID", "PARAMETER_TYPE_ID"),) @@ -994,7 +1007,7 @@ class SAMPLEPARAMETER(Base, EntityHelper): ) -class SESSION(Base, EntityHelper): +class SESSION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "SESSION_" ID = Column(String(255), primary_key=True) @@ -1002,7 +1015,7 @@ class SESSION(Base, EntityHelper): USERNAME = Column(String(255)) -class SHIFT(Base, EntityHelper): +class SHIFT(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "SHIFT" __table_args__ = (Index("UNQ_SHIFT_0", "INVESTIGATION_ID", "STARTDATE", "ENDDATE"),) @@ -1023,7 +1036,7 @@ class SHIFT(Base, EntityHelper): ) -class USER(Base, EntityHelper): +class USER(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "USER_" ID = Column(BigInteger, primary_key=True) @@ -1037,7 +1050,7 @@ class USER(Base, EntityHelper): ORCIDID = Column(String(255)) -class USERGROUP(Base, EntityHelper): +class USERGROUP(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "USERGROUP" __table_args__ = (Index("UNQ_USERGROUP_0", "USER_ID", "GROUP_ID"),) @@ -1059,7 +1072,7 @@ class USERGROUP(Base, EntityHelper): ) -class STUDYINVESTIGATION(Base, EntityHelper): +class STUDYINVESTIGATION(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "STUDYINVESTIGATION" __table_args__ = ( Index("UNQ_STUDYINVESTIGATION_0", "STUDY_ID", "INVESTIGATION_ID"), @@ -1087,7 +1100,7 @@ class STUDYINVESTIGATION(Base, EntityHelper): ) -class STUDY(Base, EntityHelper): +class STUDY(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "STUDY" ID = Column(BigInteger, primary_key=True) @@ -1106,7 +1119,7 @@ class STUDY(Base, EntityHelper): ) -class SAMPLETYPE(Base, EntityHelper): +class SAMPLETYPE(Base, EntityHelper, metaclass=EntityMeta): __tablename__ = "SAMPLETYPE" __table_args__ = ( Index("UNQ_SAMPLETYPE_0", "FACILITY_ID", "NAME", "MOLECULARFORMULA"), diff --git a/datagateway_api/common/date_handler.py b/datagateway_api/common/date_handler.py index 04709de8..097e23e6 100644 --- a/datagateway_api/common/date_handler.py +++ b/datagateway_api/common/date_handler.py @@ -1,8 +1,6 @@ -from datetime import datetime - from dateutil.parser import parse +from icat import helper -from datagateway_api.common.constants import Constants from datagateway_api.common.exceptions import BadRequestError @@ -41,33 +39,32 @@ def str_to_datetime_object(data): elegant solution to this conversion operation since dates are converted into ISO format within this file, however, the production instance of this API is typically built on Python 3.6, and it doesn't seem of enough value to mandate - 3.7 for a single line of code + 3.7 for a single line of code. Instead, a helper function from `python-icat` is + used which does the conversion using `suds`. This will convert inputs in the ISO + format (i.e. the format which Python ICAT, and therefore DataGateway API outputs + data) but also allows for conversion of other "sensible" formats. :param data: Single data value from the request body - :type data: Data type of the data as per user's request body + :type data: Data type of the data as per user's request body, :class:`str` is + assumed :return: Date converted into a :class:`datetime` object - :raises BadRequestError: If the date is entered in the incorrect format, as per - `Constants.ACCEPTED_DATE_FORMAT` + :raises BadRequestError: If there is an issue with the date format """ try: - data = datetime.strptime(data, Constants.ACCEPTED_DATE_FORMAT) - except ValueError: - raise BadRequestError( - "Bad request made, the date entered is not in the correct format. Use" - f" the {Constants.ACCEPTED_DATE_FORMAT} format to submit dates to the" - " API", - ) + datetime_obj = helper.parse_attr_string(data, "Date") + except ValueError as e: + raise BadRequestError(e) - return data + return datetime_obj @staticmethod - def datetime_object_to_str(date_obj): + def datetime_object_to_str(datetime_obj): """ Convert a datetime object to a string so it can be outputted in JSON - :param date_obj: Datetime object from data from an ICAT entity - :type date_obj: :class:`datetime.datetime` + :param datetime_obj: Datetime object from data from an ICAT entity + :type datetime_obj: :class:`datetime.datetime` :return: Datetime (of type string) in the agreed format """ - return date_obj.replace(tzinfo=None).strftime(Constants.ACCEPTED_DATE_FORMAT) + return datetime_obj.isoformat(" ") diff --git a/datagateway_api/common/icat/helpers.py b/datagateway_api/common/icat/helpers.py index 3b4fde95..0676733b 100644 --- a/datagateway_api/common/icat/helpers.py +++ b/datagateway_api/common/icat/helpers.py @@ -88,16 +88,16 @@ def get_session_details_helper(client): :return: Details of the user's session, ready to be converted into a JSON response body """ - # Remove rounding session_time_remaining = client.getRemainingMinutes() - session_expiry_time = datetime.now() + timedelta(minutes=session_time_remaining) - + session_expiry_time = ( + datetime.now() + timedelta(minutes=session_time_remaining) + ).replace(microsecond=0) username = client.getUserName() return { - "ID": client.sessionId, - "EXPIREDATETIME": str(session_expiry_time), - "USERNAME": username, + "id": client.sessionId, + "expireDateTime": DateHandler.datetime_object_to_str(session_expiry_time), + "username": username, } diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py new file mode 100644 index 00000000..036f596f --- /dev/null +++ b/datagateway_api/src/api_start_utils.py @@ -0,0 +1,160 @@ +import json +import logging +from pathlib import Path + +from apispec import APISpec +from flask_cors import CORS +from flask_restful import Api +from flask_swagger_ui import get_swaggerui_blueprint + +from datagateway_api.common.backends import create_backend +from datagateway_api.common.config import config +from datagateway_api.src.resources.entities.entity_endpoint import ( + get_count_endpoint, + get_endpoint, + get_find_one_endpoint, + get_id_endpoint, +) +from datagateway_api.src.resources.entities.entity_map import endpoints +from datagateway_api.src.resources.non_entities.sessions_endpoints import ( + session_endpoints, +) +from datagateway_api.src.resources.table_endpoints.table_endpoints import ( + count_instrument_facility_cycles_endpoint, + count_instrument_investigation_endpoint, + instrument_facility_cycles_endpoint, + instrument_investigation_endpoint, +) +from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin +from datagateway_api.src.swagger.initialise_spec import initialise_spec + +log = logging.getLogger() + + +class CustomErrorHandledApi(Api): + """ + This class overrides `handle_error` function from the API class from `flask_restful` + to correctly return response codes and exception messages from uncaught exceptions + """ + + def handle_error(self, e): + return str(e), e.status_code + + +def create_app_infrastructure(flask_app): + swaggerui_blueprint = get_swaggerui_blueprint( + "", "/openapi.json", config={"app_name": "DataGateway API OpenAPI Spec"}, + ) + flask_app.register_blueprint(swaggerui_blueprint, url_prefix="/") + spec = APISpec( + title="DataGateway API", + version="1.0", + openapi_version="3.0.3", + plugins=[RestfulPlugin()], + security=[{"session_id": []}], + ) + + CORS(flask_app) + flask_app.url_map.strict_slashes = False + api = CustomErrorHandledApi(flask_app) + + initialise_spec(spec) + + return (api, spec) + + +def create_api_endpoints(flask_app, api, spec): + try: + backend_type = flask_app.config["TEST_BACKEND"] + config.set_backend_type(backend_type) + except KeyError: + backend_type = config.get_backend_type() + + backend = create_backend(backend_type) + + for entity_name in endpoints: + get_endpoint_resource = get_endpoint( + entity_name, endpoints[entity_name], backend, + ) + api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") + spec.path(resource=get_endpoint_resource, api=api) + + get_id_endpoint_resource = get_id_endpoint( + entity_name, endpoints[entity_name], backend, + ) + api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") + spec.path(resource=get_id_endpoint_resource, api=api) + + get_count_endpoint_resource = get_count_endpoint( + entity_name, endpoints[entity_name], backend, + ) + api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") + spec.path(resource=get_count_endpoint_resource, api=api) + + get_find_one_endpoint_resource = get_find_one_endpoint( + entity_name, endpoints[entity_name], backend, + ) + api.add_resource( + get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone", + ) + spec.path(resource=get_find_one_endpoint_resource, api=api) + + # Session endpoint + session_endpoint_resource = session_endpoints(backend) + api.add_resource(session_endpoint_resource, "/sessions") + spec.path(resource=session_endpoint_resource, api=api) + + # Table specific endpoints + instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) + api.add_resource( + instrument_facility_cycle_resource, "/instruments//facilitycycles", + ) + spec.path(resource=instrument_facility_cycle_resource, api=api) + + count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( + backend, + ) + api.add_resource( + count_instrument_facility_cycle_res, + "/instruments//facilitycycles/count", + ) + spec.path(resource=count_instrument_facility_cycle_res, api=api) + + instrument_investigation_resource = instrument_investigation_endpoint(backend) + api.add_resource( + instrument_investigation_resource, + "/instruments//facilitycycles//investigations", + ) + spec.path(resource=instrument_investigation_resource, api=api) + + count_instrument_investigation_resource = count_instrument_investigation_endpoint( + backend, + ) + api.add_resource( + count_instrument_investigation_resource, + "/instruments//facilitycycles//investigations" + "/count", + ) + spec.path(resource=count_instrument_investigation_resource, api=api) + + +def openapi_config(spec): + # Reorder paths (e.g. get, patch, post) so openapi.yaml only changes when there's a + # change to the Swagger docs, rather than changing on each startup + if config.is_generate_swagger(): + log.debug("Reordering OpenAPI docs to alphabetical order") + for entity_data in spec._paths.values(): + for endpoint_name in sorted(entity_data.keys()): + entity_data.move_to_end(endpoint_name) + + openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" + with open(openapi_spec_path, "w") as f: + f.write(spec.to_yaml()) + + +def create_openapi_endpoint(app, api_spec): + @app.route("/openapi.json") + def specs(): + resp = app.make_response(json.dumps(api_spec.to_dict(), indent=2)) + resp.mimetype = "application/json" + return resp diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index 35debd92..9af05599 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -1,183 +1,27 @@ -import json import logging -from pathlib import Path -from apispec import APISpec from flask import Flask -from flask_cors import CORS -from flask_restful import Api -from flask_swagger_ui import get_swaggerui_blueprint -from flask_sqlalchemy import SQLAlchemy -from datagateway_api.common.backends import create_backend from datagateway_api.common.config import config from datagateway_api.common.logger_setup import setup_logger -from datagateway_api.common.database.session_manager import db -from datagateway_api.common.constants import Constants -from datagateway_api.src.resources.entities.entity_endpoint import ( - get_count_endpoint, - get_endpoint, - get_find_one_endpoint, - get_id_endpoint, +from datagateway_api.src.api_start_utils import ( + create_api_endpoints, + create_app_infrastructure, + create_openapi_endpoint, + openapi_config, ) -from datagateway_api.src.resources.entities.entity_map import endpoints -from datagateway_api.src.resources.non_entities.sessions_endpoints import ( - session_endpoints, -) -from datagateway_api.src.resources.table_endpoints.table_endpoints import ( - count_instrument_facility_cycles_endpoint, - count_instrument_investigation_endpoint, - instrument_facility_cycles_endpoint, - instrument_investigation_endpoint, -) -from datagateway_api.src.swagger.apispec_flask_restful import RestfulPlugin -from datagateway_api.src.swagger.initialise_spec import initialise_spec setup_logger() log = logging.getLogger() log.info("Logging now setup") app = Flask(__name__) - - -class CustomErrorHandledApi(Api): - """ - This class overrides `handle_error` function from the API class from `flask_restful` - to correctly return response codes and exception messages from uncaught exceptions - """ - - def handle_error(self, e): - return str(e), e.status_code - - -def create_app_infrastructure(flask_app): - swaggerui_blueprint = get_swaggerui_blueprint( - "", "/openapi.json", config={"app_name": "DataGateway API OpenAPI Spec"}, - ) - flask_app.register_blueprint(swaggerui_blueprint, url_prefix="/") - spec = APISpec( - title="DataGateway API", - version="1.0", - openapi_version="3.0.3", - plugins=[RestfulPlugin()], - security=[{"session_id": []}], - ) - - CORS(flask_app) - flask_app.url_map.strict_slashes = False - api = CustomErrorHandledApi(flask_app) - app.config["SQLALCHEMY_DATABASE_URI"] = Constants.DATABASE_URL - db.init_app(app) - - initialise_spec(spec) - - return (api, spec) - - -def handle_error(e): - return str(e), e.status_code - - -def create_api_endpoints(flask_app, api, spec): - try: - backend_type = flask_app.config["TEST_BACKEND"] - config.set_backend_type(backend_type) - except KeyError: - backend_type = config.get_backend_type() - - backend = create_backend(backend_type) - - for entity_name in endpoints: - get_endpoint_resource = get_endpoint( - entity_name, endpoints[entity_name], backend, - ) - api.add_resource(get_endpoint_resource, f"/{entity_name.lower()}") - spec.path(resource=get_endpoint_resource, api=api) - - get_id_endpoint_resource = get_id_endpoint( - entity_name, endpoints[entity_name], backend, - ) - api.add_resource(get_id_endpoint_resource, f"/{entity_name.lower()}/") - spec.path(resource=get_id_endpoint_resource, api=api) - - get_count_endpoint_resource = get_count_endpoint( - entity_name, endpoints[entity_name], backend, - ) - api.add_resource(get_count_endpoint_resource, f"/{entity_name.lower()}/count") - spec.path(resource=get_count_endpoint_resource, api=api) - - get_find_one_endpoint_resource = get_find_one_endpoint( - entity_name, endpoints[entity_name], backend, - ) - api.add_resource( - get_find_one_endpoint_resource, f"/{entity_name.lower()}/findone", - ) - spec.path(resource=get_find_one_endpoint_resource, api=api) - - # Session endpoint - session_endpoint_resource = session_endpoints(backend) - api.add_resource(session_endpoint_resource, "/sessions") - spec.path(resource=session_endpoint_resource, api=api) - - # Table specific endpoints - instrument_facility_cycle_resource = instrument_facility_cycles_endpoint(backend) - api.add_resource( - instrument_facility_cycle_resource, "/instruments//facilitycycles", - ) - spec.path(resource=instrument_facility_cycle_resource, api=api) - - count_instrument_facility_cycle_res = count_instrument_facility_cycles_endpoint( - backend, - ) - api.add_resource( - count_instrument_facility_cycle_res, - "/instruments//facilitycycles/count", - ) - spec.path(resource=count_instrument_facility_cycle_res, api=api) - - instrument_investigation_resource = instrument_investigation_endpoint(backend) - api.add_resource( - instrument_investigation_resource, - "/instruments//facilitycycles//investigations", - ) - spec.path(resource=instrument_investigation_resource, api=api) - - count_instrument_investigation_resource = count_instrument_investigation_endpoint( - backend, - ) - api.add_resource( - count_instrument_investigation_resource, - "/instruments//facilitycycles//investigations" - "/count", - ) - spec.path(resource=count_instrument_investigation_resource, api=api) - - -def openapi_config(spec): - # Reorder paths (e.g. get, patch, post) so openapi.yaml only changes when there's a - # change to the Swagger docs, rather than changing on each startup - if config.is_generate_swagger(): - log.debug("Reordering OpenAPI docs to alphabetical order") - for entity_data in spec._paths.values(): - for endpoint_name in sorted(entity_data.keys()): - entity_data.move_to_end(endpoint_name) - - openapi_spec_path = Path(__file__).parent / "swagger/openapi.yaml" - with open(openapi_spec_path, "w") as f: - f.write(spec.to_yaml()) - - -@app.route("/openapi.json") -def specs(): - resp = app.make_response(json.dumps(spec.to_dict(), indent=2)) - resp.mimetype = "application/json" - return resp - api, spec = create_app_infrastructure(app) create_api_endpoints(app, api, spec) +openapi_config(spec) +create_openapi_endpoint(app, spec) if __name__ == "__main__": - openapi_config(spec) app.run( host=config.get_host(), port=config.get_port(), debug=config.is_debug_mode(), ) diff --git a/noxfile.py b/noxfile.py index 6960f90d..4a97e124 100644 --- a/noxfile.py +++ b/noxfile.py @@ -31,7 +31,7 @@ def install_with_constraints(session, *args, **kwargs): session.log("Error: The temporary requirements file could not be closed") -@nox.session(python="3.6", reuse_venv=True) +@nox.session(reuse_venv=True) def black(session): args = session.posargs or code_locations @@ -39,7 +39,7 @@ def black(session): session.run("black", *args, external=True) -@nox.session(python="3.6", reuse_venv=True) +@nox.session(reuse_venv=True) def lint(session): args = session.posargs or code_locations install_with_constraints( @@ -59,7 +59,7 @@ def lint(session): session.run("flake8", *args) -@nox.session(python="3.6", reuse_venv=True) +@nox.session(reuse_venv=True) def safety(session): install_with_constraints(session, "safety") with tempfile.NamedTemporaryFile(delete=False) as requirements: @@ -82,7 +82,7 @@ def safety(session): session.log("Error: The temporary requirements file could not be closed") -@nox.session(python=["3.6", "3.7", "3.8"], reuse_venv=True) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"], reuse_venv=True) def tests(session): args = session.posargs session.run("poetry", "install", external=True) diff --git a/poetry.lock b/poetry.lock index 0ec30e5d..4a95e699 100644 --- a/poetry.lock +++ b/poetry.lock @@ -474,6 +474,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" pyparsing = ">=2.0.2" six = "*" +[[package]] +name = "pathlib2" +version = "2.3.5" +description = "Object-oriented filesystem paths" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + [[package]] name = "pathspec" version = "0.8.1" @@ -541,7 +552,7 @@ python-versions = "*" [[package]] name = "py" -version = "1.9.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" optional = false @@ -565,13 +576,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pymysql" -version = "0.9.3" +version = "1.0.2" description = "Pure Python MySQL Driver" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.extras] +ed25519 = ["PyNaCl (>=1.4.0)"] rsa = ["cryptography"] [[package]] @@ -592,6 +604,8 @@ python-versions = ">=3.5" [package.dependencies] packaging = "*" +pathlib2 = {version = ">=2.2.0", markers = "python_version < \"3.6\""} +pluggy = ">=0.12,<1.0" py = ">=1.8.2" iniconfig = "*" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -726,7 +740,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "sqlalchemy" -version = "1.3.8" +version = "1.3.22" description = "Database Abstraction Library" category = "main" optional = false @@ -827,8 +841,8 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "270d9adcbf9a704468e4e264bf85ca1c27fe66122d402e24a1ff7193636894ce" +python-versions = "^3.5" +content-hash = "db66f1e49ef4c5f9929c189d886b7c6b7d4306044a31e0eed7993d3ab6e9baab" [metadata.files] aniso8601 = [ @@ -1053,6 +1067,10 @@ packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, ] +pathlib2 = [ + {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"}, + {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"}, +] pathspec = [ {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, @@ -1078,8 +1096,8 @@ pprintpp = [ {file = "pprintpp-0.4.0.tar.gz", hash = "sha256:ea826108e2c7f49dc6d66c752973c3fc9749142a798d6b254e1e301cfdbc6403"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, @@ -1090,8 +1108,8 @@ pyflakes = [ {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] pymysql = [ - {file = "PyMySQL-0.9.3-py2.py3-none-any.whl", hash = "sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a"}, - {file = "PyMySQL-0.9.3.tar.gz", hash = "sha256:d8c059dcd81dedb85a9f034d5e22dcb4442c0b201908bede99e306d65ea7c8e7"}, + {file = "PyMySQL-1.0.2-py3-none-any.whl", hash = "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641"}, + {file = "PyMySQL-1.0.2.tar.gz", hash = "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, @@ -1196,7 +1214,44 @@ smmap = [ {file = "smmap-3.0.4.tar.gz", hash = "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.3.8.tar.gz", hash = "sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:61628715931f4962e0cdb2a7c87ff39eea320d2aa96bd471a3c293d146f90394"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:81d8d099a49f83111cce55ec03cc87eef45eec0d90f9842b4fc674f860b857b0"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:d055ff750fcab69ca4e57b656d9c6ad33682e9b8d564f2fbe667ab95c63591b0"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-win32.whl", hash = "sha256:9bf572e4f5aa23f88dd902f10bb103cb5979022a38eec684bfa6d61851173fec"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27m-win_amd64.whl", hash = "sha256:7d4b8de6bb0bc736161cb0bbd95366b11b3eb24dd6b814a143d8375e75af9990"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4a84c7c7658dd22a33dab2e2aa2d17c18cb004a42388246f2e87cb4085ef2811"}, + {file = "SQLAlchemy-1.3.22-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:f1e88b30da8163215eab643962ae9d9252e47b4ea53404f2c4f10f24e70ddc62"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:f115150cc4361dd46153302a640c7fa1804ac207f9cc356228248e351a8b4676"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6aaa13ee40c4552d5f3a59f543f0db6e31712cc4009ec7385407be4627259d41"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3ab5b44a07b8c562c6dcb7433c6a6c6e03266d19d64f87b3333eda34e3b9936b"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:426ece890153ccc52cc5151a1a0ed540a5a7825414139bb4c95a868d8da54a52"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-win32.whl", hash = "sha256:bd4b1af45fd322dcd1fb2a9195b4f93f570d1a5902a842e3e6051385fac88f9c"}, + {file = "SQLAlchemy-1.3.22-cp35-cp35m-win_amd64.whl", hash = "sha256:62285607a5264d1f91590abd874d6a498e229d5840669bd7d9f654cfaa599bd0"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:314f5042c0b047438e19401d5f29757a511cfc2f0c40d28047ca0e4c95eabb5b"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:62fb881ba51dbacba9af9b779211cf9acff3442d4f2993142015b22b3cd1f92a"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:bde677047305fe76c7ee3e4492b545e0018918e44141cc154fe39e124e433991"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:0c6406a78a714a540d980a680b86654feadb81c8d0eecb59f3d6c554a4c69f19"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-win32.whl", hash = "sha256:95bde07d19c146d608bccb9b16e144ec8f139bcfe7fd72331858698a71c9b4f5"}, + {file = "SQLAlchemy-1.3.22-cp36-cp36m-win_amd64.whl", hash = "sha256:888d5b4b5aeed0d3449de93ea80173653e939e916cc95fe8527079e50235c1d2"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:d53f59744b01f1440a1b0973ed2c3a7de204135c593299ee997828aad5191693"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:70121f0ae48b25ef3e56e477b88cd0b0af0e1f3a53b5554071aa6a93ef378a03"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:54da615e5b92c339e339fe8536cce99fe823b6ed505d4ea344852aefa1c205fb"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:68428818cf80c60dc04aa0f38da20ad39b28aba4d4d199f949e7d6e04444ea86"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-win32.whl", hash = "sha256:17610d573e698bf395afbbff946544fbce7c5f4ee77b5bcb1f821b36345fae7a"}, + {file = "SQLAlchemy-1.3.22-cp37-cp37m-win_amd64.whl", hash = "sha256:216ba5b4299c95ed179b58f298bda885a476b16288ab7243e89f29f6aeced7e0"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:0c72b90988be749e04eff0342dcc98c18a14461eb4b2ad59d611b57b31120f90"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:491fe48adc07d13e020a8b07ef82eefc227003a046809c121bea81d3dbf1832d"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:f8191fef303025879e6c3548ecd8a95aafc0728c764ab72ec51a0bdf0c91a341"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:108580808803c7732f34798eb4a329d45b04c562ed83ee90f09f6a184a42b766"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-win32.whl", hash = "sha256:bab5a1e15b9466a25c96cda19139f3beb3e669794373b9ce28c4cf158c6e841d"}, + {file = "SQLAlchemy-1.3.22-cp38-cp38-win_amd64.whl", hash = "sha256:318b5b727e00662e5fc4b4cd2bf58a5116d7c1b4dd56ffaa7d68f43458a8d1ed"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:1418f5e71d6081aa1095a1d6b567a562d2761996710bdce9b6e6ba20a03d0864"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:5a7f224cdb7233182cec2a45d4c633951268d6a9bcedac37abbf79dd07012aea"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:715b34578cc740b743361f7c3e5f584b04b0f1344f45afc4e87fbac4802eb0a0"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2ff132a379838b1abf83c065be54cef32b47c987aedd06b82fc76476c85225eb"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-win32.whl", hash = "sha256:c389d7cc2b821853fb018c85457da3e7941db64f4387720a329bc7ff06a27963"}, + {file = "SQLAlchemy-1.3.22-cp39-cp39-win_amd64.whl", hash = "sha256:04f995fcbf54e46cddeb4f75ce9dfc17075d6ae04ac23b2bacb44b3bc6f6bf11"}, + {file = "SQLAlchemy-1.3.22.tar.gz", hash = "sha256:758fc8c4d6c0336e617f9f6919f9daea3ab6bb9b07005eda9a1a682e24a6cacc"}, ] stevedore = [ {file = "stevedore-3.2.2-py3-none-any.whl", hash = "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62"}, diff --git a/pyproject.toml b/pyproject.toml index 1891a50d..d92a4b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,10 +9,10 @@ repository = "https://github.com/ral-facilities/datagateway-api" authors = ["Matthew Richards "] [tool.poetry.dependencies] -python = "^3.6" +python = "^3.5" Flask-RESTful = "0.3.7" -SQLAlchemy = "1.3.8" -PyMySQL = "0.9.3" +SQLAlchemy = "^1.3.8" +PyMySQL = "1.0.2" Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" @@ -43,7 +43,6 @@ pytest-cov = "^2.10.1" pytest-icdiff = "^0.5" [tool.poetry.scripts] -datagateway-api = "datagateway_api.src.main:run_api" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test/conftest.py b/test/conftest.py index 05313099..9d29d01e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,7 +8,10 @@ insert_row_into_table, ) from datagateway_api.common.database.models import SESSION -from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure +from datagateway_api.src.api_start_utils import ( + create_api_endpoints, + create_app_infrastructure, +) @pytest.fixture() @@ -33,6 +36,10 @@ def flask_test_app(): @pytest.fixture(scope="package") def flask_test_app_db(): + """ + This is in the common conftest file because this test app is also used in + non-backend specific tests + """ db_app = Flask(__name__) db_app.config["TESTING"] = True db_app.config["TEST_BACKEND"] = "db" diff --git a/test/icat/conftest.py b/test/icat/conftest.py index 2549dd83..48198dad 100644 --- a/test/icat/conftest.py +++ b/test/icat/conftest.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone import uuid from flask import Flask @@ -8,7 +8,11 @@ import pytest from datagateway_api.common.config import config -from datagateway_api.src.main import create_api_endpoints, create_app_infrastructure +from datagateway_api.src.api_start_utils import ( + create_api_endpoints, + create_app_infrastructure, +) +from test.icat.endpoints.test_create_icat import TestICATCreateData from test.icat.test_query import prepare_icat_data_for_assertion @@ -39,10 +43,10 @@ def create_investigation_test_data(client, num_entities=1): f"Test data for the Python ICAT Backend on DataGateway API {i}" ) investigation.startDate = datetime( - year=2020, month=1, day=4, hour=1, minute=1, second=1, + year=2020, month=1, day=4, hour=1, minute=1, second=1, tzinfo=timezone.utc, ) investigation.endDate = datetime( - year=2020, month=1, day=8, hour=1, minute=1, second=1, + year=2020, month=1, day=8, hour=1, minute=1, second=1, tzinfo=timezone.utc, ) # UUID visit ID means uniquesness constraint should always be met investigation.visitId = str(uuid.uuid1()) @@ -102,10 +106,10 @@ def isis_specific_endpoint_data(icat_client): facility_cycle = icat_client.new("facilityCycle") facility_cycle.name = "Test cycle for DataGateway API testing" facility_cycle.startDate = datetime( - year=2020, month=1, day=1, hour=1, minute=1, second=1, + year=2020, month=1, day=1, hour=1, minute=1, second=1, tzinfo=timezone.utc, ) facility_cycle.endDate = datetime( - year=2020, month=2, day=1, hour=1, minute=1, second=1, + year=2020, month=2, day=1, hour=1, minute=1, second=1, tzinfo=timezone.utc, ) facility_cycle.facility = icat_client.get("Facility", 1) facility_cycle.create() @@ -151,3 +155,37 @@ def final_facilitycycle_id(flask_test_app_icat, valid_icat_credentials_header): headers=valid_icat_credentials_header, ) return final_facilitycycle_result.json["id"] + + +@pytest.fixture() +def remove_test_created_investigation_data( + flask_test_app_icat, valid_icat_credentials_header, +): + """ + This is used to delete the data created inside `test_valid` test functions in + TestICATCreateData + + This is done by fetching the data which has been created in + those functions (by using the investigation name prefix, as defined in the test + class), extracting the IDs from the results, and iterating over those to perform + DELETE by ID requests + """ + + yield + + created_test_data = flask_test_app_icat.get( + '/investigations?where={"name":{"like":' + f'"{TestICATCreateData.investigation_name_prefix}"' + "}}", + headers=valid_icat_credentials_header, + ) + + investigation_ids = [] + for investigation in created_test_data.json: + investigation_ids.append(investigation["id"]) + + for investigation_id in investigation_ids: + flask_test_app_icat.delete( + f"/investigations/{investigation_id}", + headers=valid_icat_credentials_header, + ) diff --git a/test/icat/endpoints/test_create_icat.py b/test/icat/endpoints/test_create_icat.py index 79e4e198..c20aeaba 100644 --- a/test/icat/endpoints/test_create_icat.py +++ b/test/icat/endpoints/test_create_icat.py @@ -1,18 +1,23 @@ +import pytest + from test.icat.test_query import prepare_icat_data_for_assertion class TestICATCreateData: + investigation_name_prefix = "Test Data for API Testing, Data Creation" + + @pytest.mark.usefixtures("remove_test_created_investigation_data") def test_valid_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): create_investigations_json = [ { - "name": f"Test Data for API Testing, Data Creation {i}", + "name": f"{self.investigation_name_prefix} {i}", "title": "Test data for the Python ICAT Backend on DataGateway API", "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", + "releaseDate": "2020-03-03 08:00:08+00:00", + "startDate": "2020-02-02 09:00:09+00:00", + "endDate": "2020-02-03 10:00:10+00:00", "visitId": "Data Creation Visit", "doi": "DataGateway API Test DOI", "facility": 1, @@ -27,13 +32,9 @@ def test_valid_create_data( json=create_investigations_json, ) - test_data_ids = [] - for investigation_request, investigation_response in zip( - create_investigations_json, test_response.json, - ): + for investigation_request in create_investigations_json: investigation_request.pop("facility") investigation_request.pop("type") - test_data_ids.append(investigation_response["id"]) response_json = prepare_icat_data_for_assertion( test_response.json, remove_id=True, @@ -41,25 +42,19 @@ def test_valid_create_data( assert create_investigations_json == response_json - # Delete the entities created by this test - for investigation_id in test_data_ids: - flask_test_app_icat.delete( - f"/investigations/{investigation_id}", - headers=valid_icat_credentials_header, - ) - + @pytest.mark.usefixtures("remove_test_created_investigation_data") def test_valid_boundary_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): """Create a single investigation, as opposed to multiple""" create_investigation_json = { - "name": "Test Data for API Testing, Data Creation 0", + "name": f"{self.investigation_name_prefix} 0", "title": "Test data for the Python ICAT Backend on the API", "summary": "Test data for DataGateway API testing", - "releaseDate": "2020-03-03 08:00:08", - "startDate": "2020-02-02 09:00:09", - "endDate": "2020-02-03 10:00:10", + "releaseDate": "2020-03-03 08:00:08+00:00", + "startDate": "2020-02-02 09:00:09+00:00", + "endDate": "2020-02-03 10:00:10+00:00", "visitId": "Data Creation Visit", "doi": "DataGateway API Test DOI", "facility": 1, @@ -74,7 +69,6 @@ def test_valid_boundary_create_data( create_investigation_json.pop("facility") create_investigation_json.pop("type") - created_test_data_id = test_response.json[0]["id"] response_json = prepare_icat_data_for_assertion( test_response.json, remove_id=True, @@ -82,11 +76,6 @@ def test_valid_boundary_create_data( assert [create_investigation_json] == response_json - flask_test_app_icat.delete( - f"/investigations/{created_test_data_id}", - headers=valid_icat_credentials_header, - ) - def test_invalid_create_data( self, flask_test_app_icat, valid_icat_credentials_header, ): diff --git a/test/icat/test_session_handling.py b/test/icat/test_session_handling.py index 2018ca01..1e291c60 100644 --- a/test/icat/test_session_handling.py +++ b/test/icat/test_session_handling.py @@ -16,7 +16,7 @@ def test_get_valid_session_details( ) session_expiry_datetime = datetime.strptime( - session_details.json["EXPIREDATETIME"], "%Y-%m-%d %H:%M:%S.%f", + session_details.json["expireDateTime"], "%Y-%m-%d %H:%M:%S", ) current_datetime = datetime.now() time_diff = abs(session_expiry_datetime - current_datetime) @@ -27,13 +27,13 @@ def test_get_valid_session_details( # Check username is correct assert ( - session_details.json["USERNAME"] == f"{config.get_test_mechanism()}/" + session_details.json["username"] == f"{config.get_test_mechanism()}/" f"{config.get_test_user_credentials()['username']}" ) # Check session ID matches the header from the request assert ( - session_details.json["ID"] + session_details.json["id"] == valid_icat_credentials_header["Authorization"].split()[1] ) @@ -62,8 +62,8 @@ def test_refresh_session(self, valid_icat_credentials_header, flask_test_app_ica assert refresh_session.status_code == 200 assert ( - pre_refresh_session_details.json["EXPIREDATETIME"] - != post_refresh_session_details.json["EXPIREDATETIME"] + pre_refresh_session_details.json["expireDateTime"] + != post_refresh_session_details.json["expireDateTime"] ) @pytest.mark.usefixtures("single_investigation_test_data") diff --git a/test/test_config.py b/test/test_config.py index 647ad98f..4d5da769 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -33,7 +33,7 @@ def test_invalid_backend_type(self, invalid_config): class TestGetDBURL: def test_valid_db_url(self, valid_config): db_url = valid_config.get_db_url() - assert db_url == "mysql+pymysql://root:rootpw@localhost:13306/icatdb" + assert db_url == "mysql+pymysql://icatdbuser:icatdbuserpw@localhost:3306/icatdb" def test_invalid_db_url(self, invalid_config): with pytest.raises(SystemExit): @@ -43,7 +43,7 @@ def test_invalid_db_url(self, invalid_config): class TestICATURL: def test_valid_icat_url(self, valid_config): icat_url = valid_config.get_icat_url() - assert icat_url == "https://localhost.localdomain:8181" + assert icat_url == "https://localhost:8181" def test_invalid_icat_url(self, invalid_config): with pytest.raises(SystemExit): @@ -73,7 +73,9 @@ def test_invalid_log_level(self, invalid_config): class TestGetLogLocation: def test_valid_log_location(self, valid_config): log_location = valid_config.get_log_location() - assert log_location == "/home/user1/datagateway-api/logs.log" + assert ( + log_location == "/home/runner/work/datagateway-api/datagateway-api/logs.log" + ) def test_invalid_log_location(self, invalid_config): with pytest.raises(SystemExit):