From a4ca2df69c2adc1859e8df592596c303b6272277 Mon Sep 17 00:00:00 2001 From: Matthew Richards <32678030+MRichards99@users.noreply.github.com> Date: Fri, 4 Dec 2020 13:35:03 +0000 Subject: [PATCH 1/5] Update README.md Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f9e00a89..2ababa99 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ machine. ### pyenv (Python Version Management) To start, install [pyenv](https://github.com/pyenv/pyenv). There is a Windows version of this tool ([pyenv-win](https://github.com/pyenv-win/pyenv-win)), however this is -currently untested on this repoThis is used to manage the various versions of Python +currently untested on this repo. This is used to manage the various versions of Python that will be used to test/lint Python during development. Install by executing the following: ```bash From b80347ff5ba975e0a370704e47eeb90ec4fa57b1 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Dec 2020 16:37:39 +0000 Subject: [PATCH 2/5] #165: Update repo tree structure - Also added a command to generate this tree in the future (I just used to update it manually). --- README.md | 102 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index f9e00a89..aa959710 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,8 @@ used to lint/format/test the code in the included `noxfile.py`. To install Nox, as shown below. Nox is not listed as a Poetry dependency because this has the potential to cause issues if Nox was executed inside Poetry (see [here](https://medium.com/@cjolowicz/nox-is-a-part-of-your-global-developer-environment-like-poetry-pre-commit-pyenv-or-pipx-1cdeba9198bd) -for more detailed reasoning): +for more detailed reasoning). If you do choose to install these packages within a +virtual environment, you do not need the `--user` option: ```bash pip install --user --upgrade nox @@ -278,45 +279,68 @@ This is illustrated below. ````` -─── datagateway_api - ├── common - │ ├── database - │ │ ├── backend.py - │ │ ├── filters.py - │ │ ├── helpers.py - │ │ ├── models.py - │ │ └── session_manager.py - │ ├── icat - │ ├── backends.py - │ ├── constants.py - │ ├── exceptions.py - │ ├── filters.py - │ └── helpers.py - ├── src - │ ├── resources - │ │ ├── entities - │ │ │ ├── entity_endpoint.py - │ │ │ └── entity_map.py - │ │ └── non_entities - │ │ └── _endpoints.py - │ ├── swagger - │ │ ├── openapi.yaml - │ │ └── swagger_generator.py - │ └── main.py - ├── test - │ ├── resources - │ │ ├── entities - │ │ │ └──test_.py - │ │ └── non_entities - │ │ └── test_.py - │ └── test_base - │ ├── constants.py - │ └── rest_test.py - ├── util - │ └── icat_db_generator.py - ├── logs.log - └── config.json +. +├── .flake8 +├── .gitignore +├── .pre-commit-config.yaml +├── LICENSE +├── README.md +├── config.json.example +├── datagateway_api +│ ├── common +│ │ ├── backend.py +│ │ ├── backends.py +│ │ ├── config.py +│ │ ├── constants.py +│ │ ├── database +│ │ │ ├── backend.py +│ │ │ ├── filters.py +│ │ │ ├── helpers.py +│ │ │ ├── models.py +│ │ │ └── session_manager.py +│ │ ├── date_handler.py +│ │ ├── exceptions.py +│ │ ├── filter_order_handler.py +│ │ ├── filters.py +│ │ ├── helpers.py +│ │ ├── icat +│ │ │ ├── backend.py +│ │ │ ├── filters.py +│ │ │ ├── helpers.py +│ │ │ └── query.py +│ │ └── logger_setup.py +│ └── src +│ ├── main.py +│ ├── resources +│ │ ├── entities +│ │ │ ├── entity_endpoint.py +│ │ │ └── entity_map.py +│ │ ├── non_entities +│ │ │ └── sessions_endpoints.py +│ │ └── table_endpoints +│ │ └── table_endpoints.py +│ └── swagger +│ ├── apispec_flask_restful.py +│ ├── initialise_spec.py +│ └── openapi.yaml +├── noxfile.py +├── poetry.lock +├── postman_collection_icat.json +├── pyproject.toml +├── test +│ ├── test_base.py +│ ├── test_database_helpers.py +│ ├── test_entityHelper.py +│ └── test_helpers.py +└── util + └── icat_db_generator.py ````` + +The directory tree can be generated using the following command: + + `git ls-tree -r --name-only HEAD | grep -v __init__.py | tree --fromfile` + + #### Main `main.py` is where the flask_restful api is set up. This is where each endpoint resource class is generated and mapped to an endpoint. From 0366d08d0b531c98d229e9ce3c849e00c2d8f043 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 8 Dec 2020 10:42:27 +0000 Subject: [PATCH 3/5] #165: Allow tmp directory to be configured for nox sessions --- README.md | 18 ++++++++++- .../src/swagger/initialise_spec.py | 2 +- noxfile.py | 32 +++++++++++++++---- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7deb930b..b3f43209 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ machine. To start, install [pyenv](https://github.com/pyenv/pyenv). There is a Windows version of this tool ([pyenv-win](https://github.com/pyenv-win/pyenv-win)), however this is currently untested on this repo. This is used to manage the various versions of Python -that will be used to test/lint Python during development. Install by executing the following: +that will be used to test/lint Python during development. Install by executing the +following: ```bash curl https://pyenv.run | bash @@ -154,6 +155,21 @@ Currently, the following Nox sessions have been created: dependencies (pulled directly from Poetry) for any known vulnerabilities. This session gives the output in a full ASCII style report. +Each Nox session builds an environment using the repo's dependencies (defined using +Poetry) using `install_with_constraints()`. This stores the dependencies in a +`requirements.txt`-like format temporarily during this process, using the OS' default +temporary location. This could result in permissions issues (this has been seen by a +colleague on Windows), so adding the `--tmpdir [DIRECTORY PATH]` allows the user to +define where this file should be stored. Due to Nox session being initiated in the +command line, this argument needs to be a positional argument (denoted by the `--` in +the Nox command). This argument is optional, but **must** be the final argument avoid +interference with Nox's argument parsing. An example: + +```bash +nox -s lint -- util datagateway_api --tmpdir /root +``` + + ### Pre Commit (Automated Checks during Git Commit) To make use of Git's ability to run custom hooks, [pre-commit](https://pre-commit.com/) is used. Like Nox, Pip is used to install this tool: diff --git a/datagateway_api/src/swagger/initialise_spec.py b/datagateway_api/src/swagger/initialise_spec.py index 7186a8de..4b635a44 100644 --- a/datagateway_api/src/swagger/initialise_spec.py +++ b/datagateway_api/src/swagger/initialise_spec.py @@ -145,7 +145,7 @@ def initialise_spec(spec): "lt": {"value": [{"id": {"lt": 10}}]}, "lte": {"value": [{"id": {"lte": 50}}]}, "gt": {"value": [{"id": {"gt": 10}}]}, - "gte": {"value": [{"id": {"gte": 50}}]}, + "gte": {"value": [{"id": {"gte": 50}}]}, "in": {"value": [{"id": {"in": [1, 2, 3]}}]}, }, }, diff --git a/noxfile.py b/noxfile.py index a256a64a..e0abbe40 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,13 +1,15 @@ -import nox import tempfile +import nox + + # Separating Black away from the rest of the sessions nox.options.sessions = "lint", "safety" code_locations = "datagateway_api", "test", "util", "noxfile.py" -def install_with_constraints(session, *args, **kwargs): - with tempfile.NamedTemporaryFile() as requirements: +def install_with_constraints(session, req_dir=None, *args, **kwargs): + with tempfile.NamedTemporaryFile(dir=req_dir) as requirements: session.run( "poetry", "export", @@ -19,18 +21,35 @@ def install_with_constraints(session, *args, **kwargs): session.install(f"--constraint={requirements.name}", *args, **kwargs) +def get_tmp_dir(session): + tmp_dir = None + + try: + if session.posargs[-2] == "--tmpdir": + tmp_dir = session.posargs.pop(-1) + session.posargs.remove("--tmpdir") + except IndexError: + session.log("Info: No --tmpdir option given") + + return tmp_dir + + @nox.session(python="3.6", reuse_venv=True) def format(session): + tmp_dir = get_tmp_dir(session) args = session.posargs or code_locations - install_with_constraints(session, "black") + + install_with_constraints(session, tmp_dir, "black") session.run("black", *args, external=True) @nox.session(python="3.6", reuse_venv=True) def lint(session): + tmp_dir = get_tmp_dir(session) args = session.posargs or code_locations install_with_constraints( session, + tmp_dir, "flake8", "flake8-bandit", "flake8-black", @@ -48,8 +67,9 @@ def lint(session): @nox.session(python="3.6", reuse_venv=True) def safety(session): - install_with_constraints(session, "safety") - with tempfile.NamedTemporaryFile() as requirements: + tmp_dir = get_tmp_dir(session) + install_with_constraints(session, tmp_dir, "safety") + with tempfile.NamedTemporaryFile(dir=tmp_dir) as requirements: session.run( "poetry", "export", From b01eaf503210d84cfe63fd3077d8d7cae2ad910f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 8 Dec 2020 15:18:19 +0000 Subject: [PATCH 4/5] #184: Fix linting issues as brought up in PR - Most likely caused when I merged other branches after I created the PR --- datagateway_api/src/main.py | 1 - datagateway_api/src/swagger/initialise_spec.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/datagateway_api/src/main.py b/datagateway_api/src/main.py index d7d2950b..d971d724 100644 --- a/datagateway_api/src/main.py +++ b/datagateway_api/src/main.py @@ -9,7 +9,6 @@ from flask_swagger_ui import get_swaggerui_blueprint from datagateway_api.common.config import config -from datagateway_api.common.exceptions import ApiError from datagateway_api.common.logger_setup import setup_logger from datagateway_api.src.resources.entities.entity_endpoint import ( get_count_endpoint, diff --git a/datagateway_api/src/swagger/initialise_spec.py b/datagateway_api/src/swagger/initialise_spec.py index 5ad18d80..79350077 100644 --- a/datagateway_api/src/swagger/initialise_spec.py +++ b/datagateway_api/src/swagger/initialise_spec.py @@ -64,8 +64,8 @@ def initialise_spec(spec): {"type": "number"}, {"type": "integer"}, {"type": "boolean"}, - ] - } + ], + }, }, }, { @@ -105,8 +105,8 @@ def initialise_spec(spec): "oneOf": [ {"type": "number"}, {"type": "integer"}, - ] - } + ], + }, }, }, { @@ -117,8 +117,8 @@ def initialise_spec(spec): "oneOf": [ {"type": "number"}, {"type": "integer"}, - ] - } + ], + }, }, }, { From fe88196f2f7ed7291bbce6fa8108836345bbe65b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 8 Dec 2020 15:22:31 +0000 Subject: [PATCH 5/5] #184: Make response descriptions have consistent syntax - The commit also includes a rebuilt openapi.yaml --- .../src/resources/non_entities/sessions_endpoints.py | 4 ++-- datagateway_api/src/swagger/openapi.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datagateway_api/src/resources/non_entities/sessions_endpoints.py b/datagateway_api/src/resources/non_entities/sessions_endpoints.py index 13f0a2e6..348d719c 100644 --- a/datagateway_api/src/resources/non_entities/sessions_endpoints.py +++ b/datagateway_api/src/resources/non_entities/sessions_endpoints.py @@ -50,9 +50,9 @@ def post(self): description: Session ID example: xxxxxx-yyyyyyy-zzzzzz 400: - description: Bad request. User credentials not provided in request body + description: Bad request - User credentials not provided in request body 403: - description: Forbidden. User credentials were invalid + description: Forbidden - User credentials were invalid """ if not ( request.data and "username" in request.json and "password" in request.json diff --git a/datagateway_api/src/swagger/openapi.yaml b/datagateway_api/src/swagger/openapi.yaml index f52c1b93..d04bc6c8 100644 --- a/datagateway_api/src/swagger/openapi.yaml +++ b/datagateway_api/src/swagger/openapi.yaml @@ -87,9 +87,9 @@ components: schema: type: integer WHERE_FILTER: - description: Apply where filters to the query. The possible operators are like, - gte, lte, in and eq. Please modify the examples before executing a request - if you are having issues with the example values. + description: 'Apply where filters to the query. The possible operators are: + ne, like, lt, lte, gt, gte, in and eq. Please modify the examples before executing + a request if you are having issues with the example values.' examples: eq: value: @@ -10817,9 +10817,9 @@ paths: type: object description: Success - returns a session ID '400': - description: Bad request. User credentials not provided in request body + description: Bad request - User credentials not provided in request body '403': - description: Forbidden. User credentials were invalid + description: Forbidden - User credentials were invalid security: [] summary: Login tags: