Skip to content

Commit

Permalink
Merge branch 'master' into filter-input-conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@VKTB
VKTB committed Jan 31, 2022
2 parents 3802cc9 + c4bd953 commit dbe684e
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 157 deletions.
270 changes: 145 additions & 125 deletions .github/workflows/ci-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,146 +11,153 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ['3.6', '3.7', '3.8', '3.9']
python-version: ["3.6", "3.7", "3.8", "3.9"]
name: Python ${{ matrix.python-version }} Build & Tests
steps:
- name: Add apt repo
run: sudo add-apt-repository universe

# Setup Java & Python
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 8
java-package: jdk
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: x64

# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@v2
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt

# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost

# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
- name: Checkout DataGateway API
uses: actions/checkout@v2

# Prep for using the API for tests
- name: Create log file
run: touch logs.log
- name: Configure log file location
run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \
'.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example
- name: Create config.json
run: cp datagateway_api/config.json.example datagateway_api/config.json
- name: Create search_api_mapping.json
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json

# Install Nox, Poetry and API's dependencies
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9
- name: Install dependencies
run: poetry install

- name: Add dummy data to icatdb
run: |
poetry run python -m util.icat_db_generator
# Run Nox tests session, saves and uploads a coverage report to codecov
- name: Run Nox tests session
run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml
- name: Upload code coverage report
if: matrix.python-version == '3.6'
uses: codecov/codecov-action@v1
- name: Add apt repo
run: sudo add-apt-repository universe

# Setup Java & Python
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 8
java-package: jdk
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: x64

# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@v2
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt

# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost

# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
- name: Checkout DataGateway API
uses: actions/checkout@v2

# Prep for using the API for tests
- name: Create log file
run: touch logs.log
- name: Configure log file location
run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \
'.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example
- name: Create config.json
run: cp datagateway_api/config.json.example datagateway_api/config.json
- name: Create search_api_mapping.json
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json

# Install Nox, Poetry and API's dependencies
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

# Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
- name: Uninstall setuptools
run: poetry run pip uninstall -y setuptools
- name: Install older setuptools
run: poetry run pip install 'setuptools<58.0.0'

- name: Install dependencies
run: poetry install

- name: Add dummy data to icatdb
run: |
poetry run python -m util.icat_db_generator
# Run Nox tests session, saves and uploads a coverage report to codecov
- name: Run Nox tests session
run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml
- name: Upload code coverage report
if: matrix.python-version == '3.6'
uses: codecov/codecov-action@v1

linting:
runs-on: ubuntu-20.04
name: Linting
steps:
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.9.7'
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox lint session
run: nox -s lint
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox lint session
run: nox -s lint

formatting:
runs-on: ubuntu-20.04
name: Code Formatting
steps:
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.9.7'
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox black session
run: nox -s black
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox black session
run: nox -s black

safety:
runs-on: ubuntu-20.04
name: Dependency Safety
steps:
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.9.7'
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox safety session
run: nox -s safety
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@v2

- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9

- name: Run Nox safety session
run: nox -s safety

generator-script-testing:
runs-on: ubuntu-20.04
Expand All @@ -169,7 +176,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: '3.9.7'
python-version: "3.9.7"
architecture: x64

# ICAT Ansible clone and install dependencies
Expand Down Expand Up @@ -211,6 +218,13 @@ jobs:
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
- name: Install Poetry
run: pip install poetry==1.1.9

# Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
- name: Uninstall setuptools
run: poetry run pip uninstall -y setuptools
- name: Install older setuptools
run: poetry run pip install 'setuptools<58.0.0'

- name: Install dependencies
run: poetry install

Expand Down Expand Up @@ -238,7 +252,6 @@ jobs:
- name: Diff SQL dumps
run: diff -s ~/generator_script_dump_1.sql ~/generator_script_dump_2.sql


# Drop and re-create icatdb to remove generated data
- name: Drop icatdb
run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;'
Expand All @@ -257,6 +270,13 @@ jobs:
run: cp datagateway_api/config.json.example datagateway_api/config.json
- name: Create search_api_mapping.json
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json

# Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
- name: Uninstall setuptools
run: poetry run pip uninstall -y setuptools
- name: Install older setuptools
run: poetry run pip install 'setuptools<58.0.0'

- name: Install dependencies
run: poetry install

Expand Down
11 changes: 11 additions & 0 deletions noxfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,16 @@ def safety(session):
@nox.session(python=["3.6", "3.7", "3.8", "3.9"], reuse_venv=True)
def tests(session):
args = session.posargs
# Installing setuptools that will work with `2to3` which is used when building
# `python-icat` < 1.0. 58.0.0 removes support of this tool during builds:
# https://setuptools.pypa.io/en/latest/history.html#v58-0-0
# Ideally this would be done within `pyproject.toml` but specifying `setuptools` as
# a dependency requires Poetry 1.2:
# https://github.com/python-poetry/poetry/issues/4511#issuecomment-922420457
# Currently, only a pre-release exists for Poetry 1.2. Testing on the pre-release
# version didn't fix the `2to3` issue when building Python ICAT, perhaps because
# Python ICAT isn't built on the downgraded version for some reason?
session.run("poetry", "run", "pip", "uninstall", "-y", "setuptools")
session.run("poetry", "run", "pip", "install", "setuptools<58.0.0")
session.run("poetry", "install", external=True)
session.run("pytest", *args)
Loading

0 comments on commit dbe684e

Please sign in to comment.