Skip to content
CI

CI #468

Sign in to view logs

CI

CI #468

Workflow file for this run

.github/workflows/ci.yml at cbba5be
name: CI
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
jobs:
tests:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9"]
name: Python ${{ matrix.python-version }} Tests
steps:
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
# Install dependencies of python-ldap
- name: Install python-ldap dependencies
run: sudo apt-get install -y libsasl2-dev python${{ matrix.python-version }}-dev libldap2-dev libssl-dev
# Setup Python and environment dependencies (via cache)
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
# Install and start MongoDB
- name: Start MongoDB
uses: supercharge/mongodb-github-action@1.7.0
with:
mongodb-version: '5.0'
# Read the database name from the config file and store it in an environment variable
- name: Get database name from ci_config.yml
run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV
- name: Add test users to OperationsGateway database
run: mongoimport --db='${{ env.DATABASE_NAME }}' --collection='users' --file='util/users_for_mongoimport.json'
# Load Poetry virtual environment dependencies and install API dependencies
- name: Load Poetry cache
uses: actions/cache@v3
with:
path: ~/.cache/pypoetry/virtualenvs
key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
run: poetry install --without simulated-data
# Configure s4cmd
- name: Add keys to s4cmd config
run: sed -i -e "s/replace_with_access_key/$ECHO_S3_ACCESS_KEY/" -e "s/replace_with_secret_key/$ECHO_S3_SECRET_KEY/" .github/ci_s3cfg
env:
ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
- name: Move s3cfg to correct place
run: cp .github/ci_s3cfg /home/runner/.s3cfg
# Create a bucket for image storage
- name: Create bucket for current job
run: poetry run s4cmd --endpoint-url https://s3.echo.stfc.ac.uk mb s3://og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}
id: create_bucket
# Configuration for API
- name: Configure echo access key
run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_config.yml
env:
ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
- name: Configure echo secret key
run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_config.yml
env:
ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
- name: Configure bucket name for current run
run: yq -i '.echo.bucket_name = "og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}"' .github/ci_config.yml
- name: Move CI config.yml to correct place
run: cp .github/ci_config.yml operationsgateway_api/config.yml
- name: Setup logging configuration
run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
- name: Create log file
run: touch "$GITHUB_WORKSPACE/logs.log"
- name: Create SSH private key file for auth
run: 'echo "$SSH_KEY_PRIVATE" > /home/runner/work/operationsgateway-api/id_rsa'
shell: bash
env:
SSH_KEY_PRIVATE: ${{secrets.SSH_PRIVATE_KEY_FOR_AUTH_OPENSSH}}
- name: Set permissions on private key file
run: chmod 600 /home/runner/work/operationsgateway-api/id_rsa
- name: Create SSH public key file for auth
run: 'echo "$SSH_KEY_PUBLIC" > /home/runner/work/operationsgateway-api/id_rsa.pub'
shell: bash
env:
SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
# Setup steps for Echo ingestion script
- name: Configure echo access key
run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
env:
ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
- name: Configure echo secret key
run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
env:
ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
- name: Configure bucket name for current run
run: yq -i '.echo.storage_bucket = "og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}"' .github/ci_ingest_echo_config.yml
- name: Copy config for Echo Ingest script to correct place
run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
- name: Run Echo Ingest script
run: poetry run python util/realistic_data/ingest_echo_data.py
- name: tmate sessions
uses: mxschmitt/action-tmate@v3
- name: Load Poetry cache for Nox tests session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/tests*
key: ${{ runner.os }}-poetry-nox-tests-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
# Run Nox tests session, saves and uploads a coverage report to codecov
- name: Run Nox tests session
run: nox -p ${{ matrix.python-version }} -s tests -- --cov=operationsgateway_api --cov-report=xml
- name: Upload code coverage report
if: matrix.python-version == '3.8'
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
# s3cmd is installed only to remove the bucket as this functionality isn't present
# in s4cmd. There is an issue open regarding this functionality:
# https://github.com/bloomreach/s4cmd/issues/316
# s3cmd is easy to install and uses the same config file as s4cmd so is workaround
# until the feature is implemented
- name: Install s3cmd
run: sudo apt-get install -y s3cmd
if: steps.create_bucket.outcome == 'success' || failure()
# --recursive & --force used so non-empty buckets can be deleted
- name: Remove bucket for current job
run: s3cmd rb --recursive --force s3://og-actions-${{ github.sha }}-${{ github.run_id }}-${{ matrix.python-version }}
if: steps.create_bucket.outcome == 'success' || failure()
linting:
runs-on: ubuntu-latest
name: Code Linting
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox lint session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/lint*
key: ${{ runner.os }}-poetry-nox-lint-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox lint session
run: nox -s lint
formatting:
runs-on: ubuntu-latest
name: Code Formatting
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox black session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/black*
key: ${{ runner.os }}-poetry-nox-black-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox black session
run: nox -s black
safety:
runs-on: ubuntu-latest
name: Dependency Safety
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox safety session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/safety*
key: ${{ runner.os }}-poetry-nox-safety-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox safety session
run: nox -s safety