getting back online

.github/workflows/codeclimate_coverage.yml

@@ -0,0 +1,56 @@
+name: CodeClimate Coverage
+
+# Runs the coverage report module for CodeClimate and pushes it to
+# CodeClimate website. It can then be visualised and badges can be
+# generated for README.md.
+
+on: [push, pull_request]
+
+jobs:
+  upload-code-climate-coverage:
+    name: Generate and Upload Coverage Report for CodeClimate
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Create and activate virtual environment
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        source ./add_root_to_path.sh
+
+    - name: Install dependencies
+      run: |
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install pytest-cov  # Install pytest-cov which uses a newer version of coverage.py
+        pip install -r requirements.txt  # Install other dependencies
+
+    - name: Download Code Climate Reporter
+      run: |
+        curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 -o ./cc-test-reporter
+        chmod +x ./cc-test-reporter
+
+    - name: Prepare for Coverage Report
+      run: |
+        ./cc-test-reporter before-build
+
+    - name: Run tests and generate coverage report
+      run: |
+        source venv/bin/activate
+        pytest --cov=./ --cov-report=xml --cov-report=term
+
+    - name: Upload Coverage Report to Code Climate
+      env:
+        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GIT_COMMIT_SHA: ${{ github.sha }}
+        GIT_BRANCH: ${{ github.ref_name }}
+      run: |
+        ./cc-test-reporter after-build --exit-code $?

.github/workflows/codecov_coverage.yml

@@ -0,0 +1,48 @@
+name: Pytest Coverage
+
+# Runs a coverage test and pushes the result to CodeCov.
+# Reports are then generated by CodeCov 
+# and badges etc can be integrated into README.md.
+
+on: [push, pull_request]
+
+jobs:
+  run-tests-and-coverage:
+    name: Generate and Upload Coverage Report For CodeCov
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Create and activate virtual environment
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        source ./add_root_to_path.sh
+
+    - name: Install dependencies for Pytest-Cov
+      run: |
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install pytest-cov
+        pip install -r requirements.txt
+
+    - name: Run tests with coverage
+      run: |
+        source venv/bin/activate
+        pytest --cov=./ --cov-report=xml:coverage.xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage.xml
+        flags: pytest
+        name: pytest-coverage-report
+        fail_ci_if_error: true

.github/workflows/test_build_push.yml

@@ -0,0 +1,91 @@
+
+name: Deploy to Cloud
+
+# If an update is pushed to main, tests are run, and if they pass,
+# the new .sh script that dockerises modules and pushes them to the 
+# cloud is run.
+# (Tests should be in this script as well as their own)
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  run-tests:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Create and activate virtual environment
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+
+    - name: Install dependencies
+      run: |
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Set up environment
+      run: |
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        source ./add_root_to_path.sh
+
+    - name: Run tests without coverage
+      run: |
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        pytest
+
+  build-and-push:
+    name: Build and Push Docker Images
+    runs-on: ubuntu-latest
+    needs: run-tests  # This ensures the 'run-tests' job must pass before this job runs
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Verify Environment Variables
+      run: |
+        echo "AWS_REGION: ${{ secrets.AWS_REGION }}"
+        echo "ECR_URL: ${{ secrets.ECR_URL }}"
+      env:
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        ECR_URL: ${{ secrets.ECR_URL }}
+
+    - name: Log in to AWS ECR
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        ECR_URL: ${{ secrets.ECR_URL }}
+      run: |
+        aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_URL
+
+    - name: Make build_and_push.sh executable and run it
+      run: |
+        cd infrastructure
+        chmod +x build_and_push.sh
+        ./build_and_push.sh
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        ECR_URL: ${{ secrets.ECR_URL }}
+
+    - name: Logout from AWS ECR
+      run: docker logout ${{ secrets.ECR_URL }}

.github/workflows/unittests.yml

@@ -0,0 +1,47 @@
+name: Unit Tests
+
+# Runs unit tests to check that changes to code haven't broken
+# anything.
+
+# If you modify this make sure changes are also relflected in 
+# `test_build_push.yml` as well, which conditionally pushes
+# changes to the cloud if its tests pass.
+
+
+on: [push, pull_request]
+
+jobs:
+  run-tests:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Create and activate virtual environment
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+
+    - name: Install dependencies
+      run: |
+        source venv/bin/activate
+        pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Set up environment
+      run: |
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        source ./add_root_to_path.sh
+
+    - name: Run tests without coverage
+      run: |
+        source venv/bin/activate
+        export PYTHONPATH="$PYTHONPATH:$(pwd)"
+        pytest

.gitignore

@@ -2,20 +2,27 @@
 *.venv
 *.env
 *.tfvars
+*explore.ipynb
 
 # Logs and Logging-Related
-**/tmp/
-logs/*
+tmp/logs/*
+tmp/data/*
 *.parquet
 *.feather
+*.log
 
-# Misc Module Data 
+
+# Python Module Data
 #(e.g __pychache__ directories)
-__pychache__/
+__pycache__/
+**/__pycache__/
 *.loc
+*.pytest_cache
+*.DS_Store
 
 # Terraform
 *.tfstate
 *.tfstate.backup
 */.terraform
-.terraform.lock.hcl
+.terraform.lock.hcl
+*.tfvars

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 J J Marden
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.