Integration - Python Backend

.bandit.yml

@@ -8,4 +8,8 @@
 tests:
 
 skips:
-  - B101  # skip "assert used" check since assertions are required in pytests
+  # Skip "assert used" check since assertions are required in pytests
+  - B101
+  # Skip "pseudo-random generators not suitable for security"
+  # b/c it's used for sample data
+  - B311

.flake8

@@ -22,4 +22,4 @@ select = C,D,E,F,W,B,B950
 # operators.  It no longer agrees with PEP8.  See, for example, here:
 # https://github.com/ambv/black/issues/21. Guido agrees here:
 # https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b.
-ignore = E501,W503
+ignore = E203,E402,E501,W503

.github/workflows/backend.yml

@@ -5,14 +5,14 @@ on:
   push:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - backend/**
       - .github/workflows/backend.yml
   pull_request:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - backend/**
       - .github/workflows/backend.yml
@@ -22,28 +22,12 @@ defaults:
     working-directory: ./backend
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-      - name: Restore npm cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: npm ci
-      - name: Lint
-        run: npm run lint
   test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
           node-version: '18'
@@ -55,71 +39,31 @@ jobs:
           restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
       - name: Run site locally
         run: |
           cp dev.env.example .env
           docker compose up -d db backend es
-          npm install -g wait-port
-          wait-port -t 3000 5432 9200 9300
+          python -m pip install wait-for-it
+          wait-for-it --service localhost:3000 --timeout=10
+          wait-for-it --service localhost:5432 --timeout=10
+          wait-for-it --service localhost:9200 --timeout=10
+          wait-for-it --service localhost:9300 --timeout=10
         working-directory: ./
+
       - name: Sync database
-        run: npm run syncdb
-        working-directory: ./backend
-      - name: Test
-        run: npm run test -- --collectCoverage --silent
-      - name: Package
-        run: npx sls package
-        env:
-          SLS_DEBUG: '*'
-  test_worker:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-      - name: Restore npm cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: npm ci
-      - name: Build
-        run: npx webpack --config webpack.worker.config.js
-      - name: Run db locally
-        run: |
-          cp dev.env.example .env
-          docker compose up -d db
-          npm install -g wait-port
-          wait-port -t 3000 5432
-        working-directory: ./
-      - name: Test
-        run: node dist/worker.bundle.js
-        env:
-          CROSSFEED_COMMAND_OPTIONS: '{"scanName": "test"}'
-          DB_USERNAME: crossfeed
-          DB_PASSWORD: password
-  test_python:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5.0.0
-        with:
-          python-version: '3.10'
-      - name: Copy .env file
-        run: cp ../dev.env.example .env
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: pip-
-      - run: pip install -r worker/requirements.txt
-      - run: pytest
+        run: make syncdb
+
+      - name: Lint with pylint
+        run: make pylint
+
+      - name: Run pytest
+        run: make pytest
+
   build_worker:
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -139,8 +83,9 @@ jobs:
       - name: Build worker container
         run: npm run build-worker
         working-directory: ./backend
+
   deploy_staging:
-    needs: [build_worker, lint, test, test_worker, test_python]
+    needs: [build_worker, test]
     runs-on: ubuntu-latest
     environment: staging
     concurrency: '1'
@@ -150,6 +95,10 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: '18'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
       - name: Restore npm cache
         uses: actions/cache@v3
         with:
@@ -160,14 +109,14 @@ jobs:
         run: npm ci
 
       - name: Ensure domain exists
-        run: npx sls create_domain --stage=staging
+        run: npx sls create_domain --stage=staging-cd
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
       - name: Deploy backend
-        run: npx sls deploy --stage=staging
+        run: npx sls deploy --stage=staging-cd
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -182,7 +131,7 @@ jobs:
 
       - name: Run syncdb
         run: |
-          aws lambda invoke --function-name crossfeed-staging-syncdb \
+          aws lambda invoke --function-name crossfeed-staging-cd-syncdb \
           --region us-east-1 /dev/stdout
         working-directory: backend
         env:
@@ -191,24 +140,28 @@ jobs:
 
       - name: Run syncmdl
         run: |
-          aws lambda invoke --function-name crossfeed-staging-syncmdl \
+          aws lambda invoke --function-name crossfeed-staging-cd-syncmdl \
           --region us-east-1 /dev/stdout
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-  deploy_prod:
-    needs: [build_worker, lint, test, test_python]
+  deploy_integration:
+    needs: [build_worker, test]
     runs-on: ubuntu-latest
-    environment: production
+    environment: integration
     concurrency: '1'
-    if: github.event_name == 'push' && github.ref == 'refs/heads/production'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/integration'
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
           node-version: '18'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
       - name: Restore npm cache
         uses: actions/cache@v3
         with:
@@ -219,39 +172,39 @@ jobs:
         run: npm ci
 
       - name: Ensure domain exists
-        run: npx sls create_domain --stage=prod
+        run: npx sls create_domain --stage=integration
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
       - name: Deploy backend
-        run: npx sls deploy --stage=prod
+        run: npx sls deploy --stage=integration --force
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
       - name: Deploy worker
-        run: npm run deploy-worker-prod
+        run: npm run deploy-worker-integration
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Run syncdb
         run: |
-          aws lambda invoke --function-name crossfeed-prod-syncdb --region us-east-1 \
-          /dev/stdout
+          aws lambda invoke --function-name crossfeed-integration-syncdb \
+          --region us-east-1 /dev/stdout
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Run syncmdl
         run: |
-          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
-          /dev/stdout
+          aws lambda invoke --function-name crossfeed-integration-syncmdl \
+          --region us-east-1 /dev/stdout
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/codeql.yml

@@ -3,7 +3,7 @@ name: CodeQL
 
 on:
   push:
-    branches: [develop, production]
+    branches: [develop, integration]
   pull_request:
     branches: [develop]
   schedule: