feat: meilisearch integration (#453)

* chore: Update catalog.yaml file for release data * chore: Delete openedx.yaml file * fix: refactor workflows * chore: Upgrade Python requirements (#450) * fix: path import is fixed After upgrading path-py from 9.1 to 12.5.0 they upgrade the import as well from p to capital P as import path from Path * fix: enable quality checks * chore: Upgrade Python requirements (#452) * chore: Upgrade Python requirements * fix: trivy action failure --------- Co-authored-by: Irtaza Akram <irtaza.akram@arbisoft.com> * feat: meilisearch backend for notes search (#444) * feat: introduce "make compile-requirement" target This is convenient to compile dependencies without upgrading them. * chore: simplify tox/make test commands This makes it possible to run the make commands directly without going through tox (though tox targets keep working, of course). * chore: more convenient unit test running Previously, it was not possible to run unit tests locally without manually creating mysql & elasticsearch containers. Here, we create a `make pytest` target that automatically starts the required containers. * chore: refactor views for better mysql/es separation Instead of checking a boolean flag in multiple different places, we use class inheritance. This makes it possible to later override the view and implement our own using a different search backend, such as Meilisearch. * feat: meilisearch backend for notes search This is a very simple and basic backend. It is based on Django signals, just like the Elasticsearch backend. But it is much simpler, in the sense that there are just two signals: one for saving documents and one for deletion. This backend is limited, in the sense that it does not support highlighting -- but that's probably not such a big deal. To start using this backend, define the following settings: ES_DISABLED = True MEILISEARCH_ENABLED = True MEILISEARCH_URL = "http://meilisearch:7700" MEILISEARCH_API_KEY = "s3cr3t" MEILISEARCH_INDEX = "tutor_student_notes" --------- Co-authored-by: salman2013 <salman.nawaz@arbisoft.com> Co-authored-by: Irtaza Akram <irtaza.akram@arbisoft.com> Co-authored-by: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Co-authored-by: Muhammad Arslan <arslan.abdulrauf@arbisoft.com> Co-authored-by: Irtaza Akram <51848298+irtazaakram@users.noreply.github.com>
openedx · Nov 14, 2024 · 5cdc0d4 · 5cdc0d4
1 parent 6df04d2
commit 5cdc0d4
Show file tree

Hide file tree

Showing 57 changed files with 1,980 additions and 1,266 deletions.
diff --git a/.ci/docker-compose-ci.yml b/.ci/docker-compose-ci.yml
diff --git a/.ci/docker.mk b/.ci/docker.mk
diff --git a/.ci/run_check_keywords.sh b/.ci/run_check_keywords.sh
diff --git a/.ci/run_pii_checker.sh b/.ci/run_pii_checker.sh
diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh
diff --git a/.dockerignore b/.dockerignore
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,47 +2,65 @@ name: Django CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches:
+      - "**"
 
 jobs:
-  build:
-
+  run_tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
-        python-version: ['py311', 'py312']
-        django-version: ['django42']
-        db-version: ['mysql80']
+        python-version: ["3.11", "3.12"]
+        toxenv: ["django42", "quality", "pii_check", "check_keywords"]
+
+    services:
+      mysql:
+        image: mysql:8.0
+        options: '--health-cmd="mysqladmin ping -h localhost" --health-interval=10s --health-timeout=5s --health-retries=3'
+        env:
+          MYSQL_ROOT_PASSWORD:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+          MYSQL_DATABASE: "edx_notes_api"
+        ports:
+          - 3306:3306
+
+      elasticsearch:
+        image: elasticsearch:7.13.4
+        options: '--health-cmd="curl -f http://localhost:9200 || exit 1" --health-interval=10s --health-timeout=5s --health-retries=3'
+        env:
+          discovery.type: single-node
+          bootstrap.memory_lock: "true"
+          ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+        ports:
+          - 9200:9200
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Start container
-      run: |
-        docker compose -f .ci/docker-compose-ci.yml up -d
-    - name: Install Dependencies
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "apt-get update && apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config"
-    - name: setup python 311
-      if: ${{ matrix.python-version == 'py311' }}
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.11 python3.11-dev python3.11-distutils -y"    
-    - name: setup python 312
-      if: ${{ matrix.python-version == 'py312' }}
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \
-        /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.12 python3.12-dev python3.12-distutils -y"
-    - name: Run Tests
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_tests.sh
-    - name: Run PII Check
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_pii_checker.sh
-    - name: Run Reserved Keywords Check
-      run: |
-        docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_check_keywords.sh
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev
+
+      - name: Install pip and Tox
+        run: pip install --upgrade pip tox
+
+      - name: Run Tox tests
+        env:
+          CONN_MAX_AGE: 60
+          DB_ENGINE: django.db.backends.mysql
+          DB_HOST: 127.0.0.1
+          DB_NAME: edx_notes_api
+          DB_PASSWORD:
+          DB_PORT: 3306
+          DB_USER: root
+          ENABLE_DJANGO_TOOLBAR: 1
+          ELASTICSEARCH_URL: http://127.0.0.1:9200
+        run: tox -e ${{ matrix.toxenv }}
diff --git a/.github/workflows/migrations-mysql8-check.yml b/.github/workflows/migrations-mysql8-check.yml
@@ -1,4 +1,4 @@
-name: Migrations check on mysql8
+name: Migrations check on MySQL 8
 
 on:
   workflow_dispatch:
@@ -9,74 +9,53 @@ on:
 
 jobs:
   check_migrations:
-    name: check migrations
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ '3.11', '3.12' ]
+        python-version: ["3.11", "3.12"]
 
     steps:
-    - name: Checkout repo
-      uses: actions/checkout@v4
-
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install system Packages
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libxmlsec1-dev
-    - name: Get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-    - name: Cache pip dependencies
-      id: cache-dependencies
-      uses: actions/cache@v4
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip-tools.txt') }}
-        restore-keys: ${{ runner.os }}-pip-
-
-    - name: Ubuntu and sql Versions
-      run: |
-        lsb_release -a
-        mysql -V
-
-    # pinning xmlsec to version 1.3.13 to avoid the CI error, migration checks are failing due to an issue in the latest release of python-xmlsec
-    # https://github.com/xmlsec/python-xmlsec/issues/314
-    - name: Install Python dependencies
-      run: |
-        pip install -r requirements/pip-tools.txt
-        pip install -r requirements/test.txt
-        pip install -r requirements/base.txt
-        pip uninstall -y mysqlclient
-        pip install --no-binary mysqlclient mysqlclient
-        pip uninstall -y xmlsec
-        pip install --no-binary xmlsec xmlsec==1.3.13
-
-    - name: Initiate Services
-      run: |
-        sudo /etc/init.d/mysql start
-    - name: Reset mysql password
-      run: |
-        cat <<EOF | mysql -h 127.0.0.1 -u root --password=root
-          UPDATE mysql.user SET authentication_string = null WHERE user = 'root';
-          FLUSH PRIVILEGES;
-        EOF
-    - name: Run Tests
-      env:
-        DB_ENGINE: django.db.backends.mysql
-        DB_NAME: edx_notes_api
-        DB_USER: root
-        DB_PASSWORD:
-        DB_HOST: localhost
-        DB_PORT: 3306
-      run: |
-        echo "CREATE DATABASE IF NOT EXISTS edx_notes_api;" | sudo mysql -u root
-        echo "Running the migrations."
-        python manage.py migrate --settings=notesserver.settings.test
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }} with cache
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: "**/pip-tools.txt"
+
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev
+
+      # pinning xmlsec to version 1.3.13 to avoid the CI error, migration checks are failing due to an issue in the latest release of python-xmlsec
+      # https://github.com/xmlsec/python-xmlsec/issues/314
+      - name: Install Python dependencies
+        run: |
+          pip install -r requirements/pip-tools.txt
+          pip install -r requirements/test.txt
+          pip install -r requirements/base.txt
+          pip uninstall -y mysqlclient
+          pip install --no-binary mysqlclient mysqlclient
+          pip uninstall -y xmlsec
+          pip install --no-binary xmlsec xmlsec==1.3.13
+
+      - name: Start MySQL service
+        run: sudo service mysql start
+
+      - name: Reset MySQL root password
+        run: |
+          mysql -h 127.0.0.1 -u root -proot -e "UPDATE mysql.user SET authentication_string = null WHERE user = 'root'; FLUSH PRIVILEGES;"
+
+      - name: Run migrations
+        env:
+          DB_ENGINE: django.db.backends.mysql
+          DB_NAME: edx_notes_api
+          DB_USER: root
+          DB_PASSWORD:
+          DB_HOST: localhost
+          DB_PORT: 3306
+        run: |
+          echo "CREATE DATABASE IF NOT EXISTS edx_notes_api;" | sudo mysql -u root
+          python manage.py migrate --settings=notesserver.settings.test
diff --git a/.github/workflows/push-docker-image.yml b/.github/workflows/push-docker-image.yml