Merge pull request #1381 from watchdogpolska/develop

 v1.3.1 - Llm evaluation

.contrib/docker/Dockerfile.web

@@ -1,5 +1,5 @@
 # This is Dockerfile for development purposes only.
-ARG PYTHON_VERSION='3.10.6'
+ARG PYTHON_VERSION='3.10.12'
 FROM python:${PYTHON_VERSION}-slim
 RUN python --version
 RUN mkdir /code /code/production
@@ -9,22 +9,19 @@ WORKDIR /code
 ENV PYTHONUNBUFFERED 1
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
-   default-libmysqlclient-dev \
+   python3-dev default-libmysqlclient-dev build-essential \
+   default-mysql-client libssl-dev pkg-config \
    gcc \
    build-essential \
    git \
    curl \
    nano \
    htop \
-   gettext libgettextpo-dev wait-for-it \
-&& rm -rf /var/lib/apt/lists/*
+   gettext libgettextpo-dev wait-for-it
 RUN pip install --upgrade pip
 RUN pip --version
 COPY requirements/*.txt ./requirements/
 ARG DJANGO_VERSION='==3.2.20'
-# TODO: Move to /requirements/base.txt after fixing following bug:
-#       https://github.com/readthedocs/readthedocs-docker-images/issues/158
-RUN pip install mysqlclient==2.1.1
 RUN bash -c "if [[ "${DJANGO_VERSION}" == 'master' ]]; then \
 pip install --no-cache-dir -r requirements/dev.txt https://github.com/django/django/archive/master.tar.gz; else \
 pip install --no-cache-dir -r requirements/dev.txt \"django${DJANGO_VERSION}\"; fi"

.contrib/docker/Dockerfile.web.production

@@ -1,5 +1,5 @@
 # This is Dockerfile for development purposes only.
-ARG PYTHON_VERSION='3.10.6'
+ARG PYTHON_VERSION='3.10.12'
 FROM python:${PYTHON_VERSION}-slim
 RUN python --version
 RUN mkdir /code code/media code/staticfiles
@@ -9,22 +9,21 @@ WORKDIR /code
 ENV PYTHONUNBUFFERED 1
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
-   default-libmysqlclient-dev \
+   python3-dev default-libmysqlclient-dev build-essential \
+   default-mysql-client libssl-dev pkg-config \
    gcc \
    build-essential \
    git \
    curl \
    nano \
    htop \
-   gettext libgettextpo-dev wait-for-it \
-&& rm -rf /var/lib/apt/lists/*
+   gettext libgettextpo-dev wait-for-it
 RUN pip install --upgrade pip
 RUN pip --version
 COPY requirements/*.txt ./requirements/
 ARG DJANGO_VERSION='==3.2.20'
 # TODO: Move to /requirements/base.txt after fixing following bug:
 #       https://github.com/readthedocs/readthedocs-docker-images/issues/158
-RUN pip install mysqlclient==2.1.1
 RUN pip install --no-cache-dir -r requirements/production.txt
 COPY . /code/
 # Start container with bash shell to allow run and debug django app

.pre-commit-config.yaml

@@ -37,9 +37,9 @@ repos:
     hooks:
       - id: yamllint
         args: [-c, .yamllint.yml]
-        exclude: "feder/letters/logs/cassettes/.*"
+  #        exclude: (feder/letters/logs/cassettes/.*|docker-compose.yml)
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 23.10.1
     hooks:
       - id: black
         args:

.prettierignore

@@ -0,0 +1 @@
+**/*.html

.yamllint.yml

@@ -1,5 +1,7 @@
+---
 ignore: |
-  **/cassettes/*.yaml
+  **/docker-compose.yml
+  **/cassettes/*.*
 extends: default
 rules:
   line-length:

config/settings/common.py

@@ -41,6 +41,7 @@
     "allauth.socialaccount",  # registration
     "dal",
     "dal_select2",
+    "django_extensions",
     "tinymce",
     "ajax_datatable",
     "formtools",
@@ -78,6 +79,7 @@
     "feder.virus_scan",
     "feder.organisations",
     "feder.es_search.apps.EsSearchConfig",
+    "feder.llm_evaluation",
     # Your stuff: custom apps go here
 )
 
@@ -105,7 +107,9 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "reversion.middleware.RevisionMiddleware",
     # Add the account middleware:
-    "allauth.account.middleware.AccountMiddleware",
+    # needed for django-allauth==0.57.0
+    # but v0.57.0 does not work with production mariadb 10.1
+    # "allauth.account.middleware.AccountMiddleware",
 )
 
 # MIGRATIONS CONFIGURATION
@@ -168,7 +172,7 @@
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.
 # In a Windows environment this must be set to your system time zone.
-TIME_ZONE = "UTC"
+TIME_ZONE = "Europe/Warsaw"
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
 LANGUAGE_CODE = "pl"
@@ -461,7 +465,7 @@
     # "height": 500,
     "menubar": True,
     "lineheight": 1,
-    "plugins": "advlist,autolink,lists,link,image,charmap,print,preview,anchor,"
+    "plugins": "autolink,lists,link,image,charmap,print,preview,anchor,"
     "searchreplace,visualblocks,code,fullscreen,insertdatetime,media,table,paste,"
     "code,help,wordcount",
     "toolbar": "undo redo | formatselect | lineheight | fontsizeselect |"
@@ -507,9 +511,11 @@
 }
 
 OPENAI_API_KEY = env.str("OPENAI_API_KEY", "")
-OPENAI_LLM_MODEL = env.str("OPENAI_LLM_MODEL", "gpt-3.5-turbo")
-OPENAI_LLM_MODEL_MAX_TOKENS = env.int("OPENAI_LLM_MODEL_MAX_TOKENS", 4000)
-OPENAI_LLM_MODEL_LARGE = env.str("OPENAI_LLM_MODEL", "gpt-3.5-turbo-16k")
-OPENAI_LLM_MODEL_LARGE_MAX_TOKENS = env.int("OPENAI_LLM_MODEL_LARGE_MAX_TOKENS", 16000)
-OPENAI_LLM_TEMPERATURE = env.float("OPENAI_LLM_TEMPERATURE", 0.0)
-OPENAI_MAX_RETRIES = env.int("OPENAI_MAX_RETRIES", 5)
+OPENAI_API_VERSION = env.str("OPENAI_API_VERSION", "")
+OPENAI_API_TYPE = env.str("OPENAI_API_TYPE", "")
+OPENAI_API_BASE = env.str("OPENAI_API_BASE", "")
+OPENAI_API_ENGINE_35 = env.str("OPENAI_API_ENGINE_35", "")
+OPENAI_API_ENGINE_35_MAX_TOKENS = env.int("OPENAI_API_ENGINE_35_MAX_TOKENS", 4096)
+OPENAI_API_ENGINE_4 = env.str("OPENAI_API_ENGINE_4", "")
+OPENAI_API_ENGINE_4_MAX_TOKENS = env.int("OPENAI_API_ENGINE_4_MAX_TOKENS", 8000)
+OPENAI_API_TEMPERATURE = env.float("OPENAI_API_TEMPERATURE", 0.0)

docker-compose.yml

@@ -1,16 +1,21 @@
 ---
-version: '2.1'
+# yamllint disable-line-length
+version: "2.1"
 # This is application’s services configuration for development purposes only.
 
 services:
   db:
-    image: mariadb:10.11
-    command: '--character-set-server=utf8 --collation-server=utf8_polish_ci --max_allowed_packet=1024M' # yamllint disable-line
+    image: mariadb:10.1
+    command: >-
+      --character-set-server=utf8
+      --collation-server=utf8_polish_ci
+      --max_allowed_packet=1024M
+      --sql-mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
     ports:
       - "3306:3306"
     environment:
       MYSQL_ROOT_PASSWORD: "password"
-      MYSQL_DATABASE: feder3_copy
+      MYSQL_DATABASE: feder # feder3_copy
     volumes:
       - mariadb-data:/var/lib/mysql
 
@@ -28,17 +33,17 @@ services:
       context: .
       dockerfile: .contrib/docker/Dockerfile.web
       # args:
-        # PYTHON_VERSION: '${PYTHON_VERSION:-3.6.9}' # Match to production environment
-        # PYTHON_VERSION: '${PYTHON_VERSION:-3.10.6}' # Match to staging environment
-        # DJANGO_VERSION: '${DJANGO_VERSION:-==3.1.5}'
+      # PYTHON_VERSION: '${PYTHON_VERSION:-3.6.9}' # Match to production environment
+      # PYTHON_VERSION: '${PYTHON_VERSION:-3.10.6}' # Match to staging environment
+      # DJANGO_VERSION: '${DJANGO_VERSION:-==3.1.5}'
     volumes: &webVolumes
       - .:/code
     environment: &webEnvironment
       DATABASE_URL: mysql://root:password@db/feder3_copy
       ELASTICSEARCH_URL: http://elasticsearch:9200/
-      APACHE_TIKA_URL: 'http://tika:9998/'
+      APACHE_TIKA_URL: "http://tika:9998/"
       # APACHE_TIKA_URL: 'http://192.168.100.120:9998/'
-      MEDIA_ROOT_ENV: 'media_prod'
+      MEDIA_ROOT_ENV: "media_prod"
       DJANGO_EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
       DJANGO_EMAIL_HOST: "maildump"
       DJANGO_EMAIL_PORT: "1025"
@@ -59,31 +64,35 @@ services:
       FILE_TO_TEXT_URL: ${FILE_TO_TEXT_URL}
       FILE_TO_TEXT_TOKEN: ${FILE_TO_TEXT_TOKEN}
       OPENAI_API_KEY: ${OPENAI_API_KEY}
-      OPENAI_LLM_MODEL: ${OPENAI_LLM_MODEL}
-      OPENAI_LLM_MODEL_MAX_TOKENS: ${OPENAI_LLM_MODEL_MAX_TOKENS:-4000}
-      OPENAI_LLM_MODEL_LARGE: ${OPENAI_LLM_MODEL_LARGE}
-      OPENAI_LLM_MODEL_LARGE_MAX_TOKENS: ${OPENAI_LLM_MODEL_LARGE_MAX_TOKENS:-16000}
+      OPENAI_API_BASE: ${OPENAI_API_BASE}
+      OPENAI_API_ENGINE_35: "gpt-35-turbo-16k"
+      OPENAI_API_ENGINE_4: "gpt-4"
+      OPENAI_API_TYPE: "azure"
+      OPENAI_API_ENGINE_35_MAX_TOKENS: 16000
+      OPENAI_API_ENGINE_4_MAX_TOKENS: 8000
+      OPENAI_API_TEMPERATURE: 0.0
+      OPENAI_API_VERSION: "2023-07-01-preview"
     ports:
       - "8000:8000"
     # Following allows to execute `docker attach feder_web_1`
     # necessary when using ipdb in development:
     stdin_open: true
     tty: true
 
-# *
-# * for debugging purposes it is better to run worker in VScode launch on web container
-# *
-#  worker:
-#    command: python manage.py process_tasks
-#    build: *webBuild
-#    volumes: *webVolumes
-#    environment: *webEnvironment
+  # *
+  # * for debugging purposes it is better to run worker in VScode launch on web container
+  # *
+  #  worker:
+  #    command: python manage.py process_tasks
+  #    build: *webBuild
+  #    volumes: *webVolumes
+  #    environment: *webEnvironment
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
     environment:
       discovery.type: single-node
-      cluster.routing.allocation.disk.threshold_enabled: 'false'
+      cluster.routing.allocation.disk.threshold_enabled: "false"
     ports:
       - "9200:9200"
 
@@ -113,7 +122,7 @@ services:
   #     SENTRY_ENVIRONMENT: "feder-dev"
   #     COMPRESS_EML: "True"
   #   env_file:
-  #     - secrets_imap-to-webhook-url.env
+  #     - secrets_imap-to-webhook-url.env:optional
 
 volumes:
   mariadb-data:

feder/__init__.py

@@ -1,5 +1,5 @@
 # PEP 396: The __version__ attribute's value SHOULD be a string.
-__version__ = "1.2.6"
+__version__ = "1.3.1"
 
 
 # Compatibility to eg. django-rest-framework

feder/alerts/locale/pl/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: alerts 0.0.1\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-10-16 11:55+0000\n"
+"POT-Creation-Date: 2023-10-30 18:11+0100\n"
 "PO-Revision-Date: 2016-09-18 01:57+0200\n"
 "Last-Translator: Adam Dobrawy <naczelnik@jawnosc.tk>\n"
 "Language-Team: pl_PL <naczelnik@jawnosc.tk>\n"

feder/cases/locale/pl/LC_MESSAGES/django.po

@@ -7,16 +7,15 @@ msgid ""
 msgstr ""
 "Project-Id-Version: cases 0.1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-10-16 11:55+0000\n"
-"PO-Revision-Date: 2023-08-22 10:20+0000\n"
+"POT-Creation-Date: 2023-10-30 18:11+0100\n"
+"PO-Revision-Date: 2023-10-30 18:16+0124\n"
 "Last-Translator:   <piotr.iwanski@gmail.com>\n"
 "Language-Team: Adam Dobrawy <naczelnik@jawnosc.tk>\n"
 "Language: pl_PL\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 "
-"|| n%100>=20) ? 1 : 2);\n"
+"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n"
 "X-Generator: Poedit 1.8.7.1\n"
 "X-Translated-Using: django-rosetta 0.9.9\n"
 
@@ -93,7 +92,7 @@ msgid "Last request"
 msgstr "Ostatni wniosek"
 
 #: feder/cases/models.py:266 feder/cases/models.py:267
-#: feder/cases/models.py:343
+#: feder/cases/models.py:356
 msgid "Case"
 msgstr "Sprawa"
 
@@ -149,16 +148,12 @@ msgstr "Dodaj list"
 #, python-format
 msgid ""
 "\n"
-"    The case \"%(object)s\" is in quarantine. It is not available to users "
-"who do not have permission\n"
-"    'view_quarantined_case' to monitoring. Remove legal restrictions and "
-"edit the case to disclose it.\n"
+"    The case \"%(object)s\" is in quarantine. It is not available to users who do not have permission\n"
+"    'view_quarantined_case' to monitoring. Remove legal restrictions and edit the case to disclose it.\n"
 "    "
 msgstr ""
 "\n"
-"    Sprawa \"%(object)s\" jest poddana kwarantannie. Jest ona niedostępna "
-"dla użytkowników, którzy nie mają uprawnienia 'view_quarantined_case' w "
-"monitoringu. Usuń ograniczenia prawne i edytuj sprawę, aby ją ujawnić.\n"
+"    Sprawa \"%(object)s\" jest poddana kwarantannie. Jest ona niedostępna dla użytkowników, którzy nie mają uprawnienia 'view_quarantined_case' w monitoringu. Usuń ograniczenia prawne i edytuj sprawę, aby ją ujawnić.\n"
 "    "
 
 #: feder/cases/templates/cases/_desc.html:9
@@ -186,11 +181,15 @@ msgstr "Sprawy"
 msgid "Confirm delete"
 msgstr "Potwierdź usunięcie"
 
-#: feder/cases/templates/cases/case_detail.html:42
+#: feder/cases/templates/cases/case_detail.html:39
+msgid "Normalized answer"
+msgstr "Znormalizowana odpowiedź"
+
+#: feder/cases/templates/cases/case_detail.html:52
 msgid "Content"
 msgstr "Treść"
 
-#: feder/cases/templates/cases/case_detail.html:49
+#: feder/cases/templates/cases/case_detail.html:59
 msgid "No rows."
 msgstr "Brak wierszy."
 

feder/cases/models.py

@@ -326,6 +326,19 @@ def get_response_received(self):
             .exists()
         )
 
+    def get_normalized_answer(self):
+        normalized_response = (
+            apps.get_model("letters", "Letter")
+            .objects.filter(record__case=self, author_user_id__isnull=True)
+            .exclude_automatic()
+            .exclude(normalized_response="")
+            .exclude(normalized_response__isnull=True)
+        )
+        result = [
+            item.get_normalized_response_html_table() for item in normalized_response
+        ]
+        return result
+
     @property
     def letter_count(self):
         return self.record_set.exclude(letters_letters__is_spam=2).count()

feder/cases/templates/cases/case_detail.html

@@ -34,6 +34,16 @@
             <div class="gray">
                 {% include 'cases/_desc.html' with object=object %}
             </div>
+            <div class="gray">
+                <h4>
+                    {% trans 'Normalized answer' %}
+                </h4>
+                {% for response in object.get_normalized_answer %}
+                <p>
+                    {{ response }}
+                </p>
+                {% endfor %}
+            </div>
         </div>
 
         <div class="col-sm-7">