From f5f1ca2bab8c3c3c1760161160adbc5c5744c412 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Thu, 24 Oct 2024 09:26:01 +0200 Subject: [PATCH 01/56] feat: Use the official python:3.9 image --- Dockerfile | 39 +++++++++++++++++++-------------------- requirements.txt | 2 ++ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 57e87186ce..543bced3f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,20 @@ -FROM ubuntu:20.04 +FROM python:3.9 ENV DEBIAN_FRONTEND noninteractive # build-essential RUN apt-get -qq -y update \ - && apt-get -qq --no-install-recommends -y install locales \ - ca-certificates postgresql-client libpq-dev curl jq \ - python3-pip python3-icu python3-psycopg2 \ - python3-lxml python3-crypto git \ - && apt-get -qq -y autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + && apt-get -qq --no-install-recommends -y install locales \ + ca-certificates postgresql-client libpq-dev curl jq git \ + libxml2-dev libxslt1-dev python3-dev \ + && apt-get -qq -y autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 ENV LANG='en_US.UTF-8' RUN groupadd -g 1000 -r app \ - && useradd -m -u 1000 -s /bin/false -g app app + && useradd -m -u 1000 -s /bin/false -g app app # Install Python dependencies RUN pip3 install --no-cache-dir -q -U pip setuptools six @@ -32,19 +31,19 @@ RUN pip install --no-cache-dir -q -e /aleph ENV ALEPH_WORD_FREQUENCY_URI=https://public.data.occrp.org/develop/models/word-frequencies/word_frequencies-v0.4.1.zip ENV ALEPH_FTM_COMPARE_MODEL_URI=https://public.data.occrp.org/develop/models/xref/glm_bernoulli_2e_wf-v0.4.1.pkl RUN mkdir -p /opt/ftm-compare/word-frequencies/ && \ - curl -L -o "/opt/ftm-compare/word-frequencies/word-frequencies.zip" "$ALEPH_WORD_FREQUENCY_URI" && \ - python3 -m zipfile --extract /opt/ftm-compare/word-frequencies/word-frequencies.zip /opt/ftm-compare/word-frequencies/ && \ - curl -L -o "/opt/ftm-compare/model.pkl" "$ALEPH_FTM_COMPARE_MODEL_URI" + curl -L -o "/opt/ftm-compare/word-frequencies/word-frequencies.zip" "$ALEPH_WORD_FREQUENCY_URI" && \ + python3 -m zipfile --extract /opt/ftm-compare/word-frequencies/word-frequencies.zip /opt/ftm-compare/word-frequencies/ && \ + curl -L -o "/opt/ftm-compare/model.pkl" "$ALEPH_FTM_COMPARE_MODEL_URI" # Configure some docker defaults: ENV ALEPH_ELASTICSEARCH_URI=http://elasticsearch:9200/ \ - ALEPH_DATABASE_URI=postgresql://aleph:aleph@postgres/aleph \ - FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \ - REDIS_URL=redis://redis:6379/0 \ - ARCHIVE_TYPE=file \ - ARCHIVE_PATH=/data \ - FTM_COMPARE_FREQUENCIES_DIR=/opt/ftm-compare/word-frequencies/ \ - FTM_COMPARE_MODEL=/opt/ftm-compare/model.pkl + ALEPH_DATABASE_URI=postgresql://aleph:aleph@postgres/aleph \ + FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \ + REDIS_URL=redis://redis:6379/0 \ + ARCHIVE_TYPE=file \ + ARCHIVE_PATH=/data \ + FTM_COMPARE_FREQUENCIES_DIR=/opt/ftm-compare/word-frequencies/ \ + FTM_COMPARE_MODEL=/opt/ftm-compare/model.pkl RUN mkdir /run/prometheus diff --git a/requirements.txt b/requirements.txt index 24bed71223..54eb8de6b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,8 @@ zipstream-new==1.1.8 pika==1.3.2 sentry-sdk[flask]==2.10.0 prometheus-client==0.17.1 +lxml==5.3.0 +lxml_html_clean==0.3.1 # Testing dependencies From 7b89c2c53f72b3390c0ebcb31fa6ba8705fb849e Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Thu, 24 Oct 2024 09:26:58 +0200 Subject: [PATCH 02/56] feat: run tests in dev mode --- contrib/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/test.sh b/contrib/test.sh index 0b0f00a2ae..5a5e0ce9ab 100755 --- a/contrib/test.sh +++ b/contrib/test.sh @@ -3,4 +3,4 @@ psql -c "DROP DATABASE IF EXISTS aleph_test;" $ALEPH_DATABASE_URI psql -c "CREATE DATABASE aleph_test;" $ALEPH_DATABASE_URI -pytest aleph/ --cov=aleph --cov-report html --cov-report term $@ +PYTHONDEVMODE=1 PYTHONTRACEMALLOC=1 pytest aleph/ --cov=aleph --cov-report html --cov-report term $@ From 198e2f0467117d0ece9432513286567e6e36c2fb Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Thu, 24 Oct 2024 09:27:04 +0200 Subject: [PATCH 03/56] chore: updated deprecated version query --- aleph/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aleph/__init__.py b/aleph/__init__.py index cd4d560d5e..b41c6668f4 100644 --- a/aleph/__init__.py +++ b/aleph/__init__.py @@ -1,9 +1,9 @@ import logging import warnings from sqlalchemy.exc import SAWarning -from pkg_resources import get_distribution +from importlib.metadata import version -__version__ = get_distribution("aleph").version +__version__ = version("aleph") # shut up useless SA warning: warnings.filterwarnings( From 234f7d31a9973d553b812c6604a6ae123341d7a9 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:05:37 +0100 Subject: [PATCH 04/56] Increase exporter readiness/liveness probe timeout (#3998) --- helm/charts/aleph/templates/exporter.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/charts/aleph/templates/exporter.yaml b/helm/charts/aleph/templates/exporter.yaml index 84615909da..860d380b13 100644 --- a/helm/charts/aleph/templates/exporter.yaml +++ b/helm/charts/aleph/templates/exporter.yaml @@ -69,12 +69,14 @@ spec: path: /metrics?name[]=None port: 9100 initialDelaySeconds: 5 + timeoutSeconds: 3 livenessProbe: httpGet: # Unselect all metrics as generating them can take multiple seconds path: /metrics?name[]=None port: 9100 initialDelaySeconds: 5 + timeoutSeconds: 3 volumes: - name: tmp-volume emptyDir: {} From 370ffd3b287b73530cf552a432d2a8b9a8e5d9c1 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Wed, 6 Nov 2024 16:34:04 +0300 Subject: [PATCH 05/56] Show the Aleph version when using the -v or --version options Signed-off-by: Mark Ng'ang'a --- aleph/manage.py | 26 +++++++++++++++++++++++++- aleph/tests/test_manage.py | 6 ++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/aleph/manage.py b/aleph/manage.py index 155c1d1e5d..036663573b 100644 --- a/aleph/manage.py +++ b/aleph/manage.py @@ -1,4 +1,6 @@ # coding: utf-8 +import importlib.metadata +import platform import sys import json import click @@ -8,6 +10,7 @@ from itertools import count from normality import slugify from tabulate import tabulate +from typing import Any from flask.cli import FlaskGroup from followthemoney.cli.util import write_object from servicelayer.taskqueue import flush_queues @@ -82,7 +85,28 @@ def ensure_collection(foreign_id, label): return collection -@click.group(cls=FlaskGroup, create_app=create_app) +def get_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> None: + if not value or ctx.resilient_parsing: + return + + click.echo( + f"Aleph {importlib.metadata.version('aleph')}\n" + f"Python {platform.python_version()}\n" + f"Flask {importlib.metadata.version('flask')}\n" + f"Werkzeug {importlib.metadata.version('werkzeug')}\n", + ) + ctx.exit() + + +@click.group(cls=FlaskGroup, create_app=create_app, add_version_option=False) +@click.option( + "-v", + "--version", + help="Show the Aleph version.", + callback=get_aleph_version, + expose_value=False, + is_flag=True, +) def cli(): """Server-side command line for aleph.""" diff --git a/aleph/tests/test_manage.py b/aleph/tests/test_manage.py index f04a519c5f..3636a21631 100644 --- a/aleph/tests/test_manage.py +++ b/aleph/tests/test_manage.py @@ -308,3 +308,9 @@ def test_userdel_nonexistent_group(self): ) assert result.exit_code != 0 assert isinstance(result.exception, SystemExit) + + def test_version_option_shows_aleph_version(self): + for option in ["-v", "--version"]: + result = self.runner.invoke(manage.cli, option) + assert result.exit_code == 0 + assert "Aleph" in result.output From 099554d9e222f8fb388f23310a6d3c2d4d79c602 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Thu, 14 Nov 2024 12:17:27 +0300 Subject: [PATCH 06/56] Change callback function name --- aleph/manage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aleph/manage.py b/aleph/manage.py index 036663573b..d976edd4b6 100644 --- a/aleph/manage.py +++ b/aleph/manage.py @@ -85,7 +85,7 @@ def ensure_collection(foreign_id, label): return collection -def get_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> None: +def print_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> None: if not value or ctx.resilient_parsing: return @@ -93,7 +93,7 @@ def get_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> Non f"Aleph {importlib.metadata.version('aleph')}\n" f"Python {platform.python_version()}\n" f"Flask {importlib.metadata.version('flask')}\n" - f"Werkzeug {importlib.metadata.version('werkzeug')}\n", + f"Werkzeug {importlib.metadata.version('werkzeug')}", ) ctx.exit() @@ -103,7 +103,7 @@ def get_aleph_version(ctx: click.Context, _: click.Parameter, value: Any) -> Non "-v", "--version", help="Show the Aleph version.", - callback=get_aleph_version, + callback=print_aleph_version, expose_value=False, is_flag=True, ) From 8c78e10fa9b981486a776740874fe63bf2a2da87 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Wed, 20 Nov 2024 19:01:11 +0100 Subject: [PATCH 07/56] Clarify which mentions are displayed in the UI (#4003) --- .../developers/explanation/entity-extraction/index.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/src/pages/developers/explanation/entity-extraction/index.mdx b/docs/src/pages/developers/explanation/entity-extraction/index.mdx index e7eaf3c31a..89151bbf73 100644 --- a/docs/src/pages/developers/explanation/entity-extraction/index.mdx +++ b/docs/src/pages/developers/explanation/entity-extraction/index.mdx @@ -91,3 +91,9 @@ Aleph also stores each extracted person or company as a [Mention](https://follow ### Extracting entities in (semi-)structured data The default spaCy models Aleph uses are trained on generic web content. They tend to show lower recall for (semi-)structured data such as listings from database websites. If you’re handling (semi-)structured files, we recommend that you manually parse the relevant data from these documents. To learn about this approach read ["How to create mixed document/entity graphs"](/developers/how-to/data/mixed-graphs). + +### Viewing extracted data in the UI + +Aleph currently doesn’t display all extracted data in the UI. The "Mentions" tab for a file will only list names of people and companies, email addresses, phone numbers, IBANs, and addresses if there is at least one other occurrence of the mention in another dataset or investigation the current user has access to. + +For example, if you upload a PDF document to Aleph that mentions the company name "ACME, Inc.", it will only be displayed in the "Mentions" tab if searching for "ACME, Inc." in Aleph would return at least one result. From 716a67e7bb0af2994c9ae005fb2c6949a4459ba3 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Tue, 26 Nov 2024 10:11:29 +0100 Subject: [PATCH 08/56] fix: docker-compose image tax syntax broken after merge --- contrib/aleph-traefik-minio-keycloak/docker-compose.yml | 8 ++++---- contrib/keycloak/docker-compose.dev-keycloak.yml | 8 ++++---- docker-compose.yml | 8 ++++---- helm/charts/aleph/README.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml index 6efde9516b..557f6427c0 100644 --- a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml +++ b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml @@ -54,7 +54,7 @@ services: - "traefik.enable=false" worker: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph worker restart: on-failure links: @@ -79,7 +79,7 @@ services: - "traefik.enable=false" shell: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: /bin/bash depends_on: - postgres @@ -99,7 +99,7 @@ services: - "traefik.enable=false" api: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: gunicorn -w 6 -b 0.0.0.0:8000 --log-level debug --log-file - aleph.wsgi:app expose: - 8000 @@ -121,7 +121,7 @@ services: - "traefik.enable=false" ui: - image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.2} depends_on: - api - traefik diff --git a/contrib/keycloak/docker-compose.dev-keycloak.yml b/contrib/keycloak/docker-compose.dev-keycloak.yml index 417c0d48d8..7098e7fb2f 100644 --- a/contrib/keycloak/docker-compose.dev-keycloak.yml +++ b/contrib/keycloak/docker-compose.dev-keycloak.yml @@ -16,7 +16,7 @@ services: elasticsearch: build: context: services/elasticsearch - image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.2} hostname: elasticsearch environment: - discovery.type=single-node @@ -55,7 +55,7 @@ services: app: build: context: . - image: alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph:${ALEPH_TAG:-4.0.2} hostname: aleph command: /bin/bash links: @@ -83,7 +83,7 @@ services: api: build: context: . - image: alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph run -h 0.0.0.0 -p 5000 --with-threads --reload --debugger ports: - "127.0.0.1:5000:5000" @@ -117,7 +117,7 @@ services: ui: build: context: ui - image: alephdata/aleph-ui:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.2} links: - api command: npm run start diff --git a/docker-compose.yml b/docker-compose.yml index 2c601d2982..4b20236e2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,7 +46,7 @@ services: - aleph.env worker: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph worker restart: on-failure depends_on: @@ -62,7 +62,7 @@ services: - aleph.env shell: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: /bin/bash depends_on: - postgres @@ -80,7 +80,7 @@ services: - aleph.env api: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} expose: - 8000 depends_on: @@ -97,7 +97,7 @@ services: - aleph.env ui: - image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.2} depends_on: - api ports: diff --git a/helm/charts/aleph/README.md b/helm/charts/aleph/README.md index 81d9f92072..618482682b 100644 --- a/helm/charts/aleph/README.md +++ b/helm/charts/aleph/README.md @@ -11,7 +11,7 @@ Helm chart for Aleph | global.amazon | bool | `true` | Are we using AWS services like s3? | | global.google | bool | `false` | Are we using GCE services like storage, vision api? | | global.image.repository | string | `"alephdata/aleph"` | Aleph docker image repo | -| global.image.tag | string | `"global.image.tag | string | `"4.0.2"` | Aleph docker image tag | +| global.image.tag | string | `"4.0.2"` | Aleph docker image tag | | global.image.tag | string | `"Always"` | | | global.namingPrefix | string | `"aleph"` | Prefix for the names of k8s resources | From fa1314392361a45a87c6ad7375acdf8903e1fe14 Mon Sep 17 00:00:00 2001 From: Martin Tournoij Date: Tue, 26 Nov 2024 10:21:12 +0000 Subject: [PATCH 09/56] Send correct sort order when listing entitysets (#4014) It would send a sort order of "created_at ASC", but in the backend it does: q.order_by(EntitySet.updated_at.desc()) So that defaultSortBy() doesn't really do anything. Could perhaps remove that backend check; I just left it in for now. This improves compatibility with Alfred as it doesn't have this hard-coded in the backend and just uses the URL. --- ui/src/screens/EntitySetIndexScreen/EntitySetIndexScreen.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/screens/EntitySetIndexScreen/EntitySetIndexScreen.jsx b/ui/src/screens/EntitySetIndexScreen/EntitySetIndexScreen.jsx index 1f5fcab0bf..4a5b28e2de 100644 --- a/ui/src/screens/EntitySetIndexScreen/EntitySetIndexScreen.jsx +++ b/ui/src/screens/EntitySetIndexScreen/EntitySetIndexScreen.jsx @@ -117,7 +117,7 @@ const mapStateToProps = (state, ownProps) => { location, context, 'entitySets' - ).defaultSortBy('created_at', 'asc'); + ).defaultSortBy('updated_at', 'desc'); return { query, result: selectEntitySetsResult(state, query), From 42514973db21555f5dd73433acc72fc1da14e2a8 Mon Sep 17 00:00:00 2001 From: Martin Tournoij Date: Tue, 26 Nov 2024 10:23:11 +0000 Subject: [PATCH 10/56] More relaxed check when deleting objects (#4013) When deleting a dataset or investigation you have to type in the name of it to confirm. I typically just copy the name, but it's easy to copy an extra space. Don't be too anal about all of this. --- ui/src/components/common/DeleteDialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/common/DeleteDialog.tsx b/ui/src/components/common/DeleteDialog.tsx index a0cd68910d..f7aef324d1 100644 --- a/ui/src/components/common/DeleteDialog.tsx +++ b/ui/src/components/common/DeleteDialog.tsx @@ -30,7 +30,7 @@ const DeleteDialog: FC = ({ }) => { const [isLoading, setIsLoading] = useState(false); const [confirmationValue, setConfirmationValue] = useState(''); - const enabled = confirmationValue === expectedConfirmationValue; + const enabled = confirmationValue.toLowerCase().trim() === expectedConfirmationValue.toLowerCase().trim(); return ( From 22406222852ef514c3015700055e60d4af0dc13e Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Tue, 26 Nov 2024 10:11:29 +0100 Subject: [PATCH 11/56] fix: docker-compose image tax syntax broken after merge --- contrib/aleph-traefik-minio-keycloak/docker-compose.yml | 8 ++++---- contrib/keycloak/docker-compose.dev-keycloak.yml | 8 ++++---- docker-compose.yml | 8 ++++---- helm/charts/aleph/README.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml index 6efde9516b..557f6427c0 100644 --- a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml +++ b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml @@ -54,7 +54,7 @@ services: - "traefik.enable=false" worker: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph worker restart: on-failure links: @@ -79,7 +79,7 @@ services: - "traefik.enable=false" shell: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: /bin/bash depends_on: - postgres @@ -99,7 +99,7 @@ services: - "traefik.enable=false" api: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: gunicorn -w 6 -b 0.0.0.0:8000 --log-level debug --log-file - aleph.wsgi:app expose: - 8000 @@ -121,7 +121,7 @@ services: - "traefik.enable=false" ui: - image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.2} depends_on: - api - traefik diff --git a/contrib/keycloak/docker-compose.dev-keycloak.yml b/contrib/keycloak/docker-compose.dev-keycloak.yml index 417c0d48d8..7098e7fb2f 100644 --- a/contrib/keycloak/docker-compose.dev-keycloak.yml +++ b/contrib/keycloak/docker-compose.dev-keycloak.yml @@ -16,7 +16,7 @@ services: elasticsearch: build: context: services/elasticsearch - image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.2} hostname: elasticsearch environment: - discovery.type=single-node @@ -55,7 +55,7 @@ services: app: build: context: . - image: alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph:${ALEPH_TAG:-4.0.2} hostname: aleph command: /bin/bash links: @@ -83,7 +83,7 @@ services: api: build: context: . - image: alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph run -h 0.0.0.0 -p 5000 --with-threads --reload --debugger ports: - "127.0.0.1:5000:5000" @@ -117,7 +117,7 @@ services: ui: build: context: ui - image: alephdata/aleph-ui:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.2} links: - api command: npm run start diff --git a/docker-compose.yml b/docker-compose.yml index 2c601d2982..4b20236e2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,7 +46,7 @@ services: - aleph.env worker: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: aleph worker restart: on-failure depends_on: @@ -62,7 +62,7 @@ services: - aleph.env shell: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} command: /bin/bash depends_on: - postgres @@ -80,7 +80,7 @@ services: - aleph.env api: - image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.2} expose: - 8000 depends_on: @@ -97,7 +97,7 @@ services: - aleph.env ui: - image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-ALEPH_TAG:-4.0.2} + image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.2} depends_on: - api ports: diff --git a/helm/charts/aleph/README.md b/helm/charts/aleph/README.md index 81d9f92072..618482682b 100644 --- a/helm/charts/aleph/README.md +++ b/helm/charts/aleph/README.md @@ -11,7 +11,7 @@ Helm chart for Aleph | global.amazon | bool | `true` | Are we using AWS services like s3? | | global.google | bool | `false` | Are we using GCE services like storage, vision api? | | global.image.repository | string | `"alephdata/aleph"` | Aleph docker image repo | -| global.image.tag | string | `"global.image.tag | string | `"4.0.2"` | Aleph docker image tag | +| global.image.tag | string | `"4.0.2"` | Aleph docker image tag | | global.image.tag | string | `"Always"` | | | global.namingPrefix | string | `"aleph"` | Prefix for the names of k8s resources | From 9563a6cf37533191aa0ec139fce73839ee4a9836 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Tue, 26 Nov 2024 11:50:28 +0100 Subject: [PATCH 12/56] Drop already installed packages --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 543bced3f3..30f134b958 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,7 @@ ENV DEBIAN_FRONTEND noninteractive # build-essential RUN apt-get -qq -y update \ && apt-get -qq --no-install-recommends -y install locales \ - ca-certificates postgresql-client libpq-dev curl jq git \ - libxml2-dev libxslt1-dev python3-dev \ + postgresql-client jq python3-dev \ && apt-get -qq -y autoremove \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ From 492157acfb96b8bc615065dae9d59e2eb69cd479 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Tue, 26 Nov 2024 15:30:48 +0100 Subject: [PATCH 13/56] Add pyicu and an appropriate test Co-authored-by: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> --- aleph/tests/test_entities_api.py | 19 ++++++++++++++++++- requirements.txt | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/aleph/tests/test_entities_api.py b/aleph/tests/test_entities_api.py index c081da3dfa..1297169a8d 100644 --- a/aleph/tests/test_entities_api.py +++ b/aleph/tests/test_entities_api.py @@ -716,4 +716,21 @@ def test_expand(self): prop = res["property"] assert prop == "holder", prop assert res["count"] == 1, pformat(res) - assert len(res["entities"]) == 1, pformat(res) + assert len(res["entities"]) == 1, pformat(res) + + def test_view_transliterate(self): + _, headers = self.login(is_admin=True) + + data = { + "id": "1", + "schema": "Person", + "properties": { + "name": ["İlham Əliyev"], + }, + } + entity = self.create_entity(data, self.col) + index_entity(entity) + + res = self.client.get(f"/api/2/entities/{entity.id}", headers=headers) + assert res.json["properties"]["name"][0] == "İlham Əliyev" + assert res.json["latinized"]["İlham Əliyev"] == "Ilham Aliyev" diff --git a/requirements.txt b/requirements.txt index 54eb8de6b2..6848a2ac23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ sentry-sdk[flask]==2.10.0 prometheus-client==0.17.1 lxml==5.3.0 lxml_html_clean==0.3.1 +pyicu==2.14 # Testing dependencies From 4647f6ab24522a522a27c717ae800e37c413cc82 Mon Sep 17 00:00:00 2001 From: Martin Tournoij Date: Wed, 27 Nov 2024 17:30:34 +0000 Subject: [PATCH 14/56] Always display "0" in dashboard if there are no entities (#4009) Previously it was inconsistent: "Alerts", "Exports", and "Investigations" would always display "0" if there's nothing, and the entitysets (Network diagrams etc.) didn't display anything. --- ui/src/components/Dashboard/Dashboard.jsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/src/components/Dashboard/Dashboard.jsx b/ui/src/components/Dashboard/Dashboard.jsx index ae5618edab..27a2d8302e 100644 --- a/ui/src/components/Dashboard/Dashboard.jsx +++ b/ui/src/components/Dashboard/Dashboard.jsx @@ -92,14 +92,14 @@ class Dashboard extends React.Component { } + label={} to="/alerts" active={current === '/alerts'} /> } + label={} to="/exports" active={current === '/exports'} /> @@ -115,28 +115,28 @@ class Dashboard extends React.Component { } + label={} to="/investigations" active={current === '/investigations'} /> } + label={} to="/diagrams" active={current === '/diagrams'} /> } + label={} to="/timelines" active={current === '/timelines'} /> } + label={} to="/lists" active={current === '/lists'} /> From 03f7059c4f302bcab1db406a4617cc606ec0cc77 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Wed, 27 Nov 2024 19:41:36 +0100 Subject: [PATCH 15/56] Fix formatting (#4019) --- ui/src/components/Dashboard/Dashboard.jsx | 4 +++- ui/src/components/common/DeleteDialog.tsx | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ui/src/components/Dashboard/Dashboard.jsx b/ui/src/components/Dashboard/Dashboard.jsx index 27a2d8302e..7e7a236554 100644 --- a/ui/src/components/Dashboard/Dashboard.jsx +++ b/ui/src/components/Dashboard/Dashboard.jsx @@ -129,7 +129,9 @@ class Dashboard extends React.Component { } + label={ + + } to="/timelines" active={current === '/timelines'} /> diff --git a/ui/src/components/common/DeleteDialog.tsx b/ui/src/components/common/DeleteDialog.tsx index f7aef324d1..4d6e0c8cab 100644 --- a/ui/src/components/common/DeleteDialog.tsx +++ b/ui/src/components/common/DeleteDialog.tsx @@ -30,7 +30,9 @@ const DeleteDialog: FC = ({ }) => { const [isLoading, setIsLoading] = useState(false); const [confirmationValue, setConfirmationValue] = useState(''); - const enabled = confirmationValue.toLowerCase().trim() === expectedConfirmationValue.toLowerCase().trim(); + const enabled = + confirmationValue.toLowerCase().trim() === + expectedConfirmationValue.toLowerCase().trim(); return ( From 7a7e9a7bbbf2528aaed96e5ba6633d53eb537e37 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Wed, 27 Nov 2024 19:49:20 +0100 Subject: [PATCH 16/56] Handle layouts without vertices key gracefully (#4020) This is an alternative to #4012 with proper types. --- ui/src/components/Timeline/state.test.ts | 16 +++++++++------- ui/src/components/Timeline/state.ts | 6 +++--- ui/src/components/Timeline/types.ts | 2 +- ui/src/components/Timeline/util.ts | 6 +++--- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/ui/src/components/Timeline/state.test.ts b/ui/src/components/Timeline/state.test.ts index 19f2a2a678..a83ba13f0c 100644 --- a/ui/src/components/Timeline/state.test.ts +++ b/ui/src/components/Timeline/state.test.ts @@ -12,7 +12,7 @@ let state: State; beforeEach(() => { state = { entities: [entity], - layout: { vertices: [] }, + layout: {}, selectedId: null, renderer: 'list', zoomLevel: 'months', @@ -65,17 +65,19 @@ describe('UPDATE_VERTEX', () => { }); expect(newState.layout.vertices).toHaveLength(1); - expect(newState.layout.vertices[0]).toEqual({ + expect(newState.layout.vertices?.[0]).toEqual({ entityId: '123', color: 'pink', }); }); it('updates existing vertex', () => { - state.layout.vertices.push({ - entityId: '123', - color: 'blue', - }); + state.layout.vertices = [ + { + entityId: '123', + color: 'blue', + }, + ]; const vertex = { entityId: '123', @@ -88,7 +90,7 @@ describe('UPDATE_VERTEX', () => { }); expect(newState.layout.vertices).toHaveLength(1); - expect(newState.layout.vertices[0]).toEqual({ + expect(newState.layout.vertices?.[0]).toEqual({ entityId: '123', color: 'pink', }); diff --git a/ui/src/components/Timeline/state.ts b/ui/src/components/Timeline/state.ts index e4a6472d23..8e0ebeab2e 100644 --- a/ui/src/components/Timeline/state.ts +++ b/ui/src/components/Timeline/state.ts @@ -168,7 +168,7 @@ export function reducer(state: State, action: Action): State { export function useTimelineState(entities: Array, layout?: Layout) { return useReducer(reducer, { entities, - layout: layout || { vertices: [] }, + layout: layout || {}, selectedId: null, renderer: 'list', zoomLevel: 'months', @@ -190,10 +190,10 @@ export function selectSelectedVertex(state: State): Vertex | null { } const defaultVertex = { entityId: entity.id }; + const vertices = state.layout.vertices || []; return ( - state.layout.vertices.find((vertex) => vertex.entityId === entity.id) || - defaultVertex + vertices.find((vertex) => vertex.entityId === entity.id) || defaultVertex ); } diff --git a/ui/src/components/Timeline/types.ts b/ui/src/components/Timeline/types.ts index 653ea40852..f8cd51ab00 100644 --- a/ui/src/components/Timeline/types.ts +++ b/ui/src/components/Timeline/types.ts @@ -15,7 +15,7 @@ export type Vertex = { }; export type Layout = { - vertices: Array; + vertices?: Array; }; export type EdgeSchema = Schema & { diff --git a/ui/src/components/Timeline/util.ts b/ui/src/components/Timeline/util.ts index e2423b59ec..5fb396926f 100644 --- a/ui/src/components/Timeline/util.ts +++ b/ui/src/components/Timeline/util.ts @@ -118,7 +118,7 @@ export class TimelineItem { } getColor() { - if (!this.layout) { + if (!this.layout || !this.layout.vertices) { return DEFAULT_COLOR; } @@ -403,8 +403,8 @@ export function useFormValidity(formRef: RefObject) { * Return a new layout object, replacing or inserting the given vertex. */ export function updateVertex(layout: Layout, updatedVertex: Vertex): Layout { - const { vertices } = layout; - const index = layout.vertices.findIndex( + const vertices = layout.vertices || []; + const index = vertices.findIndex( (vertex) => vertex.entityId === updatedVertex.entityId ); From c39618669be4ad1b40ec3bbf0785434ad07080f2 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Thu, 28 Nov 2024 17:09:29 +0100 Subject: [PATCH 17/56] Prepopulate document search with query string (#4004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a small enhancement and follow-up #3879. It automatically prepopulates the document-scope search input with the query string of the global search. If for example a user searches for "Elizabeth Bennet" and they click on one of the PDF results, the document-scope search input will be prepopulated with "Elizabeth Bennet". If they want to find pages relevant to their search term, they don’t have to retype the search query and can simply press enter. As an alternative I’ve considered adding a small banner below the tab bar ("Do you want to search for 'Elizabeth Bennet' in this document?") which would probably be more obvious. Decided to try this option first as it adds less clutter to the UI. If it turns out the functionality isn’t clear to users, we can always iterate. --- .../components/EntitySearch/EntitySearchResults.jsx | 2 ++ .../EntitySearch/EntitySearchResultsRow.jsx | 11 +++++++++-- ui/src/components/common/Entity.jsx | 11 +++++++++-- ui/src/util/togglePreview.js | 11 ++++++++++- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/ui/src/components/EntitySearch/EntitySearchResults.jsx b/ui/src/components/EntitySearch/EntitySearchResults.jsx index b3dc3b4ccc..dabaf8f24a 100644 --- a/ui/src/components/EntitySearch/EntitySearchResults.jsx +++ b/ui/src/components/EntitySearch/EntitySearchResults.jsx @@ -56,6 +56,7 @@ class EntitySearchResults extends Component { showPreview = true, updateSelection, selection, + query, } = this.props; if (result.isError) { @@ -89,6 +90,7 @@ class EntitySearchResults extends Component { selection={selection} writeable={writeable} columns={columns} + queryText={query.getString('q')} /> ))} {result.isPending && diff --git a/ui/src/components/EntitySearch/EntitySearchResultsRow.jsx b/ui/src/components/EntitySearch/EntitySearchResultsRow.jsx index 36a91f6a3a..828c9f5ad2 100644 --- a/ui/src/components/EntitySearch/EntitySearchResultsRow.jsx +++ b/ui/src/components/EntitySearch/EntitySearchResultsRow.jsx @@ -40,11 +40,18 @@ class EntitySearchResultsRow extends Component { } renderCellContent(column) { - const { entity, model, showPreview } = this.props; + const { entity, model, showPreview, queryText } = this.props; const { isProperty, name, type } = column; if (name === 'caption') { - return ; + return ( + + ); } else if (name === 'collection_id') { return ( Date: Thu, 28 Nov 2024 17:03:14 +0000 Subject: [PATCH 18/56] Always use POST from ui, never PUT (#4010) Aleph supports both POST and PUT for endpoints that modify data, which are always the same. Just always send POST from the frontend, so we don't need to add every Post handler twice. It already uses POST most of the time. --- ui/src/actions/entityActions.js | 4 ++-- ui/src/actions/entityMappingActions.js | 6 +++--- ui/src/actions/entitySetActions.js | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/src/actions/entityActions.js b/ui/src/actions/entityActions.js index 814721cdc3..4bd15d6e73 100644 --- a/ui/src/actions/entityActions.js +++ b/ui/src/actions/entityActions.js @@ -41,7 +41,7 @@ export const createEntity = asyncActionCreator( const config = { params: { sync: true } }; const payload = entity.toJSON(); payload.collection_id = collection_id; - const response = await endpoint.put( + const response = await endpoint.post( `entities/${entity.id}`, payload, config @@ -55,7 +55,7 @@ export const updateEntity = asyncActionCreator( (entity) => async () => { const config = { params: { sync: true } }; const payload = entity.toJSON(); - const response = await endpoint.put( + const response = await endpoint.post( `entities/${entity.id}`, payload, config diff --git a/ui/src/actions/entityMappingActions.js b/ui/src/actions/entityMappingActions.js index eb9b033d43..62e823c0d6 100644 --- a/ui/src/actions/entityMappingActions.js +++ b/ui/src/actions/entityMappingActions.js @@ -20,7 +20,7 @@ export const fetchEntityMapping = asyncActionCreator( ); const executeTrigger = async (collectionId, mappingId) => - endpoint.put(`collections/${collectionId}/mappings/${mappingId}/trigger`); + endpoint.post(`collections/${collectionId}/mappings/${mappingId}/trigger`); export const createEntityMapping = asyncActionCreator( ({ id, collection }, mapping) => @@ -51,7 +51,7 @@ export const deleteEntityMapping = asyncActionCreator( export const flushEntityMapping = asyncActionCreator( ({ id, collection }, mappingId) => async () => { - await endpoint.put( + await endpoint.post( `collections/${collection.id}/mappings/${mappingId}/flush` ); return { id }; @@ -62,7 +62,7 @@ export const flushEntityMapping = asyncActionCreator( export const updateEntityMapping = asyncActionCreator( ({ id, collection }, mappingId, mapping) => async () => { - const response = await endpoint.put( + const response = await endpoint.post( `collections/${collection.id}/mappings/${mappingId}`, mapping ); diff --git a/ui/src/actions/entitySetActions.js b/ui/src/actions/entitySetActions.js index e253f23611..18ae25a34e 100644 --- a/ui/src/actions/entitySetActions.js +++ b/ui/src/actions/entitySetActions.js @@ -35,7 +35,7 @@ export const createEntitySetNoMutate = asyncActionCreator(createEntitySet, { export const updateEntitySet = asyncActionCreator( (entitySetId, entitySet) => async () => { - const response = await endpoint.put(`entitysets/${entitySetId}`, entitySet); + const response = await endpoint.post(`entitysets/${entitySetId}`, entitySet); return { entitySetId, data: response.data }; }, { name: 'UPDATE_ENTITYSET' } @@ -54,7 +54,7 @@ export const entitySetAddEntity = asyncActionCreator( async () => { const config = { params: { sync } }; const payload = entity.toJSON(); - const response = await endpoint.put( + const response = await endpoint.post( `entitysets/${entitySetId}/entities`, payload, config From c9e1a426624fc2319a7cc1d1c73f3886f6fb8867 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:11:28 +0100 Subject: [PATCH 19/56] Bump rollup from 2.78.1 to 2.79.2 in /ui (#3891) Bumps [rollup](https://github.com/rollup/rollup) from 2.78.1 to 2.79.2. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/master/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v2.78.1...v2.79.2) --- updated-dependencies: - dependency-name: rollup dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/package-lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 47e00873b1..3944874b8b 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "aleph-ui", - "version": "3.17.0", + "version": "4.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "aleph-ui", - "version": "3.17.0", + "version": "4.0.0", "dependencies": { "@alephdata/followthemoney": "^3.5.5", "@blueprintjs/colors": "^4.1.5", @@ -22698,9 +22698,9 @@ } }, "node_modules/rollup": { - "version": "2.78.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.78.1.tgz", - "integrity": "sha512-VeeCgtGi4P+o9hIg+xz4qQpRl6R401LWEXBmxYKOV4zlF82lyhgh2hTZnheFUbANE8l2A41F458iwj2vEYaXJg==", + "version": "2.79.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", + "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", "dev": true, "bin": { "rollup": "dist/bin/rollup" @@ -42911,9 +42911,9 @@ } }, "rollup": { - "version": "2.78.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.78.1.tgz", - "integrity": "sha512-VeeCgtGi4P+o9hIg+xz4qQpRl6R401LWEXBmxYKOV4zlF82lyhgh2hTZnheFUbANE8l2A41F458iwj2vEYaXJg==", + "version": "2.79.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", + "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", "dev": true, "requires": { "fsevents": "~2.3.2" From 5117dca27d958474d1cf8b191c257ba235c0ee40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:11:59 +0100 Subject: [PATCH 20/56] Bump mermaid from 10.9.1 to 10.9.3 in /docs (#3950) Bumps [mermaid](https://github.com/mermaid-js/mermaid) from 10.9.1 to 10.9.3. - [Release notes](https://github.com/mermaid-js/mermaid/releases) - [Changelog](https://github.com/mermaid-js/mermaid/blob/develop/CHANGELOG.md) - [Commits](https://github.com/mermaid-js/mermaid/compare/v10.9.1...v10.9.3) --- updated-dependencies: - dependency-name: mermaid dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index 1c25cae2dd..a51df284a5 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -4322,9 +4322,9 @@ } }, "node_modules/mermaid": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.1.tgz", - "integrity": "sha512-Mx45Obds5W1UkW1nv/7dHRsbfMM1aOKA2+Pxs/IGHNonygDHwmng8xTHyS9z4KWVi0rbko8gjiBmuwwXQ7tiNA==", + "version": "10.9.3", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.3.tgz", + "integrity": "sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==", "dependencies": { "@braintree/sanitize-url": "^6.0.1", "@types/d3-scale": "^4.0.3", @@ -4335,7 +4335,7 @@ "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "^3.0.5", + "dompurify": "^3.0.5 <3.1.7", "elkjs": "^0.9.0", "katex": "^0.16.9", "khroma": "^2.0.0", @@ -10442,9 +10442,9 @@ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" }, "mermaid": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.1.tgz", - "integrity": "sha512-Mx45Obds5W1UkW1nv/7dHRsbfMM1aOKA2+Pxs/IGHNonygDHwmng8xTHyS9z4KWVi0rbko8gjiBmuwwXQ7tiNA==", + "version": "10.9.3", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.9.3.tgz", + "integrity": "sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==", "requires": { "@braintree/sanitize-url": "^6.0.1", "@types/d3-scale": "^4.0.3", @@ -10455,7 +10455,7 @@ "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "^3.0.5", + "dompurify": "^3.0.5 <3.1.7", "elkjs": "^0.9.0", "katex": "^0.16.9", "khroma": "^2.0.0", From 298122bfa37cf34e7f1afde2a103e0b27d82bb3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:12:13 +0100 Subject: [PATCH 21/56] Bump cross-spawn from 7.0.3 to 7.0.6 in /docs (#4006) Bumps [cross-spawn](https://github.com/moxystudio/node-cross-spawn) from 7.0.3 to 7.0.6. - [Changelog](https://github.com/moxystudio/node-cross-spawn/blob/master/CHANGELOG.md) - [Commits](https://github.com/moxystudio/node-cross-spawn/compare/v7.0.3...v7.0.6) --- updated-dependencies: - dependency-name: cross-spawn dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index a51df284a5..3aa6cc172a 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -2311,9 +2311,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -9014,9 +9014,9 @@ } }, "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", From 24710f0cda1934edb19850086d5840b7d398cae3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:12:29 +0100 Subject: [PATCH 22/56] Bump vite from 4.4.7 to 4.5.5 in /docs (#3886) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.4.7 to 4.5.5. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.5/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.5/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 66 +++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index 3aa6cc172a..41bc1686dc 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -5888,6 +5888,21 @@ "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==" }, + "node_modules/rollup": { + "version": "3.29.5", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.5.tgz", + "integrity": "sha512-GVsDdsbJzzy4S/v3dqWPJ7EfvZJfCHiDqe80IyrF59LYuP+e6U1LJoUqeuqRbwAWoMNoXivMNeNAOf5E22VA1w==", + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/run-applescript": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz", @@ -6732,13 +6747,13 @@ } }, "node_modules/vite": { - "version": "4.4.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.7.tgz", - "integrity": "sha512-6pYf9QJ1mHylfVh39HpuSfMPojPSKVxZvnclX1K1FyZ1PXDOcLBibdq5t1qxJSnL63ca8Wf4zts6mD8u8oc9Fw==", + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.5.tgz", + "integrity": "sha512-ifW3Lb2sMdX+WU91s3R0FyQlAyLxOzCSCP37ujw0+r5POeHPwe6udWVIElKQq8gk3t7b8rkmvqC6IHBpCff4GQ==", "dependencies": { "esbuild": "^0.18.10", - "postcss": "^8.4.26", - "rollup": "^3.25.2" + "postcss": "^8.4.27", + "rollup": "^3.27.1" }, "bin": { "vite": "bin/vite.js" @@ -7151,21 +7166,6 @@ "@esbuild/win32-x64": "0.18.17" } }, - "node_modules/vite/node_modules/rollup": { - "version": "3.26.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.26.3.tgz", - "integrity": "sha512-7Tin0C8l86TkpcMtXvQu6saWH93nhG3dGQ1/+l5V2TDMceTxO7kDiK6GzbfLWNNxqJXm591PcEZUozZm51ogwQ==", - "bin": { - "rollup": "dist/bin/rollup" - }, - "engines": { - "node": ">=14.18.0", - "npm": ">=8.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, "node_modules/vitefu": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-0.2.4.tgz", @@ -11445,6 +11445,14 @@ "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==" }, + "rollup": { + "version": "3.29.5", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.5.tgz", + "integrity": "sha512-GVsDdsbJzzy4S/v3dqWPJ7EfvZJfCHiDqe80IyrF59LYuP+e6U1LJoUqeuqRbwAWoMNoXivMNeNAOf5E22VA1w==", + "requires": { + "fsevents": "~2.3.2" + } + }, "run-applescript": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz", @@ -12006,14 +12014,14 @@ } }, "vite": { - "version": "4.4.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.7.tgz", - "integrity": "sha512-6pYf9QJ1mHylfVh39HpuSfMPojPSKVxZvnclX1K1FyZ1PXDOcLBibdq5t1qxJSnL63ca8Wf4zts6mD8u8oc9Fw==", + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.5.tgz", + "integrity": "sha512-ifW3Lb2sMdX+WU91s3R0FyQlAyLxOzCSCP37ujw0+r5POeHPwe6udWVIElKQq8gk3t7b8rkmvqC6IHBpCff4GQ==", "requires": { "esbuild": "^0.18.10", "fsevents": "~2.3.2", - "postcss": "^8.4.26", - "rollup": "^3.25.2" + "postcss": "^8.4.27", + "rollup": "^3.27.1" }, "dependencies": { "@esbuild/android-arm": { @@ -12176,14 +12184,6 @@ "@esbuild/win32-ia32": "0.18.17", "@esbuild/win32-x64": "0.18.17" } - }, - "rollup": { - "version": "3.26.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.26.3.tgz", - "integrity": "sha512-7Tin0C8l86TkpcMtXvQu6saWH93nhG3dGQ1/+l5V2TDMceTxO7kDiK6GzbfLWNNxqJXm591PcEZUozZm51ogwQ==", - "requires": { - "fsevents": "~2.3.2" - } } } }, From d66cf424c5fc46fef7472be3338ed84f118e4d47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:12:43 +0100 Subject: [PATCH 23/56] Bump undici from 5.26.3 to 5.28.4 in /docs (#3677) Bumps [undici](https://github.com/nodejs/undici) from 5.26.3 to 5.28.4. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v5.26.3...v5.28.4) --- updated-dependencies: - dependency-name: undici dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index 41bc1686dc..0dda550452 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -6474,9 +6474,9 @@ } }, "node_modules/undici": { - "version": "5.26.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", - "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", + "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dependencies": { "@fastify/busboy": "^2.0.0" }, @@ -11840,9 +11840,9 @@ "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==" }, "undici": { - "version": "5.26.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", - "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", + "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "requires": { "@fastify/busboy": "^2.0.0" } From b6f604a543f94d3a3d13505bf6d057a1f3d0f251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:18:10 +0100 Subject: [PATCH 24/56] Bump rollup from 3.26.3 to 3.29.5 in /docs (#3887) Bumps [rollup](https://github.com/rollup/rollup) from 3.26.3 to 3.29.5. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/master/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v3.26.3...v3.29.5) --- updated-dependencies: - dependency-name: rollup dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From 8626144097304f461f506ff18d7abdd093e29b6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:18:21 +0100 Subject: [PATCH 25/56] Bump path-to-regexp from 6.2.1 to 6.3.0 in /docs (#4049) Bumps [path-to-regexp](https://github.com/pillarjs/path-to-regexp) from 6.2.1 to 6.3.0. - [Release notes](https://github.com/pillarjs/path-to-regexp/releases) - [Changelog](https://github.com/pillarjs/path-to-regexp/blob/master/History.md) - [Commits](https://github.com/pillarjs/path-to-regexp/compare/v6.2.1...v6.3.0) --- updated-dependencies: - dependency-name: path-to-regexp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index 0dda550452..38e8922e3e 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -5331,9 +5331,9 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-to-regexp": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", - "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==" + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" }, "node_modules/periscopic": { "version": "3.1.0", @@ -11063,9 +11063,9 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "path-to-regexp": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", - "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==" + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==" }, "periscopic": { "version": "3.1.0", From 1276d401a6b6dca9f2cbee03b34b0a5cfbab8758 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:26:04 +0100 Subject: [PATCH 26/56] Bump dset from 3.1.2 to 3.1.4 in /docs (#4051) Bumps [dset](https://github.com/lukeed/dset) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/lukeed/dset/releases) - [Commits](https://github.com/lukeed/dset/compare/v3.1.2...v3.1.4) --- updated-dependencies: - dependency-name: dset dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index 38e8922e3e..54bb1a5319 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -2921,9 +2921,9 @@ "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" }, "node_modules/dset": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.2.tgz", - "integrity": "sha512-g/M9sqy3oHe477Ar4voQxWtaPIFw1jTdKZuomOjhCcBx9nHUNn0pu6NopuFFrTh/TRZIKEj+76vLWFu9BNKk+Q==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", "engines": { "node": ">=4" } @@ -9461,9 +9461,9 @@ "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" }, "dset": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.2.tgz", - "integrity": "sha512-g/M9sqy3oHe477Ar4voQxWtaPIFw1jTdKZuomOjhCcBx9nHUNn0pu6NopuFFrTh/TRZIKEj+76vLWFu9BNKk+Q==" + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==" }, "eastasianwidth": { "version": "0.2.0", From e0fa3f6cde2a89e91b05c8cd22b8fdc4865dbec9 Mon Sep 17 00:00:00 2001 From: Till Prochaska <1512805+tillprochaska@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:49:00 +0100 Subject: [PATCH 27/56] House cleaning (#4021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix formatting * Remove unused imports * Remove unused assignments * Fix linter ignore This was ignored before already, but it didn’t apply because the comment wasn’t actually placed on the preceding line. It would be better to fix this properly, but that’s not possible without backend changes. * Use strict type comparisons * Fix typo (Ctl/Ctrl) I think `Ctl` is occasionally used as well, but `Ctrl` is much more common and what you’ll find on most keyboards * Use strict comparisons in table editor * Use `forEach` instead of `map` This caused a linter warning. `map` is only used here to iterate over the arrays, so `forEach` is the proper replacement. * Remove unused files These are leftovers from the time when react-ftm was a separate package * Remove useless constructors * Remove unused network diagram layout helper The code in this file has been commented out for a long time, the file isn’t imported anywhere else, so I assume it’s safe to remove. * Configure ESLint to fail if there are warnings Now that all linter warnings are resolved, let’s make sure we don’t introduce any new ones. --- ui/package.json | 10 +---- ui/src/actions/entitySetActions.js | 5 ++- ui/src/components/Entity/EntityViews.jsx | 4 +- ui/src/components/common/BookmarksDrawer.tsx | 1 - ui/src/react-ftm/__mocks__/fileMock.js | 1 - ui/src/react-ftm/__mocks__/styleMock.js | 0 .../components/EntityTable/TableEditor.tsx | 10 ++--- .../layout/tools/getForceData.ts | 4 +- .../layout/tools/removeCollisions.ts | 40 ------------------- .../toolbox/ToolbarButtonGroup.tsx | 4 -- ui/src/react-ftm/editors/EdgeTypeSelect.tsx | 4 -- ui/src/react-ftm/types/Transliterate.tsx | 3 +- ui/src/screens/GroupScreen/GroupScreen.jsx | 2 +- ui/src/screens/OAuthScreen/OAuthScreen.jsx | 2 +- ui/src/viewers/PdfViewer.jsx | 22 +--------- ui/src/viewers/PdfViewerSearch.jsx | 4 +- 16 files changed, 22 insertions(+), 94 deletions(-) delete mode 100644 ui/src/react-ftm/__mocks__/fileMock.js delete mode 100644 ui/src/react-ftm/__mocks__/styleMock.js delete mode 100644 ui/src/react-ftm/components/NetworkDiagram/layout/tools/removeCollisions.ts diff --git a/ui/package.json b/ui/package.json index dabbdc4885..e7779ac048 100644 --- a/ui/package.json +++ b/ui/package.json @@ -60,7 +60,7 @@ "scripts": { "start": "craco start", "build": "craco build", - "lint": "eslint --ext js,jsx,ts,tsx src", + "lint": "eslint --ext js,jsx,ts,tsx src --max-warnings 0", "test": "craco test", "eject": "craco eject", "format": "prettier --write 'src/**/*.{js,jsx,ts,tsx,css,scss}'", @@ -72,13 +72,7 @@ }, "proxy": "http://api:5000/", "eslintConfig": { - "extends": "react-app", - "rules": { - "no-console": "error", - "no-debugger": "error", - "react/no-unknown-property": "error", - "react/jsx-key": "error" - } + "extends": "react-app" }, "jest": { "transformIgnorePatterns": [] diff --git a/ui/src/actions/entitySetActions.js b/ui/src/actions/entitySetActions.js index 18ae25a34e..54c7eda8fe 100644 --- a/ui/src/actions/entitySetActions.js +++ b/ui/src/actions/entitySetActions.js @@ -35,7 +35,10 @@ export const createEntitySetNoMutate = asyncActionCreator(createEntitySet, { export const updateEntitySet = asyncActionCreator( (entitySetId, entitySet) => async () => { - const response = await endpoint.post(`entitysets/${entitySetId}`, entitySet); + const response = await endpoint.post( + `entitysets/${entitySetId}`, + entitySet + ); return { entitySetId, data: response.data }; }, { name: 'UPDATE_ENTITYSET' } diff --git a/ui/src/components/Entity/EntityViews.jsx b/ui/src/components/Entity/EntityViews.jsx index 1cf9b60277..8c91777fb9 100644 --- a/ui/src/components/Entity/EntityViews.jsx +++ b/ui/src/components/Entity/EntityViews.jsx @@ -348,7 +348,7 @@ class EntityViews extends React.Component { } renderPdfSearchMode() { - const { entity, isPreview, activeMode } = this.props; + const { entity, isPreview } = this.props; if (!this.hasPdfSearchMode()) { return; @@ -364,7 +364,7 @@ class EntityViews extends React.Component { } renderPdfSearchForm() { - const { location, isPreview, activeMode } = this.props; + const { location, isPreview } = this.props; if (!this.hasPdfSearchMode()) { return; diff --git a/ui/src/components/common/BookmarksDrawer.tsx b/ui/src/components/common/BookmarksDrawer.tsx index b269991966..d09eef8db8 100644 --- a/ui/src/components/common/BookmarksDrawer.tsx +++ b/ui/src/components/common/BookmarksDrawer.tsx @@ -1,6 +1,5 @@ import { FC } from 'react'; import { FormattedMessage } from 'react-intl'; -import { useSelector } from 'react-redux'; import { Classes, Drawer, DrawerSize } from '@blueprintjs/core'; import BookmarksList from './BookmarksList'; diff --git a/ui/src/react-ftm/__mocks__/fileMock.js b/ui/src/react-ftm/__mocks__/fileMock.js deleted file mode 100644 index 602eb23ee2..0000000000 --- a/ui/src/react-ftm/__mocks__/fileMock.js +++ /dev/null @@ -1 +0,0 @@ -export default 'test-file-stub'; diff --git a/ui/src/react-ftm/__mocks__/styleMock.js b/ui/src/react-ftm/__mocks__/styleMock.js deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ui/src/react-ftm/components/EntityTable/TableEditor.tsx b/ui/src/react-ftm/components/EntityTable/TableEditor.tsx index f4e97c932a..ca7040ed51 100644 --- a/ui/src/react-ftm/components/EntityTable/TableEditor.tsx +++ b/ui/src/react-ftm/components/EntityTable/TableEditor.tsx @@ -248,7 +248,7 @@ class TableEditorBase extends React.Component< this.getHeaderCell(property) ); const entityLinkPlaceholder = - visitEntity != undefined ? [this.getEntityLinkCell()] : []; + visitEntity !== undefined ? [this.getEntityLinkCell()] : []; if (writeable) { const addEntityCell = this.getAddEntityCell(); @@ -288,7 +288,7 @@ class TableEditorBase extends React.Component< }); const entityLinkCell = - visitEntity != undefined ? [this.getEntityLinkCell(entity)] : []; + visitEntity !== undefined ? [this.getEntityLinkCell(entity)] : []; if (!writeable) { return [...entityLinkCell, ...propCells]; @@ -335,7 +335,7 @@ class TableEditorBase extends React.Component< const skeletonRowCount = 8; const entityLinkPlaceholder = - visitEntity != undefined ? [this.getEntityLinkCell()] : []; + visitEntity !== undefined ? [this.getEntityLinkCell()] : []; const actionCellPlaceholder = writeable ? [{ ...getCellBase('checkbox') }] : []; @@ -356,7 +356,7 @@ class TableEditorBase extends React.Component< const { visibleProps } = this.state; const entityLinkPlaceholder = - visitEntity != undefined ? [this.getEntityLinkCell()] : []; + visitEntity !== undefined ? [this.getEntityLinkCell()] : []; const addRowCells = visibleProps.map((property) => ({ ...getCellBase('property'), @@ -591,7 +591,7 @@ class TableEditorBase extends React.Component< renderEntityLinkButton = ({ entity }: { entity: FTMEntity }) => { const { visitEntity } = this.props; - if (visitEntity == undefined) return null; + if (visitEntity === undefined) return null; return ( diff --git a/ui/src/components/AuthButtons/AuthButtons.scss b/ui/src/components/AuthButtons/AuthButtons.scss index 7a71f95036..d94d8cd980 100644 --- a/ui/src/components/AuthButtons/AuthButtons.scss +++ b/ui/src/components/AuthButtons/AuthButtons.scss @@ -25,4 +25,11 @@ } } } + + &__label { + display: -webkit-box; + max-width: 120px; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + } } diff --git a/ui/src/components/common/Summary.jsx b/ui/src/components/common/Summary.jsx index da914c77b7..0c694f63a8 100644 --- a/ui/src/components/common/Summary.jsx +++ b/ui/src/components/common/Summary.jsx @@ -1,9 +1,10 @@ import React from 'react'; import ReactMarkdown from 'react-markdown'; -import Truncate from 'react-truncate'; import { Classes } from '@blueprintjs/core'; import c from 'classnames'; +import './Summary.scss'; + // formats markdown elements to plain text const simpleRenderer = ({ children }) => ( <> @@ -28,13 +29,21 @@ const Summary = ({ className, text, truncate }) => { return (
- {truncate && {content}} + {truncate && ( + + {content} + + )} {!truncate && content}
); diff --git a/ui/src/components/common/Summary.scss b/ui/src/components/common/Summary.scss new file mode 100644 index 0000000000..e6269f7f7e --- /dev/null +++ b/ui/src/components/common/Summary.scss @@ -0,0 +1,5 @@ +.Summary__truncate { + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; +} diff --git a/ui/src/react-ftm/components/NetworkDiagram/renderer/Canvas.tsx b/ui/src/react-ftm/components/NetworkDiagram/renderer/Canvas.tsx index 4895b33837..ff2fff518b 100644 --- a/ui/src/react-ftm/components/NetworkDiagram/renderer/Canvas.tsx +++ b/ui/src/react-ftm/components/NetworkDiagram/renderer/Canvas.tsx @@ -209,11 +209,11 @@ export class Canvas extends React.Component { this.props.actions.addVertex({ initialPosition: gridTarget }); } - componentWillReceiveProps(nextProps: Readonly): void { + componentDidUpdate(prevProps: Readonly): void { this.animationHandler( - nextProps.animateTransition, - this.props.viewBox || '', - nextProps.viewBox || '' + this.props.animateTransition, + prevProps.viewBox || '', + this.props.viewBox || '' ); } diff --git a/ui/src/react-ftm/components/NetworkDiagram/toolbox/EntityViewer.tsx b/ui/src/react-ftm/components/NetworkDiagram/toolbox/EntityViewer.tsx index dffef8d3c0..9e55192ffe 100644 --- a/ui/src/react-ftm/components/NetworkDiagram/toolbox/EntityViewer.tsx +++ b/ui/src/react-ftm/components/NetworkDiagram/toolbox/EntityViewer.tsx @@ -28,7 +28,7 @@ interface IEntityViewerProps { } interface IEntityViewerState { - visibleProps: FTMProperty[]; + selectedProperties: FTMProperty[]; currEditing: FTMProperty | null; } @@ -37,17 +37,13 @@ export class EntityViewer extends React.PureComponent< IEntityViewerState > { static contextType = GraphContext; - private schemaProperties: FTMProperty[]; constructor(props: IEntityViewerProps) { super(props); - this.schemaProperties = props.entity.schema - .getEditableProperties() - .sort((a, b) => a.label.localeCompare(b.label)); this.state = { - visibleProps: this.getVisibleProperties(props), currEditing: null, + selectedProperties: [], }; this.onNewPropertySelected = this.onNewPropertySelected.bind(this); @@ -56,29 +52,27 @@ export class EntityViewer extends React.PureComponent< this.onEditPropertyClick = this.onEditPropertyClick.bind(this); } - getVisibleProperties(props = this.props) { - const { entity } = props; + getSchemaProperties(): FTMProperty[] { + return this.props.entity.schema + .getEditableProperties() + .sort((a, b) => a.label.localeCompare(b.label)); + } + + getVisibleProperties(): FTMProperty[] { + const { entity } = this.props; return Array.from( new Set([ ...entity.schema.getFeaturedProperties(), ...entity.getProperties(), + ...this.state.selectedProperties, ]) ); } - componentWillReceiveProps(nextProps: Readonly): void { - if (this.props.entity !== nextProps.entity) { - this.schemaProperties = nextProps.entity.schema.getEditableProperties(); - this.setState({ - visibleProps: this.getVisibleProperties(nextProps), - }); - } - } - onNewPropertySelected(p: FTMProperty) { - this.setState(({ visibleProps }) => ({ - visibleProps: [...visibleProps, ...[p]], + this.setState(({ selectedProperties }) => ({ + selectedProperties: [...selectedProperties, p], currEditing: null, })); } @@ -127,8 +121,8 @@ export class EntityViewer extends React.PureComponent< render() { const { writeable } = this.context; const { entity, vertexRef } = this.props; - const { visibleProps } = this.state; - const availableProperties = this.schemaProperties.filter( + const visibleProps = this.getVisibleProperties(); + const availableProperties = this.getSchemaProperties().filter( (p) => visibleProps.indexOf(p) < 0 ); const hasCaption = entity.getCaption() !== entity.schema.label; diff --git a/ui/src/react-ftm/components/NetworkDiagram/toolbox/Sidebar.tsx b/ui/src/react-ftm/components/NetworkDiagram/toolbox/Sidebar.tsx index 3ffcaf943d..57f1ec593f 100644 --- a/ui/src/react-ftm/components/NetworkDiagram/toolbox/Sidebar.tsx +++ b/ui/src/react-ftm/components/NetworkDiagram/toolbox/Sidebar.tsx @@ -137,6 +137,7 @@ export class Sidebar extends React.Component { } contents = ( Date: Tue, 10 Dec 2024 10:32:17 +0100 Subject: [PATCH 56/56] Hash API keys (#3842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Store hashed API keys Aleph used to store user API keys as plaintext in the database. This commit changes that to store only a hash of the API key. API keys are generated using the built-in `secrets.token_urlsafe` method which returns a random 256 bit token. In contrast to passwords, API keys are not provided by users, have a high entropy, and need to be validated on every request. It seems to be generally accepted that, given 256 bit tokens, salting or using an expensive key derivation functions isn't necessary. For this reason, we’re storing an unsalted SHA-256 hash of the API key which also makes it easy to look up and verify a given API key. I've added a separate column for the hashed API key rather than reusing the existing column. This allows us to batch-hash all existing plaintext keys without having to differentiate between keys that have already been hashed and those that haven't. Once all existing plaintext API keys have been hashed, the old `api_key` column can simply be dropped. * Add CLI command to store legacy plaintext API keys * Remove prefilled API key from OpenRefine endpoints Required as we do not store plaintext API keys anymore. Also, we want to remove the option to pass API keys via URL parameters in the future. This makes it impossible to use OpenRefine with non-public collections. This was never documented, and most users weren't aware that they can indeed use OpenRefine with non-public collections anyway. --- aleph/logic/api_keys.py | 33 +++++++++++++--- aleph/logic/util.py | 9 +++++ aleph/manage.py | 11 +++++- .../31e24765dee3_add_api_key_digest_column.py | 28 ++++++++++++++ aleph/model/role.py | 12 ++++-- aleph/tests/test_api_keys.py | 38 +++++++++++++++---- aleph/tests/test_role_model.py | 13 ++++--- aleph/tests/test_view_context.py | 9 +++-- aleph/views/reconcile_api.py | 2 - 9 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py diff --git a/aleph/logic/api_keys.py b/aleph/logic/api_keys.py index 98680ffe2d..96f960f3f5 100644 --- a/aleph/logic/api_keys.py +++ b/aleph/logic/api_keys.py @@ -9,7 +9,7 @@ from aleph.model.common import make_token from aleph.logic.mail import email_role from aleph.logic.roles import update_role -from aleph.logic.util import ui_url +from aleph.logic.util import ui_url, hash_api_key # Number of days after which API keys expire API_KEY_EXPIRATION_DAYS = 90 @@ -29,7 +29,8 @@ def generate_user_api_key(role): email_role(role, subject, html=html, plain=plain) now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) - role.api_key = make_token() + api_key = make_token() + role.api_key_digest = hash_api_key(api_key) role.api_key_expires_at = now + datetime.timedelta(days=API_KEY_EXPIRATION_DAYS) role.api_key_expiration_notification_sent = None @@ -37,7 +38,7 @@ def generate_user_api_key(role): db.session.commit() update_role(role) - return role.api_key + return api_key def send_api_key_expiration_notifications(): @@ -70,7 +71,7 @@ def _send_api_key_expiration_notification( query = query.where( and_( and_( - Role.api_key != None, # noqa: E711 + Role.api_key_digest != None, # noqa: E711 func.date(Role.api_key_expires_at) <= threshold, ), or_( @@ -104,10 +105,32 @@ def reset_api_key_expiration(): query = query.yield_per(500) query = query.where( and_( - Role.api_key != None, # noqa: E711 + Role.api_key_digest != None, # noqa: E711 Role.api_key_expires_at == None, # noqa: E711 ) ) query.update({Role.api_key_expires_at: expires_at}) db.session.commit() + + +def hash_plaintext_api_keys(): + query = Role.all_users() + query = query.yield_per(250) + query = query.where( + and_( + Role.api_key != None, # noqa: E711 + Role.api_key_digest == None, # noqa: E711 + ) + ) + + results = db.session.execute(query).scalars() + + for index, partition in enumerate(results.partitions()): + for role in partition: + role.api_key_digest = hash_api_key(role.api_key) + role.api_key = None + db.session.add(role) + log.info(f"Hashing API key: {role}") + log.info(f"Comitting partition {index}") + db.session.commit() diff --git a/aleph/logic/util.py b/aleph/logic/util.py index 2d1b3a2f52..5253feb248 100644 --- a/aleph/logic/util.py +++ b/aleph/logic/util.py @@ -1,4 +1,5 @@ import jwt +import hashlib from normality import ascii_text from urllib.parse import urlencode, urljoin from datetime import datetime, timedelta @@ -58,3 +59,11 @@ def archive_token(token): token = jwt.decode(token, key=SETTINGS.SECRET_KEY, algorithms=DECODE, verify=True) expire = datetime.utcfromtimestamp(token["exp"]) return token.get("c"), token.get("f"), token.get("m"), expire + + +def hash_api_key(api_key): + if api_key is None: + return None + + digest = hashlib.sha256(api_key.encode("utf-8")).hexdigest() + return f"sha256${digest}" diff --git a/aleph/manage.py b/aleph/manage.py index d976edd4b6..0408ec059a 100644 --- a/aleph/manage.py +++ b/aleph/manage.py @@ -23,7 +23,10 @@ from aleph.queues import get_status, cancel_queue from aleph.queues import get_active_dataset_status from aleph.index.admin import delete_index -from aleph.logic.api_keys import reset_api_key_expiration as _reset_api_key_expiration +from aleph.logic.api_keys import ( + reset_api_key_expiration as _reset_api_key_expiration, + hash_plaintext_api_keys as _hash_plaintext_api_keys, +) from aleph.index.entities import iter_proxies from aleph.index.util import AlephOperationalException from aleph.logic.collections import create_collection, update_collection @@ -597,3 +600,9 @@ def evilshit(): def reset_api_key_expiration(): """Reset the expiration date of all legacy, non-expiring API keys.""" _reset_api_key_expiration() + + +@cli.command() +def hash_plaintext_api_keys(): + """Hash legacy plaintext API keys.""" + _hash_plaintext_api_keys() diff --git a/aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py b/aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py new file mode 100644 index 0000000000..dcfd109786 --- /dev/null +++ b/aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py @@ -0,0 +1,28 @@ +"""Add api_key_digest column + +Revision ID: 31e24765dee3 +Revises: d46fc882ec6b +Create Date: 2024-07-04 11:07:19.915782 + +""" + +# revision identifiers, used by Alembic. +revision = "31e24765dee3" +down_revision = "d46fc882ec6b" + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column("role", sa.Column("api_key_digest", sa.Unicode())) + op.create_index( + index_name="ix_role_api_key_digest", + table_name="role", + columns=["api_key_digest"], + unique=True, + ) + + +def downgrade(): + op.drop_column("role", "api_key_digest") diff --git a/aleph/model/role.py b/aleph/model/role.py index eb60eb0464..7d3cc8f74c 100644 --- a/aleph/model/role.py +++ b/aleph/model/role.py @@ -9,6 +9,7 @@ from aleph.settings import SETTINGS from aleph.model.common import SoftDeleteModel, IdModel, query_like from aleph.util import anonymize_email +from aleph.logic.util import hash_api_key log = logging.getLogger(__name__) @@ -52,6 +53,7 @@ class Role(db.Model, IdModel, SoftDeleteModel): email = db.Column(db.Unicode, nullable=True) type = db.Column(db.Enum(*TYPES, name="role_type"), nullable=False) api_key = db.Column(db.Unicode, nullable=True) + api_key_digest = db.Column(db.Unicode, nullable=True) api_key_expires_at = db.Column(db.DateTime, nullable=True) api_key_expiration_notification_sent = db.Column(db.Integer, nullable=True) is_admin = db.Column(db.Boolean, nullable=False, default=False) @@ -72,7 +74,7 @@ def has_password(self): @property def has_api_key(self): - return self.api_key is not None + return self.api_key_digest is not None @property def is_public(self): @@ -197,10 +199,13 @@ def by_email(cls, email): def by_api_key(cls, api_key): if api_key is None: return None + q = cls.all() - q = q.filter_by(api_key=api_key) - utcnow = datetime.now(timezone.utc) + digest = hash_api_key(api_key) + q = q.filter(cls.api_key_digest == digest) + + utcnow = datetime.now(timezone.utc) # TODO: Exclude API keys without expiration date after deadline # See https://github.com/alephdata/aleph/issues/3729 q = q.filter( @@ -212,6 +217,7 @@ def by_api_key(cls, api_key): q = q.filter(cls.type == cls.USER) q = q.filter(cls.is_blocked == False) # noqa + return q.first() @classmethod diff --git a/aleph/tests/test_api_keys.py b/aleph/tests/test_api_keys.py index 62c494a25f..111b7f7912 100644 --- a/aleph/tests/test_api_keys.py +++ b/aleph/tests/test_api_keys.py @@ -5,33 +5,35 @@ from aleph.logic.api_keys import ( generate_user_api_key, send_api_key_expiration_notifications, + hash_plaintext_api_keys, ) +from aleph.logic.util import hash_api_key from aleph.tests.util import TestCase class ApiKeysTestCase(TestCase): def test_generate_user_api_key(self): role = self.create_user() - assert role.api_key is None + assert role.api_key_digest is None assert role.api_key_expires_at is None with time_machine.travel("2024-01-01T00:00:00Z"): generate_user_api_key(role) db.session.refresh(role) - assert role.api_key is not None + assert role.api_key_digest is not None assert role.api_key_expires_at.date() == datetime.date(2024, 3, 31) - old_key = role.api_key + old_digest = role.api_key_digest with time_machine.travel("2024-02-01T00:00:00Z"): generate_user_api_key(role) db.session.refresh(role) - assert role.api_key != old_key + assert role.api_key_digest != old_digest assert role.api_key_expires_at.date() == datetime.date(2024, 5, 1) def test_generate_user_api_key_notification(self): role = self.create_user(email="john.doe@example.org") - assert role.api_key is None + assert role.api_key_digest is None with mail.record_messages() as outbox: assert len(outbox) == 0 @@ -65,7 +67,7 @@ def test_send_api_key_expiration_notifications(self): assert len(outbox) == 1 assert outbox[0].subject == "[Aleph] API key generated" - assert role.api_key is not None + assert role.api_key_digest is not None assert role.api_key_expires_at.date() == datetime.date(2024, 3, 31) assert len(outbox) == 1 @@ -122,7 +124,7 @@ def test_send_api_key_expiration_notifications(self): def test_send_api_key_expiration_notifications_no_key(self): role = self.create_user(email="john.doe@example.org") - assert role.api_key is None + assert role.api_key_digest is None with mail.record_messages() as outbox: assert len(outbox) == 0 @@ -193,3 +195,25 @@ def test_send_api_key_expiration_notifications_regenerate(self): assert outbox[4].subject == "[Aleph] Your API key will expire in 7 days" assert outbox[5].subject == "[Aleph] Your API key has expired" + + def test_hash_plaintext_api_keys(self): + user_1 = self.create_user(foreign_id="user_1", email="user1@example.org") + user_1.api_key = "1234567890" + user_1.api_key_digest = None + + user_2 = self.create_user(foreign_id="user_2", email="user2@example.org") + user_2.api_key = None + user_2.api_key_digest = None + + db.session.add_all([user_1, user_2]) + db.session.commit() + + hash_plaintext_api_keys() + + db.session.refresh(user_1) + assert user_1.api_key is None + assert user_1.api_key_digest == hash_api_key("1234567890") + + db.session.refresh(user_2) + assert user_2.api_key is None + assert user_2.api_key_digest is None diff --git a/aleph/tests/test_role_model.py b/aleph/tests/test_role_model.py index 5ad24cc1e9..a7cf6d242d 100644 --- a/aleph/tests/test_role_model.py +++ b/aleph/tests/test_role_model.py @@ -5,6 +5,7 @@ from aleph.model import Role from aleph.tests.factories.models import RoleFactory from aleph.logic.roles import create_user, create_group +from aleph.logic.util import hash_api_key from aleph.tests.util import TestCase @@ -83,7 +84,7 @@ def test_remove_role(self): def test_role_by_api_key(self): role_ = self.create_user() - role_.api_key = "1234567890" + role_.api_key_digest = hash_api_key("1234567890") db.session.add(role_) db.session.commit() @@ -93,7 +94,7 @@ def test_role_by_api_key(self): def test_role_by_api_key_empty(self): role_ = self.create_user() - assert role_.api_key is None + assert role_.api_key_digest is None role = Role.by_api_key(None) assert role is None @@ -103,26 +104,26 @@ def test_role_by_api_key_empty(self): def test_role_by_api_key_expired(self): role_ = self.create_user() - role_.api_key = "1234567890" + role_.api_key_digest = hash_api_key("1234567890") role_.api_key_expires_at = datetime.datetime(2024, 3, 31, 0, 0, 0) db.session.add(role_) db.session.commit() with time_machine.travel("2024-03-30T23:59:59Z"): print(role_.api_key_expires_at) - role = Role.by_api_key(role_.api_key) + role = Role.by_api_key("1234567890") assert role is not None assert role.id == role_.id with time_machine.travel("2024-03-31T00:00:00Z"): - role = Role.by_api_key(role_.api_key) + role = Role.by_api_key("1234567890") assert role is None def test_role_by_api_key_legacy_without_expiration(self): # Ensure that legacy API keys that were created without an expiration # date continue to work. role_ = self.create_user() - role_.api_key = "1234567890" + role_.api_key_digest = hash_api_key("1234567890") role_.api_key_expires_at = None db.session.add(role_) db.session.commit() diff --git a/aleph/tests/test_view_context.py b/aleph/tests/test_view_context.py index f6aa460c7a..2edf9cdf08 100644 --- a/aleph/tests/test_view_context.py +++ b/aleph/tests/test_view_context.py @@ -1,5 +1,6 @@ from aleph.core import db from aleph.tests.util import TestCase +from aleph.logic.util import hash_api_key class ViewContextTest(TestCase): @@ -7,7 +8,7 @@ def setUp(self): super().setUp() self.role = self.create_user(email="john.doe@example.org") self.role.set_password("12345678") - self.role.api_key = "1234567890" + self.role.api_key_digest = hash_api_key("1234567890") self.other_role = self.create_user( foreign_id="other", @@ -47,12 +48,12 @@ def test_authz_header_session_token_invalid(self): assert res.status_code == 401 def test_authz_header_api_key(self): - headers = {"Authorization": f"ApiKey {self.role.api_key}"} + headers = {"Authorization": "ApiKey 1234567890"} res = self.client.get(f"/api/2/roles/{self.role.id}", headers=headers) assert res.status_code == 200 assert res.json["email"] == "john.doe@example.org" - headers = {"Authorization": self.role.api_key} + headers = {"Authorization": "1234567890"} res = self.client.get(f"/api/2/roles/{self.role.id}", headers=headers) assert res.status_code == 200 assert res.json["email"] == "john.doe@example.org" @@ -83,7 +84,7 @@ def test_authz_header_api_key_invalid(self): assert res.status_code == 403 def test_authz_url_param_api_key(self): - query_string = {"api_key": self.role.api_key} + query_string = {"api_key": "1234567890"} res = self.client.get(f"/api/2/roles/{self.role.id}", query_string=query_string) assert res.status_code == 200 assert res.json["email"] == "john.doe@example.org" diff --git a/aleph/views/reconcile_api.py b/aleph/views/reconcile_api.py index 0cacdcffec..14fd23be1e 100644 --- a/aleph/views/reconcile_api.py +++ b/aleph/views/reconcile_api.py @@ -57,8 +57,6 @@ def reconcile_index(collection=None): domain = SETTINGS.APP_UI_URL.strip("/") label = SETTINGS.APP_TITLE suggest_query = [] - if request.authz.id: - suggest_query.append(("api_key", request.authz.role.api_key)) schemata = list(model) if collection is not None: label = "%s (%s)" % (collection.get("label"), label)