Merge branch 'release/4.1.0' into release/4.0.3

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.3-rc9
+current_version = 4.1.0-rc4
 tag_name = {new_version}
 commit = True
 tag = True

.github/workflows/discourse.yml

@@ -13,6 +13,6 @@ jobs:
         discourse-api-key: ${{ secrets.DISCOURSE_RELEASES_API_KEY }}
         discourse-base-url: https://aleph.discourse.group/
         discourse-category: 5
-        discourse-tags:
+        discourse-tags: |
           release
           aleph

Makefile

@@ -43,7 +43,10 @@ format-check-ui:
 
 upgrade: build
 	$(COMPOSE) up -d postgres elasticsearch
-	sleep 10
+	# wait for postgres to be available
+	@$(COMPOSE) exec postgres pg_isready --timeout=30
+	# wait for elasticsearch to be available
+	@$(COMPOSE) exec elasticsearch timeout 30 bash -c "printf 'Waiting for elasticsearch'; until curl --silent --output /dev/null localhost:9200/_cat/health?h=st; do printf '.'; sleep 1; done; printf '\n'"
 	$(APPDOCKER) aleph upgrade
 
 api: services
@@ -107,20 +110,20 @@ e2e/test-results:
 
 services-e2e:
 	$(COMPOSE_E2E) up -d --remove-orphans \
-		postgres elasticsearch ingest-file \
+		postgres elasticsearch ingest-file rabbitmq worker \
 
 e2e: services-e2e e2e/test-results
 	$(COMPOSE_E2E) run --rm app aleph upgrade
 	$(COMPOSE_E2E) run --rm app aleph createuser --name="E2E Admin" --admin --password="admin" admin@admin.admin
-	$(COMPOSE_E2E) up -d api ui worker
-	BASE_URL=http://ui:8080 $(COMPOSE_E2E) run --rm e2e pytest -s -v --output=/e2e/test-results/ --screenshot=only-on-failure --video=retain-on-failure e2e/
+	$(COMPOSE_E2E) up -d api ui
+	BASE_URL=http://ui:8080 $(COMPOSE_E2E) run --rm e2e pytest -s -v --output=/e2e/test-results/ --screenshot=only-on-failure --video=retain-on-failure --tracing retain-on-failure e2e/
 
 e2e-local-setup: dev
 	python3 -m pip install -q -r e2e/requirements.txt
 	playwright install
 
 e2e-local:
-	pytest -s -v --screenshot only-on-failure e2e/
+	pytest -s -v --screenshot only-on-failure --tracing retain-on-failure e2e/
 
 .PHONY: build services e2e
 

aleph/logic/api_keys.py

@@ -0,0 +1,136 @@
+import datetime
+
+import structlog
+from flask import render_template
+from sqlalchemy import and_, or_, func
+
+from aleph.core import db
+from aleph.model import Role
+from aleph.model.common import make_token
+from aleph.logic.mail import email_role
+from aleph.logic.roles import update_role
+from aleph.logic.util import ui_url, hash_api_key
+
+# Number of days after which API keys expire
+API_KEY_EXPIRATION_DAYS = 90
+
+# Number of days before an API key expires
+API_KEY_EXPIRES_SOON_DAYS = 7
+
+log = structlog.get_logger(__name__)
+
+
+def generate_user_api_key(role):
+    event = "regenerated" if role.has_api_key else "generated"
+    params = {"role": role, "event": event}
+    plain = render_template("email/api_key_generated.txt", **params)
+    html = render_template("email/api_key_generated.html", **params)
+    subject = f"API key {event}"
+    email_role(role, subject, html=html, plain=plain)
+
+    now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
+    api_key = make_token()
+    role.api_key_digest = hash_api_key(api_key)
+    role.api_key_expires_at = now + datetime.timedelta(days=API_KEY_EXPIRATION_DAYS)
+    role.api_key_expiration_notification_sent = None
+
+    db.session.add(role)
+    db.session.commit()
+    update_role(role)
+
+    return api_key
+
+
+def send_api_key_expiration_notifications():
+    _send_api_key_expiration_notification(
+        days=7,
+        subject="Your API key will expire in 7 days",
+        plain_template="email/api_key_expires_soon.txt",
+        html_template="email/api_key_expires_soon.html",
+    )
+
+    _send_api_key_expiration_notification(
+        days=0,
+        subject="Your API key has expired",
+        plain_template="email/api_key_expired.txt",
+        html_template="email/api_key_expired.html",
+    )
+
+
+def _send_api_key_expiration_notification(
+    days,
+    subject,
+    plain_template,
+    html_template,
+):
+    now = datetime.date.today()
+    threshold = now + datetime.timedelta(days=days)
+
+    query = Role.all_users()
+    query = query.yield_per(1000)
+    query = query.where(
+        and_(
+            and_(
+                Role.api_key_digest != None,  # noqa: E711
+                func.date(Role.api_key_expires_at) <= threshold,
+            ),
+            or_(
+                Role.api_key_expiration_notification_sent == None,  # noqa: E711
+                Role.api_key_expiration_notification_sent > days,
+            ),
+        )
+    )
+
+    for role in query:
+        expires_at = role.api_key_expires_at
+        params = {
+            "role": role,
+            "expires_at": expires_at,
+            "settings_url": ui_url("settings"),
+        }
+        plain = render_template(plain_template, **params)
+        html = render_template(html_template, **params)
+        log.info(f"Sending API key expiration notification: {role} at {expires_at}")
+        email_role(role, subject, html=html, plain=plain)
+
+    query.update({Role.api_key_expiration_notification_sent: days})
+    db.session.commit()
+
+
+def reset_api_key_expiration():
+    now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
+    expires_at = now + datetime.timedelta(days=API_KEY_EXPIRATION_DAYS)
+
+    query = Role.all_users()
+    query = query.yield_per(500)
+    query = query.where(
+        and_(
+            Role.api_key_digest != None,  # noqa: E711
+            Role.api_key_expires_at == None,  # noqa: E711
+        )
+    )
+
+    query.update({Role.api_key_expires_at: expires_at})
+    db.session.commit()
+
+
+def hash_plaintext_api_keys():
+    query = Role.all_users()
+    query = query.yield_per(250)
+    query = query.where(
+        and_(
+            Role.api_key != None,  # noqa: E711
+            Role.api_key_digest == None,  # noqa: E711
+        )
+    )
+
+    results = db.session.execute(query).scalars()
+
+    for index, partition in enumerate(results.partitions()):
+        for role in partition:
+            role.api_key_digest = hash_api_key(role.api_key)
+            role.api_key = None
+            db.session.add(role)
+            log.info(f"Hashing API key: {role}")
+        log.info(f"Comitting partition {index}")
+        db.session.commit()

aleph/logic/util.py

@@ -1,4 +1,5 @@
 import jwt
+import hashlib
 from normality import ascii_text
 from urllib.parse import urlencode, urljoin
 from datetime import datetime, timedelta
@@ -58,3 +59,11 @@ def archive_token(token):
     token = jwt.decode(token, key=SETTINGS.SECRET_KEY, algorithms=DECODE, verify=True)
     expire = datetime.utcfromtimestamp(token["exp"])
     return token.get("c"), token.get("f"), token.get("m"), expire
+
+
+def hash_api_key(api_key):
+    if api_key is None:
+        return None
+
+    digest = hashlib.sha256(api_key.encode("utf-8")).hexdigest()
+    return f"sha256${digest}"

aleph/manage.py

@@ -20,6 +20,10 @@
 from aleph.queues import get_status, cancel_queue
 from aleph.queues import get_active_dataset_status
 from aleph.index.admin import delete_index
+from aleph.logic.api_keys import (
+    reset_api_key_expiration as _reset_api_key_expiration,
+    hash_plaintext_api_keys as _hash_plaintext_api_keys,
+)
 from aleph.index.entities import iter_proxies
 from aleph.index.util import AlephOperationalException
 from aleph.logic.collections import create_collection, update_collection
@@ -566,3 +570,15 @@ def evilshit():
     delete_index()
     destroy_db()
     upgrade()
+
+
+@cli.command()
+def reset_api_key_expiration():
+    """Reset the expiration date of all legacy, non-expiring API keys."""
+    _reset_api_key_expiration()
+
+
+@cli.command()
+def hash_plaintext_api_keys():
+    """Hash legacy plaintext API keys."""
+    _hash_plaintext_api_keys()

aleph/metrics/collectors.py

@@ -283,6 +283,11 @@ def _entities(self):
             collection_stats = get_collection_stats(collection.id)
             schemata = collection_stats["schema"]["values"]
 
+            # For some reason, the default/fallback value returned by `get_collection_stats`
+            # is a list, not an empty object, so we need to handle this explicitly.
+            if not schemata:
+                continue
+
             for schema, count in schemata.items():
                 stats[schema] += count
 

aleph/migrate/versions/131674bde902_add_primary_key_constraint_to_role_membership.py

@@ -0,0 +1,22 @@
+"""add primary key constraint to role_membership table
+
+Revision ID: 131674bde902
+Revises: 8adf50aadcb0
+Create Date: 2024-07-17 14:37:25.269913
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "131674bde902"
+down_revision = "8adf50aadcb0"
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_primary_key("role_membership_pkey", "role_membership", ["member_id", "group_id"])
+
+
+def downgrade():
+    pass

aleph/migrate/versions/31e24765dee3_add_api_key_digest_column.py

@@ -0,0 +1,28 @@
+"""Add api_key_digest column
+
+Revision ID: 31e24765dee3
+Revises: d46fc882ec6b
+Create Date: 2024-07-04 11:07:19.915782
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "31e24765dee3"
+down_revision = "d46fc882ec6b"
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column("role", sa.Column("api_key_digest", sa.Unicode()))
+    op.create_index(
+        index_name="ix_role_api_key_digest",
+        table_name="role",
+        columns=["api_key_digest"],
+        unique=True,
+    )
+
+
+def downgrade():
+    op.drop_column("role", "api_key_digest")

aleph/migrate/versions/d46fc882ec6b_api_key_expiration.py

@@ -0,0 +1,30 @@
+"""API key expiration
+
+Revision ID: d46fc882ec6b
+Revises: 131674bde902
+Create Date: 2024-05-02 11:43:50.993948
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d46fc882ec6b"
+down_revision = "131674bde902"
+
+
+def upgrade():
+    op.add_column("role", sa.Column("api_key_expires_at", sa.DateTime()))
+    op.add_column(
+        "role", sa.Column("api_key_expiration_notification_sent", sa.Integer())
+    )
+    op.create_index(
+        index_name="ix_role_api_key_expires_at",
+        table_name="role",
+        columns=["api_key_expires_at"],
+    )
+
+
+def downgrade():
+    op.drop_column("role", "api_key_expires_at")
+    op.drop_column("role", "api_key_expiration_notification_sent")