diff --git a/.env.example b/.env.example index fb0f7308d1..1bf6d5406f 100644 --- a/.env.example +++ b/.env.example @@ -137,3 +137,10 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. # Value should be a comma-separated list of host names. CSP_ADDITIONAL_HOSTS= + +# Time before being logged out (in seconds) +# SESSION_COOKIE_AGE=2592000 # current default: 30 days + +# Maximum allowed memory for file uploads (increase if users are having trouble +# uploading BookWyrm export files). +# DATA_UPLOAD_MAX_MEMORY_MiB=100 diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 4e7be4af37..0000000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Python Formatting (run ./bw-dev black to fix) - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - - uses: psf/black@22.12.0 - with: - version: 22.12.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 68bb05d7e5..51316ef627 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/curlylint.yaml b/.github/workflows/curlylint.yaml index 8d5c6b4f7b..10ad04ce14 100644 --- a/.github/workflows/curlylint.yaml +++ b/.github/workflows/curlylint.yaml @@ -10,7 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install curlylint run: pip install curlylint diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml deleted file mode 100644 index 78b6e142ed..0000000000 --- a/.github/workflows/django-tests.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Run Python Tests -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-20.04 - services: - postgres: - image: postgres:13 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: hunter2 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Check migrations up-to-date - run: | - python ./manage.py makemigrations --check - env: - SECRET_KEY: beepbeep - DOMAIN: your.domain.here - EMAIL_HOST: "" - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - - name: Run Tests - env: - SECRET_KEY: beepbeep - DEBUG: false - USE_HTTPS: true - DOMAIN: your.domain.here - BOOKWYRM_DATABASE_BACKEND: postgres - MEDIA_ROOT: images/ - POSTGRES_PASSWORD: hunter2 - POSTGRES_USER: postgres - POSTGRES_DB: github_actions - POSTGRES_HOST: 127.0.0.1 - CELERY_BROKER: "" - REDIS_BROKER_PORT: 6379 - REDIS_BROKER_PASSWORD: beep - USE_DUMMY_CACHE: true - FLOWER_PORT: 8888 - EMAIL_HOST: "smtp.mailgun.org" - EMAIL_PORT: 587 - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - EMAIL_USE_TLS: true - ENABLE_PREVIEW_IMAGES: false - ENABLE_THUMBNAIL_GENERATION: true - HTTP_X_FORWARDED_PROTO: false - run: | - pytest -n 3 diff --git a/.github/workflows/lint-frontend.yaml b/.github/workflows/lint-frontend.yaml index 0d0559e406..21f11ebf3f 100644 --- a/.github/workflows/lint-frontend.yaml +++ b/.github/workflows/lint-frontend.yaml @@ -19,7 +19,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install modules run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml deleted file mode 100644 index 1a641edd28..0000000000 --- a/.github/workflows/mypy.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Mypy - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Analysing the code with mypy - env: - SECRET_KEY: beepbeep - DEBUG: false - USE_HTTPS: true - DOMAIN: your.domain.here - BOOKWYRM_DATABASE_BACKEND: postgres - MEDIA_ROOT: images/ - POSTGRES_PASSWORD: hunter2 - POSTGRES_USER: postgres - POSTGRES_DB: github_actions - POSTGRES_HOST: 127.0.0.1 - CELERY_BROKER: "" - REDIS_BROKER_PORT: 6379 - REDIS_BROKER_PASSWORD: beep - USE_DUMMY_CACHE: true - FLOWER_PORT: 8888 - EMAIL_HOST: "smtp.mailgun.org" - EMAIL_PORT: 587 - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - EMAIL_USE_TLS: true - ENABLE_PREVIEW_IMAGES: false - ENABLE_THUMBNAIL_GENERATION: true - HTTP_X_FORWARDED_PROTO: false - run: | - mypy bookwyrm celerywyrm diff --git a/.github/workflows/prettier.yaml b/.github/workflows/prettier.yaml index 501516ae18..9c05c7476b 100644 --- a/.github/workflows/prettier.yaml +++ b/.github/workflows/prettier.yaml @@ -14,7 +14,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install modules run: npm install prettier@2.5.1 diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 3811c97d38..0000000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Pylint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Analysing the code with pylint - run: | - pylint bookwyrm/ - diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000000..dcbe05aee1 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,99 @@ +name: Python +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# overrides for .env.example +env: + POSTGRES_HOST: 127.0.0.1 + PGPORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + POSTGRES_DB: github_actions + SECRET_KEY: beepbeep + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" + +jobs: + pytest: + name: Tests (pytest) + runs-on: ubuntu-latest + services: + postgres: + image: postgres:13 + env: # does not inherit from jobs.build.env + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest-github-actions-annotate-failures + - name: Set up .env + run: cp .env.example .env + - name: Check migrations up-to-date + run: python ./manage.py makemigrations --check + - name: Run Tests + run: pytest -n 3 + + pylint: + name: Linting (pylint) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Analyse code with pylint + run: pylint bookwyrm/ + + mypy: + name: Typing (mypy) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Set up .env + run: cp .env.example .env + - name: Analyse code with mypy + run: mypy bookwyrm celerywyrm + + black: + name: Formatting (black; run ./bw-dev black to fix) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: psf/black@22.12.0 + with: + version: 22.12.0 diff --git a/Dockerfile b/Dockerfile index b3cd26e887..82b0c92c50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 ENV PYTHONUNBUFFERED 1 diff --git a/VERSION b/VERSION index faef31a435..7486fdbc50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.0 +0.7.2 diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index aa4b5b6877..efc9d8da2f 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) +# pylint: disable=invalid-name TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel) @@ -236,7 +237,7 @@ def serialize(self, **kwargs): omit = kwargs.get("omit", ()) data = self.__dict__.copy() # recursively serialize - for (k, v) in data.items(): + for k, v in data.items(): try: if issubclass(type(v), ActivityObject): data[k] = v.serialize() @@ -397,18 +398,14 @@ def resolve_remote_id( def get_representative(): """Get or create an actor representing the instance to sign outgoing HTTP GET requests""" - username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}" - email = "bookwyrm@localhost" - try: - user = models.User.objects.get(username=username) - except models.User.DoesNotExist: - user = models.User.objects.create_user( - username=username, - email=email, + return models.User.objects.get_or_create( + username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}", + defaults=dict( + email="bookwyrm@localhost", local=True, localname=INSTANCE_ACTOR_USERNAME, - ) - return user + ), + )[0] def get_activitypub_data(url): @@ -427,6 +424,7 @@ def get_activitypub_data(url): "Date": now, "Signature": make_signature("get", sender, url, now), }, + timeout=15, ) except requests.RequestException: raise ConnectorException() diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index 85cf444099..dfec92e4cf 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -1,5 +1,5 @@ """ actor serializer """ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Dict from .base_activity import ActivityObject @@ -35,7 +35,7 @@ class Person(ActivityObject): endpoints: Dict = None name: str = None summary: str = None - icon: Image = field(default_factory=lambda: {}) + icon: Image = None bookwyrmUser: bool = False manuallyApprovesFollowers: str = False discoverable: str = False diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index 00c9524fef..a365f4cc07 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -171,9 +171,19 @@ class Reject(Verb): type: str = "Reject" def action(self, allow_external_connections=True): - """reject a follow request""" - obj = self.object.to_model(save=False, allow_create=False) - obj.reject() + """reject a follow or follow request""" + + for model_name in ["UserFollowRequest", "UserFollows", None]: + model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None + if obj := self.object.to_model( + model=model, + save=False, + allow_create=False, + allow_external_connections=allow_external_connections, + ): + # Reject the first model that can be built. + obj.reject() + break @dataclass(init=False) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index ceb228f40c..cf48f4832e 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -43,6 +43,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, + books: Optional[QuerySet[models.Edition]] = None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -54,13 +55,15 @@ def search( # first, try searching unique identifiers # unique identifiers never have spaces, title/author usually do if not " " in query: - results = search_identifiers(query, *filters, return_first=return_first) + results = search_identifiers( + query, *filters, return_first=return_first, books=books + ) # if there were no identifier results... if not results: # then try searching title/author results = search_title_author( - query, min_confidence, *filters, return_first=return_first + query, min_confidence, *filters, return_first=return_first, books=books ) return results @@ -98,9 +101,17 @@ def format_search_result(search_result): def search_identifiers( - query, *filters, return_first=False + query, + *filters, + return_first=False, + books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: - """tries remote_id, isbn; defined as dedupe fields on the model""" + """search Editions by deduplication fields + + Best for cases when we can assume someone is searching for an exact match on + commonly unique data identifiers like isbn or specific library ids. + """ + books = books or models.Edition.objects if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? normalized_isbn = query.strip().upper().rjust(10, "0") @@ -111,7 +122,7 @@ def search_identifiers( for f in models.Edition._meta.get_fields() if hasattr(f, "deduplication_field") and f.deduplication_field ] - results = models.Edition.objects.filter( + results = books.filter( *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) ).distinct() @@ -121,12 +132,17 @@ def search_identifiers( def search_title_author( - query, min_confidence, *filters, return_first=False + query, + min_confidence, + *filters, + return_first=False, + books=None, ) -> QuerySet[models.Edition]: """searches for title and author""" + books = books or models.Edition.objects query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") results = ( - models.Edition.objects.filter(*filters, search_vector=query) + books.filter(*filters, search_vector=query) .annotate(rank=SearchRank(F("search_vector"), query)) .filter(rank__gt=min_confidence) .order_by("-rank") @@ -137,7 +153,7 @@ def search_title_author( # filter out multiple editions of the same work list_results = [] - for work_id in set(editions_of_work[:30]): + for work_id in editions_of_work[:30]: result = ( results.filter(parent_work=work_id) .order_by("-rank", "-edition_rank") diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 8b6dcb8858..aa8edbeae9 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -3,7 +3,9 @@ from abc import ABC, abstractmethod from typing import Optional, TypedDict, Any, Callable, Union, Iterator from urllib.parse import quote_plus -import imghdr + +# pylint: disable-next=deprecated-module +import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet import logging import re import asyncio diff --git a/bookwyrm/forms/books.py b/bookwyrm/forms/books.py index 4885dc0632..f73ce3f5a3 100644 --- a/bookwyrm/forms/books.py +++ b/bookwyrm/forms/books.py @@ -1,8 +1,9 @@ """ using django model forms """ from django import forms +from file_resubmit.widgets import ResubmitImageWidget + from bookwyrm import models -from bookwyrm.models.fields import ClearableFileInputWithWarning from .custom_form import CustomForm from .widgets import ArrayWidget, SelectDateWidget, Select @@ -70,9 +71,7 @@ class Meta: "published_date": SelectDateWidget( attrs={"aria-describedby": "desc_published_date"} ), - "cover": ClearableFileInputWithWarning( - attrs={"aria-describedby": "desc_cover"} - ), + "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}), "physical_format": Select( attrs={"aria-describedby": "desc_physical_format"} ), diff --git a/bookwyrm/forms/forms.py b/bookwyrm/forms/forms.py index ea60937500..3d555f308d 100644 --- a/bookwyrm/forms/forms.py +++ b/bookwyrm/forms/forms.py @@ -25,6 +25,10 @@ class ImportForm(forms.Form): csv_file = forms.FileField() +class ImportUserForm(forms.Form): + archive_file = forms.FileField() + + class ShelfForm(CustomForm): class Meta: model = models.Shelf diff --git a/bookwyrm/forms/links.py b/bookwyrm/forms/links.py index d2fd5f1162..345c5c1d4f 100644 --- a/bookwyrm/forms/links.py +++ b/bookwyrm/forms/links.py @@ -1,4 +1,5 @@ """ using django model forms """ + from urllib.parse import urlparse from django.utils.translation import gettext_lazy as _ @@ -37,10 +38,9 @@ def clean(self): ), ) if ( - not self.instance - and models.FileLink.objects.filter( - url=url, book=book, filetype=filetype - ).exists() + models.FileLink.objects.filter(url=url, book=book, filetype=filetype) + .exclude(pk=self.instance) + .exists() ): # pylint: disable=line-too-long self.add_error( diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index 6ce50f160b..8e92872f25 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,6 +1,7 @@ """ import classes """ from .importer import Importer +from .bookwyrm_import import BookwyrmImporter from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter diff --git a/bookwyrm/importers/bookwyrm_import.py b/bookwyrm/importers/bookwyrm_import.py new file mode 100644 index 0000000000..206cd62197 --- /dev/null +++ b/bookwyrm/importers/bookwyrm_import.py @@ -0,0 +1,24 @@ +"""Import data from Bookwyrm export files""" +from django.http import QueryDict + +from bookwyrm.models import User +from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob + + +class BookwyrmImporter: + """Import a Bookwyrm User export file. + This is kind of a combination of an importer and a connector. + """ + + # pylint: disable=no-self-use + def process_import( + self, user: User, archive_file: bytes, settings: QueryDict + ) -> BookwyrmImportJob: + """import user data from a Bookwyrm export file""" + + required = [k for k in settings if settings.get(k) == "on"] + + job = BookwyrmImportJob.objects.create( + user=user, archive_file=archive_file, required=required + ) + return job diff --git a/bookwyrm/isbn/isbn.py b/bookwyrm/isbn/isbn.py index 56062ff7b8..d14dc26196 100644 --- a/bookwyrm/isbn/isbn.py +++ b/bookwyrm/isbn/isbn.py @@ -26,7 +26,7 @@ class IsbnHyphenator: def update_range_message(self) -> None: """Download the range message xml file and save it locally""" - response = requests.get(self.__range_message_url) + response = requests.get(self.__range_message_url, timeout=15) with open(self.__range_file_path, "w", encoding="utf-8") as file: file.write(response.text) self.__element_tree = None diff --git a/bookwyrm/management/commands/erase_deleted_user_data.py b/bookwyrm/management/commands/erase_deleted_user_data.py new file mode 100644 index 0000000000..40c3f042b3 --- /dev/null +++ b/bookwyrm/management/commands/erase_deleted_user_data.py @@ -0,0 +1,43 @@ +""" Erase any data stored about deleted users """ +import sys +from django.core.management.base import BaseCommand, CommandError +from bookwyrm import models +from bookwyrm.models.user import erase_user_data + +# pylint: disable=missing-function-docstring +class Command(BaseCommand): + """command-line options""" + + help = "Remove Two Factor Authorisation from user" + + def add_arguments(self, parser): # pylint: disable=no-self-use + parser.add_argument( + "--dryrun", + action="store_true", + help="Preview users to be cleared without altering the database", + ) + + def handle(self, *args, **options): # pylint: disable=unused-argument + + # Check for anything fishy + bad_state = models.User.objects.filter(is_deleted=True, is_active=True) + if bad_state.exists(): + raise CommandError( + f"{bad_state.count()} user(s) marked as both active and deleted" + ) + + deleted_users = models.User.objects.filter(is_deleted=True) + self.stdout.write(f"Found {deleted_users.count()} deleted users") + if options["dryrun"]: + self.stdout.write("\n".join(u.username for u in deleted_users[:5])) + if deleted_users.count() > 5: + self.stdout.write("... and more") + sys.exit() + + self.stdout.write("Erasing user data:") + for user_id in deleted_users.values_list("id", flat=True): + erase_user_data.delay(user_id) + self.stdout.write(".", ending="") + + self.stdout.write("") + self.stdout.write("Tasks created successfully") diff --git a/bookwyrm/management/commands/instance_version.py b/bookwyrm/management/commands/instance_version.py deleted file mode 100644 index ca150d6409..0000000000 --- a/bookwyrm/management/commands/instance_version.py +++ /dev/null @@ -1,54 +0,0 @@ -""" Get your admin code to allow install """ -from django.core.management.base import BaseCommand - -from bookwyrm import models -from bookwyrm.settings import VERSION - - -# pylint: disable=no-self-use -class Command(BaseCommand): - """command-line options""" - - help = "What version is this?" - - def add_arguments(self, parser): - """specify which function to run""" - parser.add_argument( - "--current", - action="store_true", - help="Version stored in database", - ) - parser.add_argument( - "--target", - action="store_true", - help="Version stored in settings", - ) - parser.add_argument( - "--update", - action="store_true", - help="Update database version", - ) - - # pylint: disable=unused-argument - def handle(self, *args, **options): - """execute init""" - site = models.SiteSettings.objects.get() - current = site.version or "0.0.1" - target = VERSION - if options.get("current"): - print(current) - return - - if options.get("target"): - print(target) - return - - if options.get("update"): - site.version = target - site.save() - return - - if current != target: - print(f"{current}/{target}") - else: - print(current) diff --git a/bookwyrm/middleware/__init__.py b/bookwyrm/middleware/__init__.py index 03843c5a30..85c3a56fe8 100644 --- a/bookwyrm/middleware/__init__.py +++ b/bookwyrm/middleware/__init__.py @@ -1,3 +1,4 @@ """ look at all this nice middleware! """ from .timezone_middleware import TimezoneMiddleware from .ip_middleware import IPBlocklistMiddleware +from .file_too_big import FileTooBig diff --git a/bookwyrm/middleware/file_too_big.py b/bookwyrm/middleware/file_too_big.py new file mode 100644 index 0000000000..de1349d96a --- /dev/null +++ b/bookwyrm/middleware/file_too_big.py @@ -0,0 +1,30 @@ +"""Middleware to display a custom 413 error page""" + +from django.http import HttpResponse +from django.shortcuts import render +from django.core.exceptions import RequestDataTooBig + + +class FileTooBig: + """Middleware to display a custom page when a + RequestDataTooBig exception is thrown""" + + def __init__(self, get_response): + """boilerplate __init__ from Django docs""" + + self.get_response = get_response + + def __call__(self, request): + """If RequestDataTooBig is thrown, render the 413 error page""" + + try: + body = request.body # pylint: disable=unused-variable + + except RequestDataTooBig: + + rendered = render(request, "413.html") + response = HttpResponse(rendered) + return response + + response = self.get_response(request) + return response diff --git a/bookwyrm/migrations/0184_auto_20231106_0421.py b/bookwyrm/migrations/0184_auto_20231106_0421.py index e8197dea13..23bacc5026 100644 --- a/bookwyrm/migrations/0184_auto_20231106_0421.py +++ b/bookwyrm/migrations/0184_auto_20231106_0421.py @@ -22,17 +22,6 @@ def update_deleted_users(apps, schema_editor): ).update(is_deleted=True) -def erase_deleted_user_data(apps, schema_editor): - """Retroactively clear user data""" - for user in User.objects.filter(is_deleted=True): - user.erase_user_data() - user.save( - broadcast=False, - update_fields=["email", "avatar", "preview_image", "summary", "name"], - ) - user.erase_user_statuses(broadcast=False) - - class Migration(migrations.Migration): dependencies = [ @@ -43,7 +32,4 @@ class Migration(migrations.Migration): migrations.RunPython( update_deleted_users, reverse_code=migrations.RunPython.noop ), - migrations.RunPython( - erase_deleted_user_data, reverse_code=migrations.RunPython.noop - ), ] diff --git a/bookwyrm/migrations/0186_auto_20231116_0048.py b/bookwyrm/migrations/0186_auto_20231116_0048.py new file mode 100644 index 0000000000..e3b9da4fed --- /dev/null +++ b/bookwyrm/migrations/0186_auto_20231116_0048.py @@ -0,0 +1,212 @@ +# Generated by Django 3.2.20 on 2023-11-16 00:48 + +from django.conf import settings +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0185_alter_notification_notification_type"), + ] + + operations = [ + migrations.CreateModel( + name="ParentJob", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("task_id", models.UUIDField(blank=True, null=True, unique=True)), + ( + "created_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "updated_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("complete", models.BooleanField(default=False)), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="sitesettings", + name="user_import_time_limit", + field=models.IntegerField(default=48), + ), + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("BOOST", "Boost"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ("MOVE", "Move"), + ], + max_length=255, + ), + ), + migrations.CreateModel( + name="BookwyrmExportJob", + fields=[ + ( + "parentjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.parentjob", + ), + ), + ("export_data", models.FileField(null=True, upload_to="")), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.parentjob",), + ), + migrations.CreateModel( + name="BookwyrmImportJob", + fields=[ + ( + "parentjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.parentjob", + ), + ), + ("archive_file", models.FileField(blank=True, null=True, upload_to="")), + ("import_data", models.JSONField(null=True)), + ( + "required", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=50), + blank=True, + size=None, + ), + ), + ], + options={ + "abstract": False, + }, + bases=("bookwyrm.parentjob",), + ), + migrations.CreateModel( + name="ChildJob", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("task_id", models.UUIDField(blank=True, null=True, unique=True)), + ( + "created_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "updated_date", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("complete", models.BooleanField(default=False)), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ( + "parent_job", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="child_jobs", + to="bookwyrm.parentjob", + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="notification", + name="related_user_export", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.bookwyrmexportjob", + ), + ), + ] diff --git a/bookwyrm/migrations/0189_alter_user_preferred_language.py b/bookwyrm/migrations/0189_alter_user_preferred_language.py new file mode 100644 index 0000000000..d9d9777c7c --- /dev/null +++ b/bookwyrm/migrations/0189_alter_user_preferred_language.py @@ -0,0 +1,45 @@ +# Generated by Django 3.2.23 on 2023-12-12 23:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_language", + field=models.CharField( + blank=True, + choices=[ + ("en-us", "English"), + ("ca-es", "Català (Catalan)"), + ("de-de", "Deutsch (German)"), + ("eo-uy", "Esperanto (Esperanto)"), + ("es-es", "Español (Spanish)"), + ("eu-es", "Euskara (Basque)"), + ("gl-es", "Galego (Galician)"), + ("it-it", "Italiano (Italian)"), + ("fi-fi", "Suomi (Finnish)"), + ("fr-fr", "Français (French)"), + ("lt-lt", "Lietuvių (Lithuanian)"), + ("nl-nl", "Nederlands (Dutch)"), + ("no-no", "Norsk (Norwegian)"), + ("pl-pl", "Polski (Polish)"), + ("pt-br", "Português do Brasil (Brazilian Portuguese)"), + ("pt-pt", "Português Europeu (European Portuguese)"), + ("ro-ro", "Română (Romanian)"), + ("sv-se", "Svenska (Swedish)"), + ("uk-ua", "Українська (Ukrainian)"), + ("zh-hans", "简体中文 (Simplified Chinese)"), + ("zh-hant", "繁體中文 (Traditional Chinese)"), + ], + max_length=255, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py b/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py new file mode 100644 index 0000000000..eb6238f6e4 --- /dev/null +++ b/bookwyrm/migrations/0189_merge_0186_auto_20231116_0048_0188_theme_loads.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2023-11-22 10:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0186_auto_20231116_0048"), + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0190_alter_notification_notification_type.py b/bookwyrm/migrations/0190_alter_notification_notification_type.py new file mode 100644 index 0000000000..aff54c77b2 --- /dev/null +++ b/bookwyrm/migrations/0190_alter_notification_notification_type.py @@ -0,0 +1,45 @@ +# Generated by Django 3.2.23 on 2023-11-23 19:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_merge_0186_auto_20231116_0048_0188_theme_loads"), + ] + + operations = [ + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("BOOST", "Boost"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE_REQUEST", "Invite Request"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ("MOVE", "Move"), + ], + max_length=255, + ), + ), + ] diff --git a/bookwyrm/migrations/0191_merge_20240102_0326.py b/bookwyrm/migrations/0191_merge_20240102_0326.py new file mode 100644 index 0000000000..485c14af83 --- /dev/null +++ b/bookwyrm/migrations/0191_merge_20240102_0326.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-01-02 03:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0189_alter_user_preferred_language"), + ("bookwyrm", "0190_alter_notification_notification_type"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0192_make_page_positions_text.py b/bookwyrm/migrations/0192_make_page_positions_text.py new file mode 100644 index 0000000000..940a9e9413 --- /dev/null +++ b/bookwyrm/migrations/0192_make_page_positions_text.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2024-01-04 23:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.AlterField( + model_name="quotation", + name="endposition", + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name="quotation", + name="position", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py new file mode 100644 index 0000000000..db67b4e925 --- /dev/null +++ b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-02 19:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.RenameField( + model_name="sitesettings", + old_name="version", + new_name="available_version", + ), + ] diff --git a/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py new file mode 100644 index 0000000000..ec5b411e2f --- /dev/null +++ b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-16 10:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.AddField( + model_name="sitesettings", + name="user_exports_enabled", + field=models.BooleanField(default=False), + ), + ] diff --git a/bookwyrm/migrations/0193_merge_20240203_1539.py b/bookwyrm/migrations/0193_merge_20240203_1539.py new file mode 100644 index 0000000000..a88568ba1f --- /dev/null +++ b/bookwyrm/migrations/0193_merge_20240203_1539.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-02-03 15:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0192_make_page_positions_text"), + ("bookwyrm", "0192_sitesettings_user_exports_enabled"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0194_merge_20240203_1619.py b/bookwyrm/migrations/0194_merge_20240203_1619.py new file mode 100644 index 0000000000..a5c18e3006 --- /dev/null +++ b/bookwyrm/migrations/0194_merge_20240203_1619.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-02-03 16:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0192_rename_version_sitesettings_available_version"), + ("bookwyrm", "0193_merge_20240203_1539"), + ] + + operations = [] diff --git a/bookwyrm/migrations/0195_alter_user_preferred_language.py b/bookwyrm/migrations/0195_alter_user_preferred_language.py new file mode 100644 index 0000000000..1fbfa7304b --- /dev/null +++ b/bookwyrm/migrations/0195_alter_user_preferred_language.py @@ -0,0 +1,46 @@ +# Generated by Django 3.2.23 on 2024-02-21 00:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0194_merge_20240203_1619"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_language", + field=models.CharField( + blank=True, + choices=[ + ("en-us", "English"), + ("ca-es", "Català (Catalan)"), + ("de-de", "Deutsch (German)"), + ("eo-uy", "Esperanto (Esperanto)"), + ("es-es", "Español (Spanish)"), + ("eu-es", "Euskara (Basque)"), + ("gl-es", "Galego (Galician)"), + ("it-it", "Italiano (Italian)"), + ("ko-kr", "한국어 (Korean)"), + ("fi-fi", "Suomi (Finnish)"), + ("fr-fr", "Français (French)"), + ("lt-lt", "Lietuvių (Lithuanian)"), + ("nl-nl", "Nederlands (Dutch)"), + ("no-no", "Norsk (Norwegian)"), + ("pl-pl", "Polski (Polish)"), + ("pt-br", "Português do Brasil (Brazilian Portuguese)"), + ("pt-pt", "Português Europeu (European Portuguese)"), + ("ro-ro", "Română (Romanian)"), + ("sv-se", "Svenska (Swedish)"), + ("uk-ua", "Українська (Ukrainian)"), + ("zh-hans", "简体中文 (Simplified Chinese)"), + ("zh-hant", "繁體中文 (Traditional Chinese)"), + ], + max_length=255, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0196_merge_pr3134_into_main.py b/bookwyrm/migrations/0196_merge_pr3134_into_main.py new file mode 100644 index 0000000000..4967c18f26 --- /dev/null +++ b/bookwyrm/migrations/0196_merge_pr3134_into_main.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.23 on 2024-03-18 00:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0191_migrate_search_vec_triggers_to_pgtriggers'), + ('bookwyrm', '0195_alter_user_preferred_language'), + ] + + operations = [ + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index e1c862f2b8..6bb99c7f25 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -26,6 +26,8 @@ from .group import Group, GroupMember, GroupMemberInvitation from .import_job import ImportJob, ImportItem +from .bookwyrm_import_job import BookwyrmImportJob +from .bookwyrm_export_job import BookwyrmExportJob from .move import MoveUser diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 36317ad4ed..db737b8bc2 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -152,8 +152,9 @@ def get_recipients(self, software=None) -> list[str]: # find anyone who's tagged in a status, for example mentions = self.recipients if hasattr(self, "recipients") else [] - # we always send activities to explicitly mentioned users' inboxes - recipients = [u.inbox for u in mentions or [] if not u.local] + # we always send activities to explicitly mentioned users (using shared inboxes + # where available to avoid duplicate submissions to a given instance) + recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local} # unless it's a dm, all the followers should receive the activity if privacy != "direct": @@ -173,18 +174,18 @@ def get_recipients(self, software=None) -> list[str]: if user: queryset = queryset.filter(following=user) - # ideally, we will send to shared inboxes for efficiency - shared_inboxes = ( - queryset.filter(shared_inbox__isnull=False) - .values_list("shared_inbox", flat=True) - .distinct() + # as above, we prefer shared inboxes if available + recipients.update( + queryset.filter(shared_inbox__isnull=False).values_list( + "shared_inbox", flat=True + ) ) - # but not everyone has a shared inbox - inboxes = queryset.filter(shared_inbox__isnull=True).values_list( - "inbox", flat=True + recipients.update( + queryset.filter(shared_inbox__isnull=True).values_list( + "inbox", flat=True + ) ) - recipients += list(shared_inboxes) + list(inboxes) - return list(set(recipients)) + return list(recipients) def to_activity_dataclass(self): """convert from a model to an activity""" @@ -602,7 +603,7 @@ def to_ordered_collection_page( if activity_page.has_next(): next_page = f"{remote_id}?page={activity_page.next_page_number()}" if activity_page.has_previous(): - prev_page = f"{remote_id}?page=%d{activity_page.previous_page_number()}" + prev_page = f"{remote_id}?page={activity_page.previous_page_number()}" return activitypub.OrderedCollectionPage( id=f"{remote_id}?page={page}", partOf=remote_id, diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py new file mode 100644 index 0000000000..2f32cbd29a --- /dev/null +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -0,0 +1,229 @@ +"""Export user account to tar.gz file for import into another Bookwyrm instance""" + +import dataclasses +import logging +from uuid import uuid4 + +from django.db.models import FileField +from django.db.models import Q +from django.core.serializers.json import DjangoJSONEncoder +from django.core.files.base import ContentFile + +from bookwyrm.models import AnnualGoal, ReadThrough, ShelfBook, List, ListItem +from bookwyrm.models import Review, Comment, Quotation +from bookwyrm.models import Edition +from bookwyrm.models import UserFollows, User, UserBlocks +from bookwyrm.models.job import ParentJob, ParentTask +from bookwyrm.tasks import app, IMPORTS +from bookwyrm.utils.tar import BookwyrmTarFile + +logger = logging.getLogger(__name__) + + +class BookwyrmExportJob(ParentJob): + """entry for a specific request to export a bookwyrm user""" + + export_data = FileField(null=True) + + def start_job(self): + """Start the job""" + start_export_task.delay(job_id=self.id, no_children=True) + + return self + + +@app.task(queue=IMPORTS, base=ParentTask) +def start_export_task(**kwargs): + """trigger the child tasks for each row""" + job = BookwyrmExportJob.objects.get(id=kwargs["job_id"]) + + # don't start the job if it was stopped from the UI + if job.complete: + return + try: + # This is where ChildJobs get made + job.export_data = ContentFile(b"", str(uuid4())) + json_data = json_export(job.user) + tar_export(json_data, job.user, job.export_data) + job.save(update_fields=["export_data"]) + except Exception as err: # pylint: disable=broad-except + logger.exception("User Export Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + + job.set_status("complete") + + +def tar_export(json_data: str, user, file): + """wrap the export information in a tar file""" + file.open("wb") + with BookwyrmTarFile.open(mode="w:gz", fileobj=file) as tar: + tar.write_bytes(json_data.encode("utf-8")) + + # Add avatar image if present + if getattr(user, "avatar", False): + tar.add_image(user.avatar, filename="avatar") + + editions = get_books_for_user(user) + for book in editions: + if getattr(book, "cover", False): + tar.add_image(book.cover) + + file.close() + + +def json_export( + user, +): # pylint: disable=too-many-locals, too-many-statements, too-many-branches + """Generate an export for a user""" + + # User as AP object + exported_user = user.to_activity() + # I don't love this but it prevents a JSON encoding error + # when there is no user image + if exported_user.get("icon") in (None, dataclasses.MISSING): + exported_user["icon"] = {} + else: + # change the URL to be relative to the JSON file + file_type = exported_user["icon"]["url"].rsplit(".", maxsplit=1)[-1] + filename = f"avatar.{file_type}" + exported_user["icon"]["url"] = filename + + # Additional settings - can't be serialized as AP + vals = [ + "show_goal", + "preferred_timezone", + "default_post_privacy", + "show_suggested_users", + ] + exported_user["settings"] = {} + for k in vals: + exported_user["settings"][k] = getattr(user, k) + + # Reading goals - can't be serialized as AP + reading_goals = AnnualGoal.objects.filter(user=user).distinct() + exported_user["goals"] = [] + for goal in reading_goals: + exported_user["goals"].append( + {"goal": goal.goal, "year": goal.year, "privacy": goal.privacy} + ) + + # Reading history - can't be serialized as AP + readthroughs = ReadThrough.objects.filter(user=user).distinct().values() + readthroughs = list(readthroughs) + + # Books + editions = get_books_for_user(user) + exported_user["books"] = [] + + for edition in editions: + book = {} + book["work"] = edition.parent_work.to_activity() + book["edition"] = edition.to_activity() + + if book["edition"].get("cover"): + # change the URL to be relative to the JSON file + filename = book["edition"]["cover"]["url"].rsplit("/", maxsplit=1)[-1] + book["edition"]["cover"]["url"] = f"covers/{filename}" + + # authors + book["authors"] = [] + for author in edition.authors.all(): + book["authors"].append(author.to_activity()) + + # Shelves this book is on + # Every ShelfItem is this book so we don't other serializing + book["shelves"] = [] + shelf_books = ( + ShelfBook.objects.select_related("shelf") + .filter(user=user, book=edition) + .distinct() + ) + + for shelfbook in shelf_books: + book["shelves"].append(shelfbook.shelf.to_activity()) + + # Lists and ListItems + # ListItems include "notes" and "approved" so we need them + # even though we know it's this book + book["lists"] = [] + list_items = ListItem.objects.filter(book=edition, user=user).distinct() + + for item in list_items: + list_info = item.book_list.to_activity() + list_info[ + "privacy" + ] = item.book_list.privacy # this isn't serialized so we add it + list_info["list_item"] = item.to_activity() + book["lists"].append(list_info) + + # Statuses + # Can't use select_subclasses here because + # we need to filter on the "book" value, + # which is not available on an ordinary Status + for status in ["comments", "quotations", "reviews"]: + book[status] = [] + + comments = Comment.objects.filter(user=user, book=edition).all() + for status in comments: + obj = status.to_activity() + obj["progress"] = status.progress + obj["progress_mode"] = status.progress_mode + book["comments"].append(obj) + + quotes = Quotation.objects.filter(user=user, book=edition).all() + for status in quotes: + obj = status.to_activity() + obj["position"] = status.position + obj["endposition"] = status.endposition + obj["position_mode"] = status.position_mode + book["quotations"].append(obj) + + reviews = Review.objects.filter(user=user, book=edition).all() + for status in reviews: + obj = status.to_activity() + book["reviews"].append(obj) + + # readthroughs can't be serialized to activity + book_readthroughs = ( + ReadThrough.objects.filter(user=user, book=edition).distinct().values() + ) + book["readthroughs"] = list(book_readthroughs) + + # append everything + exported_user["books"].append(book) + + # saved book lists - just the remote id + saved_lists = List.objects.filter(id__in=user.saved_lists.all()).distinct() + exported_user["saved_lists"] = [l.remote_id for l in saved_lists] + + # follows - just the remote id + follows = UserFollows.objects.filter(user_subject=user).distinct() + following = User.objects.filter(userfollows_user_object__in=follows).distinct() + exported_user["follows"] = [f.remote_id for f in following] + + # blocks - just the remote id + blocks = UserBlocks.objects.filter(user_subject=user).distinct() + blocking = User.objects.filter(userblocks_user_object__in=blocks).distinct() + + exported_user["blocks"] = [b.remote_id for b in blocking] + + return DjangoJSONEncoder().encode(exported_user) + + +def get_books_for_user(user): + """Get all the books and editions related to a user""" + + editions = ( + Edition.objects.select_related("parent_work") + .filter( + Q(shelves__user=user) + | Q(readthrough__user=user) + | Q(review__user=user) + | Q(list__user=user) + | Q(comment__user=user) + | Q(quotation__user=user) + ) + .distinct() + ) + + return editions diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py new file mode 100644 index 0000000000..9a11fd932b --- /dev/null +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -0,0 +1,459 @@ +"""Import a user from another Bookwyrm instance""" + +import json +import logging + +from django.db.models import FileField, JSONField, CharField +from django.utils import timezone +from django.utils.html import strip_tags +from django.contrib.postgres.fields import ArrayField as DjangoArrayField + +from bookwyrm import activitypub +from bookwyrm import models +from bookwyrm.tasks import app, IMPORTS +from bookwyrm.models.job import ParentJob, ParentTask, SubTask +from bookwyrm.utils.tar import BookwyrmTarFile + +logger = logging.getLogger(__name__) + + +class BookwyrmImportJob(ParentJob): + """entry for a specific request for importing a bookwyrm user backup""" + + archive_file = FileField(null=True, blank=True) + import_data = JSONField(null=True) + required = DjangoArrayField(CharField(max_length=50, blank=True), blank=True) + + def start_job(self): + """Start the job""" + start_import_task.delay(job_id=self.id, no_children=True) + + +@app.task(queue=IMPORTS, base=ParentTask) +def start_import_task(**kwargs): + """trigger the child import tasks for each user data""" + job = BookwyrmImportJob.objects.get(id=kwargs["job_id"]) + archive_file = job.archive_file + + # don't start the job if it was stopped from the UI + if job.complete: + return + + try: + archive_file.open("rb") + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + + if "include_user_profile" in job.required: + update_user_profile(job.user, tar, job.import_data) + if "include_user_settings" in job.required: + update_user_settings(job.user, job.import_data) + if "include_goals" in job.required: + update_goals(job.user, job.import_data.get("goals")) + if "include_saved_lists" in job.required: + upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + if "include_follows" in job.required: + upsert_follows(job.user, job.import_data.get("follows")) + if "include_blocks" in job.required: + upsert_user_blocks(job.user, job.import_data.get("blocks")) + + process_books(job, tar) + + job.set_status("complete") + archive_file.close() + + except Exception as err: # pylint: disable=broad-except + logger.exception("User Import Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + + +def process_books(job, tar): + """ + Process user import data related to books + We always import the books even if not assigning + them to shelves, lists etc + """ + + books = job.import_data.get("books") + + for data in books: + book = get_or_create_edition(data, tar) + + if "include_shelves" in job.required: + upsert_shelves(book, job.user, data) + + if "include_readthroughs" in job.required: + upsert_readthroughs(data.get("readthroughs"), job.user, book.id) + + if "include_comments" in job.required: + upsert_statuses( + job.user, models.Comment, data.get("comments"), book.remote_id + ) + if "include_quotations" in job.required: + upsert_statuses( + job.user, models.Quotation, data.get("quotations"), book.remote_id + ) + + if "include_reviews" in job.required: + upsert_statuses( + job.user, models.Review, data.get("reviews"), book.remote_id + ) + + if "include_lists" in job.required: + upsert_lists(job.user, data.get("lists"), book.id) + + +def get_or_create_edition(book_data, tar): + """Take a JSON string of work and edition data, + find or create the edition and work in the database and + return an edition instance""" + + edition = book_data.get("edition") + existing = models.Edition.find_existing(edition) + if existing: + return existing + + # make sure we have the authors in the local DB + # replace the old author ids in the edition JSON + edition["authors"] = [] + for author in book_data.get("authors"): + parsed_author = activitypub.parse(author) + instance = parsed_author.to_model( + model=models.Author, save=True, overwrite=True + ) + + edition["authors"].append(instance.remote_id) + + # we will add the cover later from the tar + # don't try to load it from the old server + cover = edition.get("cover", {}) + cover_path = cover.get("url", None) + edition["cover"] = {} + + # first we need the parent work to exist + work = book_data.get("work") + work["editions"] = [] + parsed_work = activitypub.parse(work) + work_instance = parsed_work.to_model(model=models.Work, save=True, overwrite=True) + + # now we have a work we can add it to the edition + # and create the edition model instance + edition["work"] = work_instance.remote_id + parsed_edition = activitypub.parse(edition) + book = parsed_edition.to_model(model=models.Edition, save=True, overwrite=True) + + # set the cover image from the tar + if cover_path: + tar.write_image_to_file(cover_path, book.cover) + + return book + + +def upsert_readthroughs(data, user, book_id): + """Take a JSON string of readthroughs and + find or create the instances in the database""" + + for read_through in data: + + obj = {} + keys = [ + "progress_mode", + "start_date", + "finish_date", + "stopped_date", + "is_active", + ] + for key in keys: + obj[key] = read_through[key] + obj["user_id"] = user.id + obj["book_id"] = book_id + + existing = models.ReadThrough.objects.filter(**obj).first() + if not existing: + models.ReadThrough.objects.create(**obj) + + +def upsert_statuses(user, cls, data, book_remote_id): + """Take a JSON string of a status and + find or create the instances in the database""" + + for status in data: + if is_alias( + user, status["attributedTo"] + ): # don't let l33t hax0rs steal other people's posts + # update ids and remove replies + status["attributedTo"] = user.remote_id + status["to"] = update_followers_address(user, status["to"]) + status["cc"] = update_followers_address(user, status["cc"]) + status[ + "replies" + ] = ( + {} + ) # this parses incorrectly but we can't set it without knowing the new id + status["inReplyToBook"] = book_remote_id + parsed = activitypub.parse(status) + if not status_already_exists( + user, parsed + ): # don't duplicate posts on multiple import + + instance = parsed.to_model(model=cls, save=True, overwrite=True) + + for val in [ + "progress", + "progress_mode", + "position", + "endposition", + "position_mode", + ]: + if status.get(val): + instance.val = status[val] + + instance.remote_id = instance.get_remote_id() # update the remote_id + instance.save() # save and broadcast + + else: + logger.info("User does not have permission to import statuses") + + +def upsert_lists(user, lists, book_id): + """Take a list of objects each containing + a list and list item as AP objects + + Because we are creating new IDs we can't assume the id + will exist or be accurate, so we only use to_model for + adding new items after checking whether they exist . + + """ + + book = models.Edition.objects.get(id=book_id) + + for blist in lists: + booklist = models.List.objects.filter(name=blist["name"], user=user).first() + if not booklist: + + blist["owner"] = user.remote_id + parsed = activitypub.parse(blist) + booklist = parsed.to_model(model=models.List, save=True, overwrite=True) + + booklist.privacy = blist["privacy"] + booklist.save() + + item = models.ListItem.objects.filter(book=book, book_list=booklist).exists() + if not item: + count = booklist.books.count() + models.ListItem.objects.create( + book=book, + book_list=booklist, + user=user, + notes=blist["list_item"]["notes"], + approved=blist["list_item"]["approved"], + order=count + 1, + ) + + +def upsert_shelves(book, user, book_data): + """Take shelf JSON objects and create + DB entries if they don't already exist""" + + shelves = book_data["shelves"] + for shelf in shelves: + + book_shelf = models.Shelf.objects.filter(name=shelf["name"], user=user).first() + + if not book_shelf: + book_shelf = models.Shelf.objects.create(name=shelf["name"], user=user) + + # add the book as a ShelfBook if needed + if not models.ShelfBook.objects.filter( + book=book, shelf=book_shelf, user=user + ).exists(): + models.ShelfBook.objects.create( + book=book, shelf=book_shelf, user=user, shelved_date=timezone.now() + ) + + +def update_user_profile(user, tar, data): + """update the user's profile from import data""" + name = data.get("name", None) + username = data.get("preferredUsername") + user.name = name if name else username + user.summary = strip_tags(data.get("summary", None)) + user.save(update_fields=["name", "summary"]) + if data["icon"].get("url"): + avatar_filename = next(filter(lambda n: n.startswith("avatar"), tar.getnames())) + tar.write_image_to_file(avatar_filename, user.avatar) + + +def update_user_settings(user, data): + """update the user's settings from import data""" + + update_fields = ["manually_approves_followers", "hide_follows", "discoverable"] + + ap_fields = [ + ("manuallyApprovesFollowers", "manually_approves_followers"), + ("hideFollows", "hide_follows"), + ("discoverable", "discoverable"), + ] + + for (ap_field, bw_field) in ap_fields: + setattr(user, bw_field, data[ap_field]) + + bw_fields = [ + "show_goal", + "show_suggested_users", + "default_post_privacy", + "preferred_timezone", + ] + + for field in bw_fields: + update_fields.append(field) + setattr(user, field, data["settings"][field]) + + user.save(update_fields=update_fields) + + +@app.task(queue=IMPORTS, base=SubTask) +def update_user_settings_task(job_id): + """wrapper task for user's settings import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return update_user_settings(parent_job.user, parent_job.import_data.get("user")) + + +def update_goals(user, data): + """update the user's goals from import data""" + + for goal in data: + # edit the existing goal if there is one + existing = models.AnnualGoal.objects.filter( + year=goal["year"], user=user + ).first() + if existing: + for k in goal.keys(): + setattr(existing, k, goal[k]) + existing.save() + else: + goal["user"] = user + models.AnnualGoal.objects.create(**goal) + + +@app.task(queue=IMPORTS, base=SubTask) +def update_goals_task(job_id): + """wrapper task for user's goals import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return update_goals(parent_job.user, parent_job.import_data.get("goals")) + + +def upsert_saved_lists(user, values): + """Take a list of remote ids and add as saved lists""" + + for remote_id in values: + book_list = activitypub.resolve_remote_id(remote_id, models.List) + if book_list: + user.saved_lists.add(book_list) + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_saved_lists_task(job_id): + """wrapper task for user's saved lists import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_saved_lists( + parent_job.user, parent_job.import_data.get("saved_lists") + ) + + +def upsert_follows(user, values): + """Take a list of remote ids and add as follows""" + + for remote_id in values: + followee = activitypub.resolve_remote_id(remote_id, models.User) + if followee: + (follow_request, created,) = models.UserFollowRequest.objects.get_or_create( + user_subject=user, + user_object=followee, + ) + + if not created: + # this request probably failed to connect with the remote + # and should save to trigger a re-broadcast + follow_request.save() + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_follows_task(job_id): + """wrapper task for user's follows import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_follows(parent_job.user, parent_job.import_data.get("follows")) + + +def upsert_user_blocks(user, user_ids): + """block users""" + + for user_id in user_ids: + user_object = activitypub.resolve_remote_id(user_id, models.User) + if user_object: + exists = models.UserBlocks.objects.filter( + user_subject=user, user_object=user_object + ).exists() + if not exists: + models.UserBlocks.objects.create( + user_subject=user, user_object=user_object + ) + # remove the blocked users's lists from the groups + models.List.remove_from_group(user, user_object) + # remove the blocked user from all blocker's owned groups + models.GroupMember.remove(user, user_object) + + +@app.task(queue=IMPORTS, base=SubTask) +def upsert_user_blocks_task(job_id): + """wrapper task for user's blocks import""" + parent_job = BookwyrmImportJob.objects.get(id=job_id) + + return upsert_user_blocks( + parent_job.user, parent_job.import_data.get("blocked_users") + ) + + +def update_followers_address(user, field): + """statuses to or cc followers need to have the followers + address updated to the new local user""" + + for i, audience in enumerate(field): + if audience.rsplit("/")[-1] == "followers": + field[i] = user.followers_url + + return field + + +def is_alias(user, remote_id): + """check that the user is listed as movedTo or also_known_as + in the remote user's profile""" + + remote_user = activitypub.resolve_remote_id( + remote_id=remote_id, model=models.User, save=False + ) + + if remote_user: + + if remote_user.moved_to: + return user.remote_id == remote_user.moved_to + + if remote_user.also_known_as: + return user in remote_user.also_known_as.all() + + return False + + +def status_already_exists(user, status): + """check whether this status has already been published + by this user. We can't rely on to_model() because it + only matches on remote_id, which we have to change + *after* saving because it needs the primary key (id)""" + + return models.Status.objects.filter( + user=user, content=status.content, published_date=status.published + ).exists() diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 4bd5807059..55ec254669 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -482,7 +482,7 @@ def field_to_activity(self, value, alt=None): if not url: return None - return activitypub.Document(url=url, name=alt) + return activitypub.Image(url=url, name=alt) def field_from_activity(self, value, allow_external_connections=True): image_slug = value diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py new file mode 100644 index 0000000000..4f5cb20935 --- /dev/null +++ b/bookwyrm/models/job.py @@ -0,0 +1,308 @@ +"""Everything needed for Celery to multi-thread complex tasks.""" + +from django.db import models +from django.db import transaction +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from bookwyrm.models.user import User + +from bookwyrm.tasks import app + + +class Job(models.Model): + """Abstract model to store the state of a Task.""" + + class Status(models.TextChoices): + """Possible job states.""" + + PENDING = "pending", _("Pending") + ACTIVE = "active", _("Active") + COMPLETE = "complete", _("Complete") + STOPPED = "stopped", _("Stopped") + FAILED = "failed", _("Failed") + + task_id = models.UUIDField(unique=True, null=True, blank=True) + + created_date = models.DateTimeField(default=timezone.now) + updated_date = models.DateTimeField(default=timezone.now) + complete = models.BooleanField(default=False) + status = models.CharField( + max_length=50, choices=Status.choices, default=Status.PENDING, null=True + ) + + class Meta: + """Make it abstract""" + + abstract = True + + def complete_job(self): + """Report that the job has completed""" + if self.complete: + return + + self.status = self.Status.COMPLETE + self.complete = True + self.updated_date = timezone.now() + + self.save(update_fields=["status", "complete", "updated_date"]) + + def stop_job(self, reason=None): + """Stop the job""" + if self.complete: + return + + self.__terminate_job() + + if reason and reason == "failed": + self.status = self.Status.FAILED + else: + self.status = self.Status.STOPPED + self.complete = True + self.updated_date = timezone.now() + + self.save(update_fields=["status", "complete", "updated_date"]) + + def set_status(self, status): + """Set job status""" + if self.complete: + return + + if self.status == status: + return + + if status == self.Status.COMPLETE: + self.complete_job() + return + + if status == self.Status.STOPPED: + self.stop_job() + return + + if status == self.Status.FAILED: + self.stop_job(reason="failed") + return + + self.updated_date = timezone.now() + self.status = status + + self.save(update_fields=["status", "updated_date"]) + + def __terminate_job(self): + """Tell workers to ignore and not execute this task.""" + app.control.revoke(self.task_id, terminate=True) + + +class ParentJob(Job): + """Store the state of a Task which can spawn many :model:`ChildJob`s to spread + resource load. + + Intended to be sub-classed if necessary via proxy or + multi-table inheritance. + Extends :model:`Job`. + """ + + user = models.ForeignKey(User, on_delete=models.CASCADE) + + def complete_job(self): + """Report that the job has completed and stop pending + children. Extend. + """ + super().complete_job() + self.__terminate_pending_child_jobs() + + def notify_child_job_complete(self): + """let the job know when the items get work done""" + if self.complete: + return + + self.updated_date = timezone.now() + self.save(update_fields=["updated_date"]) + + if not self.complete and self.has_completed: + self.complete_job() + + def __terminate_job(self): # pylint: disable=unused-private-member + """Tell workers to ignore and not execute this task + & pending child tasks. Extend. + """ + super().__terminate_job() + self.__terminate_pending_child_jobs() + + def __terminate_pending_child_jobs(self): + """Tell workers to ignore and not execute any pending child tasks.""" + tasks = self.pending_child_jobs.filter(task_id__isnull=False).values_list( + "task_id", flat=True + ) + app.control.revoke(list(tasks)) + + for task in self.pending_child_jobs: + task.update(status=self.Status.STOPPED) + + @property + def has_completed(self): + """has this job finished""" + return not self.pending_child_jobs.exists() + + @property + def pending_child_jobs(self): + """items that haven't been processed yet""" + return self.child_jobs.filter(complete=False) + + +class ChildJob(Job): + """Stores the state of a Task for the related :model:`ParentJob`. + + Intended to be sub-classed if necessary via proxy or + multi-table inheritance. + Extends :model:`Job`. + """ + + parent_job = models.ForeignKey( + ParentJob, on_delete=models.CASCADE, related_name="child_jobs" + ) + + def set_status(self, status): + """Set job and parent_job status. Extend.""" + super().set_status(status) + + if ( + status == self.Status.ACTIVE + and self.parent_job.status == self.Status.PENDING + ): + self.parent_job.set_status(self.Status.ACTIVE) + + def complete_job(self): + """Report to parent_job that the job has completed. Extend.""" + super().complete_job() + self.parent_job.notify_child_job_complete() + + +class ParentTask(app.Task): + """Used with ParentJob, Abstract Tasks execute code at specific points in + a Task's lifecycle, applying to all Tasks with the same 'base'. + + All status & ParentJob.task_id assignment is managed here for you. + Usage e.g. @app.task(base=ParentTask) + """ + + def before_start( + self, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Handler called before the task starts. Override. + + Prepare ParentJob before the task starts. + + Arguments: + task_id (str): Unique id of the task to execute. + args (Tuple): Original arguments for the task to execute. + kwargs (Dict): Original keyword arguments for the task to execute. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + no_children (bool): If 'True' this is the only Task expected to run + for the given ParentJob. + + Returns: + None: The return value of this handler is ignored. + """ + job = ParentJob.objects.get(id=kwargs["job_id"]) + job.task_id = task_id + job.save(update_fields=["task_id"]) + + if kwargs["no_children"]: + job.set_status(ChildJob.Status.ACTIVE) + + def on_success( + self, retval, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Run by the worker if the task executes successfully. Override. + + Update ParentJob on Task complete. + + Arguments: + retval (Any): The return value of the task. + task_id (str): Unique id of the executed task. + args (Tuple): Original arguments for the executed task. + kwargs (Dict): Original keyword arguments for the executed task. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + no_children (bool): If 'True' this is the only Task expected to run + for the given ParentJob. + + Returns: + None: The return value of this handler is ignored. + """ + + if kwargs["no_children"]: + job = ParentJob.objects.get(id=kwargs["job_id"]) + job.complete_job() + + +class SubTask(app.Task): + """Used with ChildJob, Abstract Tasks execute code at specific points in + a Task's lifecycle, applying to all Tasks with the same 'base'. + + All status & ChildJob.task_id assignment is managed here for you. + Usage e.g. @app.task(base=SubTask) + """ + + def before_start( + self, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Handler called before the task starts. Override. + + Prepare ChildJob before the task starts. + + Arguments: + task_id (str): Unique id of the task to execute. + args (Tuple): Original arguments for the task to execute. + kwargs (Dict): Original keyword arguments for the task to execute. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + child_id (int): Unique 'id' of the ChildJob. + + Returns: + None: The return value of this handler is ignored. + """ + child_job = ChildJob.objects.get(id=kwargs["child_id"]) + child_job.task_id = task_id + child_job.save(update_fields=["task_id"]) + child_job.set_status(ChildJob.Status.ACTIVE) + + def on_success( + self, retval, task_id, args, kwargs + ): # pylint: disable=no-self-use, unused-argument + """Run by the worker if the task executes successfully. Override. + + Notify ChildJob of task completion. + + Arguments: + retval (Any): The return value of the task. + task_id (str): Unique id of the executed task. + args (Tuple): Original arguments for the executed task. + kwargs (Dict): Original keyword arguments for the executed task. + + Keyword Arguments: + job_id (int): Unique 'id' of the ParentJob. + child_id (int): Unique 'id' of the ChildJob. + + Returns: + None: The return value of this handler is ignored. + """ + subtask = ChildJob.objects.get(id=kwargs["child_id"]) + subtask.complete_job() + + +@transaction.atomic +def create_child_job(parent_job, task_callback): + """Utility method for creating a ChildJob + and running a task to avoid DB race conditions + """ + child_job = ChildJob.objects.create(parent_job=parent_job) + transaction.on_commit( + lambda: task_callback.delay(job_id=parent_job.id, child_id=child_job.id) + ) + + return child_job diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index d056c05b30..ca1e2aeb05 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -1,8 +1,16 @@ """ alert a user to activity """ from django.db import models, transaction from django.dispatch import receiver +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain +from . import ( + Boost, + Favorite, + GroupMemberInvitation, + ImportJob, + BookwyrmImportJob, + LinkDomain, +) from . import ListItem, Report, Status, User, UserFollowRequest from .site import InviteRequest @@ -23,6 +31,8 @@ class NotificationType(models.TextChoices): # Imports IMPORT = "IMPORT" + USER_IMPORT = "USER_IMPORT" + USER_EXPORT = "USER_EXPORT" # List activity ADD = "ADD" @@ -63,6 +73,9 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey( + "BookwyrmExportJob", on_delete=models.CASCADE, null=True + ) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -226,6 +239,36 @@ def notify_user_on_import_complete( ) +@receiver(models.signals.post_save, sender=BookwyrmImportJob) +# pylint: disable=unused-argument +def notify_user_on_user_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, notification_type=NotificationType.USER_IMPORT + ) + + +@receiver(models.signals.post_save, sender=BookwyrmExportJob) +# pylint: disable=unused-argument +def notify_user_on_user_export_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we exported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, + notification_type=NotificationType.USER_EXPORT, + related_user_export=instance, + ) + + @receiver(models.signals.post_save, sender=Report) @transaction.atomic # pylint: disable=unused-argument diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 7af6ad5abd..3386a02dce 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -65,6 +65,13 @@ def get_remote_id(self): base_path = self.user_subject.remote_id return f"{base_path}#follows/{self.id}" + def get_accept_reject_id(self, status): + """get id for sending an accept or reject of a local user""" + + base_path = self.user_object.remote_id + status_id = self.id or 0 + return f"{base_path}#{status}/{status_id}" + class UserFollows(ActivityMixin, UserRelationship): """Following a user""" @@ -105,6 +112,20 @@ def from_request(cls, follow_request): ) return obj + def reject(self): + """generate a Reject for this follow. This would normally happen + when a user deletes a follow they previously accepted""" + + if self.user_object.local: + activity = activitypub.Reject( + id=self.get_accept_reject_id(status="rejects"), + actor=self.user_object.remote_id, + object=self.to_activity(), + ).serialize() + self.broadcast(activity, self.user_object) + + self.delete() + class UserFollowRequest(ActivitypubMixin, UserRelationship): """following a user requires manual or automatic confirmation""" @@ -148,13 +169,6 @@ def save(self, *args, broadcast=True, **kwargs): # pylint: disable=arguments-di if not manually_approves: self.accept() - def get_accept_reject_id(self, status): - """get id for sending an accept or reject of a local user""" - - base_path = self.user_object.remote_id - status_id = self.id or 0 - return f"{base_path}#{status}/{status_id}" - def accept(self, broadcast_only=False): """turn this request into the real deal""" user = self.user_object diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index b962d597b6..201a499e55 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -10,8 +10,11 @@ from django.utils import timezone from model_utils import FieldTracker +from bookwyrm.connectors.abstract_connector import get_data from bookwyrm.preview_images import generate_site_preview_image_task from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL +from bookwyrm.settings import RELEASE_API +from bookwyrm.tasks import app, MISC from .base_model import BookWyrmModel, new_access_code from .user import User from .fields import get_absolute_url @@ -45,7 +48,7 @@ class SiteSettings(SiteModel): default_theme = models.ForeignKey( "Theme", null=True, blank=True, on_delete=models.SET_NULL ) - version = models.CharField(null=True, blank=True, max_length=10) + available_version = models.CharField(null=True, blank=True, max_length=10) # admin setup options install_mode = models.BooleanField(default=False) @@ -96,6 +99,8 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_exports_enabled = models.BooleanField(default=False) + user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) @@ -243,3 +248,14 @@ def preview_image(instance, *args, **kwargs): if len(changed_fields) > 0: generate_site_preview_image_task.delay() + + +@app.task(queue=MISC) +def check_for_updates_task(): + """See if git remote knows about a new version""" + site = SiteSettings.objects.get() + release = get_data(RELEASE_API, timeout=3) + available_version = release.get("tag_name", None) + if available_version: + site.available_version = available_version + site.save(update_fields=["available_version"]) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index cc44fe2bf5..f6235dab6c 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -12,6 +12,8 @@ from django.dispatch import receiver from django.template.loader import get_template from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django.utils.translation import ngettext_lazy from model_utils import FieldTracker from model_utils.managers import InheritanceManager @@ -107,14 +109,14 @@ def delete(self, *args, **kwargs): # pylint: disable=unused-argument @property def recipients(self): """tagged users who definitely need to get this status in broadcast""" - mentions = [u for u in self.mention_users.all() if not u.local] + mentions = {u for u in self.mention_users.all() if not u.local} if ( hasattr(self, "reply_parent") and self.reply_parent and not self.reply_parent.user.local ): - mentions.append(self.reply_parent.user) - return list(set(mentions)) + mentions.add(self.reply_parent.user) + return list(mentions) @classmethod def ignore_activity( @@ -178,6 +180,24 @@ def boostable(self): """you can't boost dms""" return self.privacy in ["unlisted", "public"] + @property + def page_title(self): + """title of the page when only this status is shown""" + return _("%(display_name)s's status") % {"display_name": self.user.display_name} + + @property + def page_description(self): + """description of the page in meta tags when only this status is shown""" + return None + + @property + def page_image(self): + """image to use as preview in meta tags when only this status is shown""" + if self.mention_books.exists(): + book = self.mention_books.first() + return book.preview_image or book.cover + return self.user.preview_image + def to_replies(self, **kwargs): """helper function for loading AP serialized replies to a status""" return self.to_ordered_collection( @@ -301,6 +321,10 @@ class Meta: abstract = True + @property + def page_image(self): + return self.book.preview_image or self.book.cover or super().page_image + class Comment(BookStatus): """like a review but without a rating and transient""" @@ -332,17 +356,26 @@ def pure_content(self): activity_serializer = activitypub.Comment + @property + def page_title(self): + return _("%(display_name)s's comment on %(book_title)s") % { + "display_name": self.user.display_name, + "book_title": self.book.title, + } + class Quotation(BookStatus): """like a review but without a rating and transient""" quote = fields.HtmlField() raw_quote = models.TextField(blank=True, null=True) - position = models.IntegerField( - validators=[MinValueValidator(0)], null=True, blank=True + position = models.TextField( + null=True, + blank=True, ) - endposition = models.IntegerField( - validators=[MinValueValidator(0)], null=True, blank=True + endposition = models.TextField( + null=True, + blank=True, ) position_mode = models.CharField( max_length=3, @@ -374,6 +407,13 @@ def pure_content(self): activity_serializer = activitypub.Quotation + @property + def page_title(self): + return _("%(display_name)s's quote from %(book_title)s") % { + "display_name": self.user.display_name, + "book_title": self.book.title, + } + class Review(BookStatus): """a book review""" @@ -403,6 +443,13 @@ def pure_content(self): """indicate the book in question for mastodon (or w/e) users""" return self.content + @property + def page_title(self): + return _("%(display_name)s's review of %(book_title)s") % { + "display_name": self.user.display_name, + "book_title": self.book.title, + } + activity_serializer = activitypub.Review pure_type = "Article" @@ -426,6 +473,18 @@ def pure_content(self): template = get_template("snippets/generated_status/rating.html") return template.render({"book": self.book, "rating": self.rating}).strip() + @property + def page_description(self): + return ngettext_lazy( + "%(display_name)s rated %(book_title)s: %(display_rating).1f star", + "%(display_name)s rated %(book_title)s: %(display_rating).1f stars", + "display_rating", + ) % { + "display_name": self.user.display_name, + "book_title": self.book.title, + "display_rating": self.rating, + } + activity_serializer = activitypub.Rating pure_type = "Note" diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 75ca1d527d..89fd39b738 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -523,6 +523,20 @@ def save(self, *args, **kwargs): return super().save(*args, **kwargs) +@app.task(queue=MISC) +def erase_user_data(user_id): + """Erase any custom data about this user asynchronously + This is for deleted historical user data that pre-dates data + being cleared automatically""" + user = User.objects.get(id=user_id) + user.erase_user_data() + user.save( + broadcast=False, + update_fields=["email", "avatar", "preview_image", "summary", "name"], + ) + user.erase_user_statuses(broadcast=False) + + @app.task(queue=MISC) def set_remote_server(user_id, allow_external_connections=False): """figure out the user's remote server in the background""" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index aaa50c56dd..77bec0d8ed 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -30,8 +30,11 @@ PAGE_LENGTH = env.int("PAGE_LENGTH", 15) DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English") +# TODO: extend maximum age to 1 year once termination of active sessions +# is implemented (see bookwyrm-social#2278, bookwyrm-social#3082). +SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 30) # 1 month -JS_CACHE = "ac315a3b" +JS_CACHE = "8a89cad7" # email EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend") @@ -99,6 +102,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.humanize", + "file_resubmit", "sass_processor", "bookwyrm", "celery", @@ -120,6 +124,7 @@ "bookwyrm.middleware.IPBlocklistMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "bookwyrm.middleware.FileTooBig", ] ROOT_URLCONF = "bookwyrm.urls" @@ -243,7 +248,11 @@ CACHES = { "default": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", - } + }, + "file_resubmit": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + "LOCATION": "/tmp/file_resubmit_tests/", + }, } else: CACHES = { @@ -253,7 +262,11 @@ "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", }, - } + }, + "file_resubmit": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/tmp/file_resubmit/", + }, } SESSION_ENGINE = "django.contrib.sessions.backends.cache" @@ -309,6 +322,7 @@ ("eu-es", _("Euskara (Basque)")), ("gl-es", _("Galego (Galician)")), ("it-it", _("Italiano (Italian)")), + ("ko-kr", _("한국어 (Korean)")), ("fi-fi", _("Suomi (Finnish)")), ("fr-fr", _("Français (French)")), ("lt-lt", _("Lietuvių (Lithuanian)")), @@ -319,6 +333,7 @@ ("pt-pt", _("Português Europeu (European Portuguese)")), ("ro-ro", _("Română (Romanian)")), ("sv-se", _("Svenska (Swedish)")), + ("uk-ua", _("Українська (Ukrainian)")), ("zh-hans", _("简体中文 (Simplified Chinese)")), ("zh-hant", _("繁體中文 (Traditional Chinese)")), ] @@ -337,8 +352,7 @@ USE_TZ = True -agent = requests.utils.default_user_agent() -USER_AGENT = f"{agent} (BookWyrm/{VERSION}; +https://{DOMAIN}/)" +USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)" # Imagekit generated thumbnails ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False) @@ -432,3 +446,7 @@ # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" + +# We only allow specifying DATA_UPLOAD_MAX_MEMORY_SIZE in MiB from .env +# (note the difference in variable names). +DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_MiB", 100) << 20 diff --git a/bookwyrm/static/js/autocomplete.js b/bookwyrm/static/js/autocomplete.js index a98cd96346..6836d356d8 100644 --- a/bookwyrm/static/js/autocomplete.js +++ b/bookwyrm/static/js/autocomplete.js @@ -111,6 +111,10 @@ const tries = { }, }, f: { + b: { + 2: "FB2", + 3: "FB3", + }, l: { a: { c: "FLAC", diff --git a/bookwyrm/templates/413.html b/bookwyrm/templates/413.html new file mode 100644 index 0000000000..337436aae5 --- /dev/null +++ b/bookwyrm/templates/413.html @@ -0,0 +1,16 @@ +{% extends 'layout.html' %} +{% load i18n %} + +{% block title %}{% trans "File too large" %}{% endblock %} + +{% block content %} +
+

{% trans "File too large" %}

+

{% trans "The file you are uploading is too large." %}

+

+ {% blocktrans %} + You you can try using a smaller file, or ask your BookWyrm server administrator to increase the DATA_UPLOAD_MAX_MEMORY_SIZE setting. + {% endblocktrans %} +

+
+{% endblock %} diff --git a/bookwyrm/templates/about/about.html b/bookwyrm/templates/about/about.html index 6705793d5e..ef3f340371 100644 --- a/bookwyrm/templates/about/about.html +++ b/bookwyrm/templates/about/about.html @@ -31,10 +31,10 @@

-
+
{% if superlatives.top_rated %} {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %} -
+
@@ -53,7 +53,7 @@

{% if superlatives.wanted %} {% with book=superlatives.wanted.default_edition %} -
+
@@ -72,7 +72,7 @@

{% if superlatives.controversial %} {% with book=superlatives.controversial.default_edition %} -
+
diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index 8e76fb0148..4c345832e1 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -9,7 +9,8 @@ {% block title %}{{ book|book_title }}{% endblock %} {% block opengraph %} - {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book.preview_image %} + {% firstof book.preview_image book.cover as book_image %} + {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book_image %} {% endblock %} {% block content %} diff --git a/bookwyrm/templates/confirm_email/confirm_email.html b/bookwyrm/templates/confirm_email/confirm_email.html index abdd3a734c..49a1ebd2da 100644 --- a/bookwyrm/templates/confirm_email/confirm_email.html +++ b/bookwyrm/templates/confirm_email/confirm_email.html @@ -6,8 +6,8 @@ {% block content %}

{% trans "Confirm your email address" %}

-
-
+
+

{% trans "A confirmation code has been sent to the email address you used to register your account." %}

diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 7ecf10b70f..1b6cf29ffb 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -41,7 +41,7 @@

{% endif %} - {% if annual_summary_year and tab.key == 'home' %} + {% if annual_summary_year and tab.key == 'home' and has_summary_read_throughs %}