Audiobook time tracking (#1288)

* Time tracking models and model tests

* Time tracking route with api models added

The api takes bulk playtime entries to insert into the DB

* Playtime summation script and tests

The cron job is slated to run every 12 hours, shifted by 8 to avoid clutter

* Playtime reporting script added with a configurable email recipient

To run once a month and send a quarterly report

* Changed API route

* Added time tracking links to feed entries

* Mypy fixes

* Python 3.8 syntax fix

* Alembic ordering fix

* Added collection and library information to the playtime entries and summaries

* Reporting summation groups on collection and library as well now

* Playtimes API validations

* Modularized playtime entries

* Fixed UTC date issue

* PR updates

* Added the 401 gone status for very old entries

* Playtime entries reaping cut off time

* Time tracking rels only for specific collections

* Mypyp fixes

* Fixed 401 Gone to 410 Gone

* Only a loans feed with active loans for a work will have time tracking rels

* 410 is now counted as a failure

* Fixed time tracking links

* Mypy fix

* API spec for the route

* Switched from collection.name to colletion.id for the playtime route

* Add missing API fields.

---------

Co-authored-by: Tim DiLauro <tdilauro@users.noreply.github.com>

alembic/versions/20230728_892c8e0c89f8_audiobook_playtime_tracking.py

@@ -0,0 +1,116 @@
+"""Audiobook playtime tracking
+
+Revision ID: 892c8e0c89f8
+Revises: b3749bac3e55
+Create Date: 2023-07-28 07:20:24.625484+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "892c8e0c89f8"
+down_revision = "b3749bac3e55"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "playtime_entries",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("identifier_id", sa.Integer(), nullable=False),
+        sa.Column("collection_id", sa.Integer(), nullable=False),
+        sa.Column("library_id", sa.Integer(), nullable=False),
+        sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("total_seconds_played", sa.Integer(), nullable=False),
+        sa.Column("tracking_id", sa.String(length=64), nullable=False),
+        sa.Column("processed", sa.Boolean(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["collection_id"],
+            ["collections.id"],
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["identifier_id"],
+            ["identifiers.id"],
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["library_id"], ["libraries.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "identifier_id", "collection_id", "library_id", "tracking_id"
+        ),
+    )
+    op.create_table(
+        "playtime_summaries",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("identifier_id", sa.Integer(), nullable=True),
+        sa.Column("collection_id", sa.Integer(), nullable=False),
+        sa.Column("library_id", sa.Integer(), nullable=False),
+        sa.Column("identifier_str", sa.String(), nullable=False),
+        sa.Column("collection_name", sa.String(), nullable=False),
+        sa.Column("library_name", sa.String(), nullable=False),
+        sa.Column("timestamp", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("total_seconds_played", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["collection_id"],
+            ["collections.id"],
+            onupdate="CASCADE",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["identifier_id"],
+            ["identifiers.id"],
+            onupdate="CASCADE",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["library_id"], ["libraries.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "identifier_str", "collection_name", "library_name", "timestamp"
+        ),
+    )
+    op.create_index(
+        op.f("ix_playtime_summaries_collection_id"),
+        "playtime_summaries",
+        ["collection_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_playtime_summaries_identifier_id"),
+        "playtime_summaries",
+        ["identifier_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_playtime_summaries_library_id"),
+        "playtime_summaries",
+        ["library_id"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_playtime_summaries_library_id"), table_name="playtime_summaries"
+    )
+    op.drop_index(
+        op.f("ix_playtime_summaries_identifier_id"), table_name="playtime_summaries"
+    )
+    op.drop_index(
+        op.f("ix_playtime_summaries_collection_id"), table_name="playtime_summaries"
+    )
+    op.drop_table("playtime_summaries")
+    op.drop_table("playtime_entries")
+    # ### end Alembic commands ###

api/controller.py

@@ -17,11 +17,13 @@
 from flask import Response, make_response, redirect
 from flask_babel import lazy_gettext as _
 from lxml import etree
+from pydantic import ValidationError
 from sqlalchemy.orm import eagerload
 from sqlalchemy.orm.exc import NoResultFound
 
 from api.authentication.access_token import AccessTokenProvider
 from api.model.patron_auth import PatronAuthAccessToken
+from api.model.time_tracking import PlaytimeEntriesPost, PlaytimeEntriesPostResponse
 from api.opds2 import OPDS2NavigationsAnnotator, OPDS2PublicationsAnnotator
 from api.saml.controller import SAMLController
 from core.analytics import Analytics
@@ -76,6 +78,7 @@
 from core.opds import AcquisitionFeed, NavigationFacets, NavigationFeed
 from core.opds2 import AcquisitonFeedOPDS2
 from core.opensearch import OpenSearchDocument
+from core.query.playtime_entries import PlaytimeEntries
 from core.user_profile import ProfileController as CoreProfileController
 from core.util.authentication_for_opds import AuthenticationForOPDSDocument
 from core.util.datetime_helpers import utc_now
@@ -189,6 +192,7 @@ class CirculationManager:
     odl_notification_controller: ODLNotificationController
     shared_collection_controller: SharedCollectionController
     static_files: StaticFileController
+    playtime_entries: PlaytimeEntriesController
 
     # Admin controllers
     admin_sign_in_controller: SignInController
@@ -458,6 +462,7 @@ def setup_one_time_controllers(self):
         self.shared_collection_controller = SharedCollectionController(self)
         self.static_files = StaticFileController(self)
         self.patron_auth_token = PatronAuthTokenController(self)
+        self.playtime_entries = PlaytimeEntriesController(self)
 
         from api.lcp.controller import LCPController
 
@@ -2421,6 +2426,48 @@ def track_event(self, identifier_type, identifier, event_type):
             return INVALID_ANALYTICS_EVENT_TYPE
 
 
+class PlaytimeEntriesController(CirculationManagerController):
+    def track_playtimes(self, collection_id, identifier_type, identifier_idn):
+        library: Library = flask.request.library
+        identifier = get_one(
+            self._db, Identifier, type=identifier_type, identifier=identifier_idn
+        )
+        collection = Collection.by_id(self._db, collection_id)
+
+        if not identifier:
+            return NOT_FOUND_ON_REMOTE.detailed(
+                f"The identifier {identifier_type}/{identifier_idn} was not found."
+            )
+        if not collection:
+            return NOT_FOUND_ON_REMOTE.detailed(
+                f"The collection {collection_id} was not found."
+            )
+
+        if collection not in library.collections:
+            return INVALID_INPUT.detailed("Collection was not found in the Library.")
+
+        if not identifier.licensed_through_collection(collection):
+            return INVALID_INPUT.detailed(
+                "This Identifier was not found in the Collection."
+            )
+
+        try:
+            data = PlaytimeEntriesPost(**flask.request.json)
+        except ValidationError as ex:
+            return INVALID_INPUT.detailed(ex.json())
+
+        responses, summary = PlaytimeEntries.insert_playtime_entries(
+            self._db, identifier, collection, library, data
+        )
+
+        response_data = PlaytimeEntriesPostResponse(
+            summary=summary, responses=responses
+        )
+        response = flask.jsonify(response_data.dict())
+        response.status_code = 207
+        return response
+
+
 class ODLNotificationController(CirculationManagerController):
     """Receive notifications from an ODL distributor when the
     status of a loan changes.

api/model/time_tracking.py

@@ -0,0 +1,68 @@
+import datetime
+import logging
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field, validator
+
+from core.util.flask_util import CustomBaseModel
+
+
+class PlaytimeTimeEntry(CustomBaseModel):
+    id: str = Field(description="An id to ensure uniqueness of the time entry")
+    during_minute: datetime.datetime = Field(
+        description="A minute boundary datetime of the format yyyy-mm-ddThh:mmZ"
+    )
+    seconds_played: int = Field(
+        description="How many seconds were played within this minute"
+    )
+
+    @validator("during_minute")
+    def validate_minute_datetime(cls, value: datetime.datetime):
+        """Coerce the datetime to a minute boundary"""
+        if value.tzname() != "UTC":
+            logging.getLogger("TimeTracking").error(
+                f"An incorrect timezone was received for a playtime ({value.tzname()})."
+            )
+            raise ValueError("Timezone MUST be UTC always")
+        value = value.replace(second=0, microsecond=0)
+        return value
+
+    @validator("seconds_played")
+    def validate_seconds_played(cls, value: int):
+        """Coerce the seconds played to a max of 60 seconds"""
+        if value > 60:
+            logging.getLogger("TimeTracking").warning(
+                "Greater than 60 seconds was received for a minute playtime."
+            )
+            value = 60
+        elif value < 0:
+            logging.getLogger("TimeTracking").warning(
+                "Less than 0 seconds was received for a minute playtime."
+            )
+            value = 0
+        return value
+
+
+class PlaytimeEntriesPost(CustomBaseModel):
+    book_id: Optional[str] = Field(
+        description="An identifier of a book (currently ignored)."
+    )
+    library_id: Optional[str] = Field(
+        description="And identifier for the library (currently ignored)."
+    )
+    time_entries: List[PlaytimeTimeEntry] = Field(description="A List of time entries")
+
+
+class PlaytimeEntriesPostSummary(CustomBaseModel):
+    total: int = 0
+    successes: int = 0
+    failures: int = 0
+
+
+class PlaytimeEntriesPostResponse(CustomBaseModel):
+    responses: List[Dict[str, Any]] = Field(
+        description="Responses as part of the multi-reponse"
+    )
+    summary: PlaytimeEntriesPostSummary = Field(
+        description="Summary of failures and successes"
+    )

api/opds.py

@@ -32,6 +32,8 @@
     Patron,
     Session,
 )
+from core.model.configuration import ExternalIntegration
+from core.model.constants import EditionConstants, LinkRelations
 from core.model.formats import FormatPriorities
 from core.model.integration import IntegrationConfiguration
 from core.opds import AcquisitionFeed, Annotator, UnfulfillableWork
@@ -1813,6 +1815,34 @@ def annotate_feed(self, feed, lane):
             for tag in tags:
                 feed.feed.append(tag)
 
+    def annotate_work_entry(
+        self, work, active_license_pool, edition, identifier, feed, entry
+    ):
+        super().annotate_work_entry(
+            work, active_license_pool, edition, identifier, feed, entry
+        )
+        # Only OPDS for Distributors should get the time tracking link
+        # And only if there is an active loan for the work
+        if (
+            edition.medium == EditionConstants.AUDIO_MEDIUM
+            and active_license_pool
+            and active_license_pool.collection.protocol
+            == ExternalIntegration.OPDS_FOR_DISTRIBUTORS
+            and work in self.active_loans_by_work
+        ):
+            feed.add_link_to_entry(
+                entry,
+                rel=LinkRelations.TIME_TRACKING,
+                href=self.url_for(
+                    "track_playtime_events",
+                    identifier_type=identifier.type,
+                    identifier=identifier.identifier,
+                    library_short_name=self.library.short_name,
+                    collection_id=active_license_pool.collection.id,
+                    _external=True,
+                ),
+            )
+
 
 class SharedCollectionLoanAndHoldAnnotator(SharedCollectionAnnotator):
     @classmethod

api/routes.py

@@ -8,6 +8,7 @@
 from werkzeug.exceptions import HTTPException
 
 from api.model.patron_auth import PatronAuthAccessToken
+from api.model.time_tracking import PlaytimeEntriesPost, PlaytimeEntriesPostResponse
 from core.app_server import ErrorHandler, compressible, returns_problem_detail
 from core.model import HasSessionCache
 from core.util.problem_detail import ProblemDetail
@@ -672,6 +673,25 @@ def track_analytics_event(identifier_type, identifier, event_type):
     )
 
 
+@library_route(
+    "/playtimes/<int:collection_id>/<identifier_type>/<path:identifier>",
+    methods=["POST"],
+)
+@api_spec.validate(
+    resp=SpecResponse(HTTP_200=PlaytimeEntriesPostResponse),
+    body=PlaytimeEntriesPost,
+    tags=["analytics"],
+)
+@has_library
+@allows_auth
+@returns_problem_detail
+def track_playtime_events(collection_id, identifier_type, identifier):
+    """The actual response type is 207, but due to a bug in flask-pydantic-spec we must document it as a 200"""
+    return app.manager.playtime_entries.track_playtimes(
+        collection_id, identifier_type, identifier
+    )
+
+
 # Route that redirects to the authentication URL for a SAML provider
 @library_route("/saml_authenticate")
 @has_library

bin/playtime_reporting

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+"""Sum the playtimes for audiobooks."""
+import os
+import sys
+
+bin_dir = os.path.split(__file__)[0]
+package_dir = os.path.join(bin_dir, "..")
+sys.path.append(os.path.abspath(package_dir))
+
+from core.jobs.playtime_entries import PlaytimeEntriesEmailReportsScript
+from core.model import production_session
+
+PlaytimeEntriesEmailReportsScript(production_session(initialize_data=False)).run()

bin/playtime_summation

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+"""Sum the playtimes for audiobooks."""
+import os
+import sys
+
+bin_dir = os.path.split(__file__)[0]
+package_dir = os.path.join(bin_dir, "..")
+sys.path.append(os.path.abspath(package_dir))
+
+from core.jobs.playtime_entries import PlaytimeEntriesSummationScript
+from core.model import production_session
+
+PlaytimeEntriesSummationScript(production_session(initialize_data=False)).run()

core/config.py

@@ -53,6 +53,9 @@ class Configuration(ConfigurationConstants):
     # Environment variable for SirsiDynix Auth
     SIRSI_DYNIX_APP_ID = "SIMPLIFIED_SIRSI_DYNIX_APP_ID"
 
+    # Environment variable for temporary reporting email
+    REPORTING_EMAIL_ENVIRONMENT_VARIABLE = "SIMPLIFIED_REPORTING_EMAIL"
+
     # ConfigurationSetting key for the base url of the app.
     BASE_URL_KEY = "base_url"