Merge branch 'feature/sequential-hard-triggers-fix' of https://github…

….com/uwcirg/true_nth_usa_portal into feature/opt-out-UI

portal/migrations/versions/80c3b1e96c45_.py

@@ -1,14 +1,16 @@
 """Add sequential hard trigger count to EMPRO trigger_states.triggers domains.
 
 Revision ID: 80c3b1e96c45
-Revises: 2e9b9e696bb8
+Revises: 3c871e710277
 Create Date: 2023-07-24 17:08:35.128975
 
 """
 from collections import defaultdict
 from copy import deepcopy
 from alembic import op
 from io import StringIO
+from flask import current_app
+import logging
 from sqlalchemy.orm import sessionmaker
 from portal.database import db
 from portal.trigger_states.empro_domains import (
@@ -19,12 +21,48 @@
 
 # revision identifiers, used by Alembic.
 revision = '80c3b1e96c45'
-down_revision = '2e9b9e696bb8'
+down_revision = '3c871e710277'
 
 Session = sessionmaker()
 
+log = logging.getLogger("alembic.runtime.migration")
+log.setLevel(logging.DEBUG)
+
+
+def validate_users_trigger_states(session, patient_id):
+    """Confirm user has sequential visits in trigger states table.
+
+    Due to allowance of moving EMPRO consents and no previous checks,
+    some users on test have invalid overlapping trigger states rows.
+    """
+    ts_rows = session.query(TriggerState).filter(
+        TriggerState.user_id == patient_id).order_by(TriggerState.id)
+    month_counter = -1
+    for row in ts_rows:
+        if row.state == 'due':
+            # skipping months is okay, but every due should be sequentially greater than previous
+            if month_counter >= row.visit_month:
+                raise ValueError(f"{patient_id} expected month > {month_counter}, got {row.visit_month}")
+            month_counter = row.visit_month
+        else:
+            # states other than 'due' should be grouped together with same visit_month
+            if month_counter != row.visit_month:
+                raise ValueError(f"{patient_id} expected month {month_counter}, got {row.visit_month}")
+
+def purge_trigger_states(session, patient_id):
+    """Clean up test system problems from moving consent dates"""
+    log.info(f"Purging trigger states for {patient_id}")
+    session.query(TriggerState).filter(TriggerState.user_id == patient_id).delete()
+
 
 def upgrade():
+    # Add sequential counts to appropriate trigger_states rows.
+
+    # this migration was applied once before, but the code wasn't correctly
+    # maintaining the sequential counts.  start by removing all for a clean
+    # slate via the same `downgrade()` step
+    downgrade()
+
     # for each active EMPRO patient with at least 1 hard triggered domain,
     # walk through their monthly reports, adding the sequential count for
     # the opt-out feature.
@@ -42,6 +80,15 @@ def upgrade():
         # can't just send through current process, as it'll attempt to
         # insert undesired rows in the trigger_states table.  need to
         # add the sequential count to existing rows.
+        try:
+            validate_users_trigger_states(session, pid)
+        except ValueError as e:
+            if current_app.config.get('SYSTEM_TYPE') in ('development', 'test'):
+                purge_trigger_states(session, pid)
+                continue
+            else:
+                raise e
+
         output.write(f"\n\nPatient: {pid}  storing all zeros for sequential hard triggers except:\n")
         output.write("  (visit month : domain : # hard sequential)\n")
         sequential_by_domain = defaultdict(list)
@@ -54,9 +101,10 @@ def upgrade():
             for d in EMPRO_DOMAINS:
                 sequential_hard_for_this_domain = 0
                 if d not in improved_triggers["domain"]:
-                    # only seen on test, fill in the missing domain
-                    print(f"missing {d} in {pid}:{ts.visit_month}?")
-                    improved_triggers["domain"][d] = {}
+                    # shouldn't happen, SDC typically includes all domains
+                    # but a few records are lacking
+                    log.warning(f"{pid} missing domain {d} in {ts.visit_month} response")
+                    continue
 
                 if any(v == "hard" for v in improved_triggers["domain"][d].values()):
                     sequential_by_domain[d].append(ts.visit_month)
@@ -81,4 +129,36 @@ def upgrade():
 
 
 def downgrade():
-    pass  # no value in removing
+    # for each active EMPRO patient with at least 1 hard triggered domain,
+    # remove any sequential counts found
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    patient_ids = []
+    for patient_id in session.execute(
+            "SELECT DISTINCT(user_id) FROM trigger_states JOIN users"
+            " ON users.id = user_id WHERE deleted_id IS NULL"):
+        patient_ids.append(patient_id[0])
+
+    output = StringIO()
+    for pid in patient_ids:
+        output.write(f"\n\nPatient: {pid}\n")
+        trigger_states = db.session.query(TriggerState).filter(
+            TriggerState.user_id == pid).filter(
+            TriggerState.state == "resolved").order_by(
+            TriggerState.timestamp.asc())
+        for ts in trigger_states:
+            improved_triggers = deepcopy(ts.triggers)
+            for d in EMPRO_DOMAINS:
+                if d not in improved_triggers["domain"]:
+                    log.warning(f"{d} missing from {ts.id}(month: {ts.visit_month}) for {pid}")
+                    continue
+                if sequential_hard_trigger_count_key in improved_triggers["domain"][d]:
+                    del improved_triggers["domain"][d][sequential_hard_trigger_count_key]
+                    output.write(f"  removed sequential from {ts.visit_month}:{d} {improved_triggers['domain'][d]}\n")
+
+            # retain triggers now containing sequential counts
+            ts.triggers = improved_triggers
+
+        db.session.commit()
+        print(output.getvalue())

portal/migrations/versions/d1f3ed8d16ef_.py

@@ -11,7 +11,7 @@
 
 # revision identifiers, used by Alembic.
 revision = 'd1f3ed8d16ef'
-down_revision = '80c3b1e96c45'
+down_revision = '2e9b9e696bb8'
 
 
 def upgrade():

portal/models/qb_status.py

@@ -520,7 +520,8 @@ def warn_on_duplicate_request(self, requested_set):
                 f" {requested_indef} already!")
 
 
-def patient_research_study_status(patient, ignore_QB_status=False):
+def patient_research_study_status(
+        patient, ignore_QB_status=False, as_of_date=None, skip_initiate=False):
     """Returns details regarding patient readiness for available studies
 
     Wraps complexity of checking multiple QB_Status and ResearchStudy
@@ -532,6 +533,8 @@ def patient_research_study_status(patient, ignore_QB_status=False):
     :param patient: subject to check
     :param ignore_QB_status: set to prevent recursive call, if used during
       process of evaluating QB_status.  Will restrict results to eligible
+    :param as_of_date: set to check status at alternative time
+    :param skip_initiate: set only when rebuilding to avoid state change
     :returns: dictionary of applicable studies keyed by research_study_id.
       Each contains a dictionary with keys:
      - eligible: set True if assigned to research study and pre-requisites
@@ -546,7 +549,8 @@ def patient_research_study_status(patient, ignore_QB_status=False):
     """
     from datetime import datetime
     from .research_study import EMPRO_RS_ID, ResearchStudy
-    as_of_date = datetime.utcnow()
+    if as_of_date is None:
+        as_of_date = datetime.utcnow()
 
     results = {}
     # check studies in required order - first found with pending work
@@ -601,7 +605,8 @@ def patient_research_study_status(patient, ignore_QB_status=False):
             elif rs_status['ready']:
                 # As user may have just entered ready status on EMPRO
                 # move trigger_states.state to due
-                from ..trigger_states.empro_states import initiate_trigger
-                initiate_trigger(patient.id)
+                if not skip_initiate:
+                    from ..trigger_states.empro_states import initiate_trigger
+                    initiate_trigger(patient.id)
 
     return results

portal/models/qb_timeline.py

@@ -374,7 +374,8 @@ def qbds_for_rp(rp, classification, trigger_date):
             )
             if curRPD.retired == nextRPD.retired:
                 raise ValueError(
-                    "Invalid state: multiple RPs w/ same retire date")
+                    "Invalid state: multiple RPs w/ same retire date: "
+                    f"{next_rp} : {curRPD.retired}")
         else:
             nextRPD = None
         yield curRPD, nextRPD

portal/trigger_states/empro_domains.py

@@ -28,7 +28,7 @@ class DomainTriggers(object):
     """
 
     def __init__(
-            self, domain, current_answers, previous_answers, initial_answers):
+            self, domain, current_answers, previous_answers, initial_answers, previous_triggers):
         self.domain = domain
         self._triggers = dict()
 
@@ -38,6 +38,9 @@ def __init__(
         self.previous_answers = previous_answers or dict()
         self.initial_answers = initial_answers or dict()
 
+        # Trigger state triggers from previous month, if defined
+        self.previous_triggers = previous_triggers
+
     @property
     def triggers(self):
         self.eval()
@@ -92,9 +95,9 @@ def eval(self):
             sequential_hard_trigger_count = 1
         if (
                 sequential_hard_trigger_count and
-                self.previous_answers and
-                sequential_hard_trigger_count_key in self.previous_answers):
-            sequential_hard_trigger_count = self.previous_answers[sequential_hard_trigger_count_key] + 1
+                self.previous_triggers and
+                sequential_hard_trigger_count_key in self.previous_triggers):
+            sequential_hard_trigger_count = self.previous_triggers[sequential_hard_trigger_count_key] + 1
         self._triggers[sequential_hard_trigger_count_key] = sequential_hard_trigger_count
 
 
@@ -145,17 +148,19 @@ def obtain_observations(self, qnr):
                     results[domain][link_id] = (int(score), severity)
                 setattr(self, f"{timepoint}_obs", results)
 
-    def eval_triggers(self):
+    def eval_triggers(self, previous_triggers):
         triggers = dict()
         triggers['domain'] = dict()
 
         for domain in EMPRO_DOMAINS:
             if domain in self.cur_obs:
+                prev_triggers_for_domain = previous_triggers["domain"][domain] if previous_triggers else None
                 dt = DomainTriggers(
                     domain=domain,
                     current_answers=self.cur_obs[domain],
                     previous_answers=self.prev_obs.get(domain),
-                    initial_answers=self.initial_obs.get(domain)
+                    initial_answers=self.initial_obs.get(domain),
+                    previous_triggers=prev_triggers_for_domain,
                 )
                 triggers['domain'][domain] = dt.triggers
 

portal/trigger_states/empro_messages.py

@@ -2,7 +2,9 @@
 from datetime import datetime
 from flask import current_app, url_for
 from flask_babel import gettext as _
+from smtplib import SMTPRecipientsRefused
 
+from portal.database import db
 from portal.models.app_text import MailResource, app_text
 from portal.models.communication import EmailMessage, load_template_args
 from portal.models.organization import UserOrganization
@@ -11,6 +13,30 @@
 from portal.models.qb_status import QB_Status
 
 
+def invite_email(user):
+    if not user.email_ready():
+        current_app.logger.error(f"{user.id} can't receive EMPRO invite email")
+        return
+    args = load_template_args(user=user)
+    item = MailResource(
+        app_text("patient invite email IRONMAN EMPRO Study"),
+        locale_code=user.locale_code,
+        variables=args)
+    msg = EmailMessage(
+        subject=item.subject,
+        body=item.body,
+        recipients=user.email,
+        sender=current_app.config['MAIL_DEFAULT_SENDER'],
+        user_id=user.id)
+    try:
+        msg.send_message()
+    except SMTPRecipientsRefused as exc:
+        current_app.logger.error(
+            "Error sending EMPRO Invite to %s: %s",
+            user.email, exc)
+    db.session.add(msg)
+
+
 def patient_email(patient, soft_triggers, hard_triggers):
     """Prepare email for patient, depending on trigger status"""