Merge pull request #490 from CityOfNewYork/develop

OpenRecords v3.1.0

.env.example

@@ -1,4 +1,6 @@
 # Flask Config
+FLASK_APP=openrecords.py
+FLASK_ENV=development
 SECRET_KEY=<FLASK SECRET KEY>
 BASE_URL=<BASE URL FOR DEPLOYED APP>
 VIEW_REQUEST_ENDPOINT=<ENDPOINT FOR VIEWING A REQUEST>
@@ -18,6 +20,7 @@ REASON_DATA=<FULL PATH TO "reasons.csv">
 STAFF_DATA=<FULL PATH TO "staff.csv">
 ENVELOPE_TEMPLATES_DATA=<FULL PATH TO "envelope_templates.csv">
 LATEX_TEMPLATE_DIRECTORY=<FULL PATH TO "latex" directory>
+OPENRECORDS_AGENCY_SUPPORT_DL=<Email for Agency Support>
 
 # JSON Schema
 JSON_SCHEMA_DIRECTORY=<FULL PATH TO "schemas directory">

app/agency/api/views.py

@@ -118,7 +118,7 @@ def get_custom_request_form_fields():
             field_name = value['name']
             field_type = value['type']
             field_values = value.get('values', None)
-            field_required = value['required']
+            field_required = value.get('required', False)
             min_length = value.get('min_length', None)
             max_length = value.get('max_length', None)
             character_counter = value.get('character_counter', None)

app/auth/utils.py

@@ -407,16 +407,9 @@ def saml_sls(saml_sp, user_guid):
         Response Object: Redirect the user to the Home Page.
 
     """
-    dscb = session.clear()
+    dscb = session.destroy()
     url = saml_sp.process_slo(delete_session_cb=dscb)
     errors = saml_sp.get_errors()
-    update_object(
-        {
-            'session_id': None
-        },
-        Users,
-        user_guid
-    )
     logout_user()
     if not errors:
         create_auth_event(
@@ -460,6 +453,14 @@ def saml_slo(saml_sp):
     if 'samlSessionIndex' in session:
         session_index = session['samlSessionIndex']
 
+    update_object(
+        {
+            'session_id': None
+        },
+        Users,
+        current_user.guid
+    )
+
     return saml_sp.logout(name_id=name_id, session_index=session_index)
 
 
@@ -509,6 +510,7 @@ def saml_acs(saml_sp, onelogin_request):
             is_anonymous_requester=False
         )
         login_user(user)
+        session.permanent = True
         create_auth_event(
             auth_event_type=event_type.USER_LOGIN,
             user_guid=user.guid,
@@ -529,7 +531,7 @@ def saml_acs(saml_sp, onelogin_request):
     current_app.logger.exception(error_message)
     create_auth_event(
         auth_event_type=event_type.USER_FAILED_LOG_IN,
-        user_guid=user_data['GUID'] if user_data['GUID'] is not None else '',
+        user_guid=user_data.get('GUID', None),
         new_value={
             'success': False,
             'type': current_app.config['AUTH_TYPE'],

app/auth/views.py

@@ -199,7 +199,8 @@ def saml():
     elif "slo" in request.args:
         return redirect(saml_slo(onelogin_saml_auth))
     elif "sls" in request.args:
-        return saml_sls(onelogin_saml_auth, current_user.guid)
+        user_guid = current_user.guid if not current_user.is_anonymous else None
+        return saml_sls(onelogin_saml_auth, user_guid)
     else:
         flash(
             "Oops! Something went wrong. Please try to perform your action again later.",

app/constants/schemas/agencies.schema

@@ -205,7 +205,6 @@
           "ein",
           "parent_ein",
           "name",
-          "acronym",
           "default_email",
           "appeals_email"
         ],
@@ -265,7 +264,7 @@
           },
           "acronym": {
             "$id": "/properties/agencies/items/properties/acronym",
-            "type": "string",
+            "type": ["string", "null"],
             "title": "Acronym",
             "description": "Acronym for the agency.",
             "default": "",
@@ -278,7 +277,7 @@
           },
           "next_request_number": {
             "$id": "/properties/agencies/items/properties/next_request_number",
-            "type": "string",
+            "type": "integer",
             "title": "Next Request Number",
             "description": "The next request number available for the agency. 5 digit number represented as a String",
             "default": "00001",

app/jobs.py

@@ -4,6 +4,7 @@
 from flask import (current_app, render_template)
 import celery
 from psycopg2 import OperationalError
+from sqlalchemy.exc import SQLAlchemyError
 
 from app import calendar, sentry, db
 from app.constants import OPENRECORDS_DL_EMAIL, request_status
@@ -20,11 +21,12 @@
 STATUSES_EMAIL_TEMPLATE = "email_templates/email_request_status_changed"
 
 
-@celery.task(autoretry_for=(OperationalError,), retry_kwargs={'max_retries': 5}, retry_backoff=True)
+@celery.task(autoretry_for=(OperationalError, SQLAlchemyError,), retry_kwargs={'max_retries': 5}, retry_backoff=True)
 def update_request_statuses():
     try:
         _update_request_statuses()
     except Exception:
+        db.session.rollback()
         send_email(
             subject="Update Request Statuses Failure",
             to=[OPENRECORDS_DL_EMAIL],
@@ -159,13 +161,16 @@ def _update_request_statuses():
         )
 
 
-@celery.task()
+@celery.task(autoretry_for=(OperationalError, SQLAlchemyError,), retry_kwargs={'max_retries': 5}, retry_backoff=True)
 def update_next_request_number():
     """
     Celery task to automatically update the next request number of each agency to 1
     :return:
     """
-    for agency in Agencies.query.all():
-        agency.next_request_number = 1
-        db.session.add(agency)
-    db.session.commit()
+    try:
+        for agency in Agencies.query.all():
+            agency.next_request_number = 1
+            db.session.add(agency)
+        db.session.commit()
+    except SQLAlchemyError:
+        db.session.rollback()

app/main/views.py

@@ -3,19 +3,22 @@
 
    :synopsis: Handles all core URL endpoints for the timeclock application
 """
-from . import main
 from flask import (
+    current_app,
     render_template,
     flash,
+    render_template,
     request,
     session
 )
 from flask_login import current_user
-from app.lib.email_utils import send_contact_email
-from app.lib.db_utils import create_object, update_object
-from app.models import Emails, Users
+
 from app.constants import OPENRECORDS_DL_EMAIL
 from app.constants.response_privacy import PRIVATE
+from app.lib.db_utils import create_object, update_object
+from app.lib.email_utils import send_contact_email
+from app.models import Emails, Users
+from . import main
 
 
 @main.route('/', methods=['GET', 'POST'])
@@ -63,7 +66,10 @@ def technical_support():
                     body=body,
                 )
             )
-            send_contact_email(subject, [OPENRECORDS_DL_EMAIL], body, email)
+            if current_user.is_agency:
+                send_contact_email(subject, [current_app.config['OPENRECORDS_AGENCY_SUPPORT_DL']], body, email)
+            else:
+                send_contact_email(subject, [OPENRECORDS_DL_EMAIL], body, email)
             flash('Your message has been sent. We will get back to you.', category='success')
         else:
             flash('Cannot send email.', category='danger')

app/models.py

@@ -290,14 +290,14 @@ def populate(cls, json_name=None):
 
             for agency in data['agencies']:
                 if Agencies.query.filter_by(ein=agency['ein']).first() is not None:
-                    warn("Duplicate EIN ({ein}); Row not imported", category=UserWarning)
+                    warn("Duplicate EIN ({ein}); Row not imported".format(ein=agency['ein']), category=UserWarning)
                     continue
                 a = cls(
                     ein=agency['ein'],
                     parent_ein=agency['parent_ein'],
                     categories=agency['categories'],
                     name=agency['name'],
-                    acronym=agency['acronym'],
+                    acronym=agency.get('acronym', None),
                     next_request_number=agency['next_request_number'],
                     default_email=agency['default_email'],
                     appeals_email=agency['appeals_email'],
@@ -334,19 +334,6 @@ class Users(UserMixin, db.Model):
     mailing_address - a JSON object containing the user's address
     """
     __tablename__ = 'users'
-    # from app.constants import user_type_auth
-    # auth_user_type = db.Column(
-    #     db.Enum(user_type_auth.AGENCY_USER,
-    #             user_type_auth.AGENCY_LDAP_USER,
-    #             user_type_auth.PUBLIC_USER_FACEBOOK,
-    #             user_type_auth.PUBLIC_USER_MICROSOFT,
-    #             user_type_auth.PUBLIC_USER_YAHOO,
-    #             user_type_auth.PUBLIC_USER_LINKEDIN,
-    #             user_type_auth.PUBLIC_USER_GOOGLE,
-    #             user_type_auth.PUBLIC_USER_NYC_ID,
-    #             user_type_auth.ANONYMOUS_USER,
-    #             name='auth_user_type'),
-    #     primary_key=True)
     guid = db.Column(db.String(64), unique=True, primary_key=True)
     is_nyc_employee = db.Column(db.Boolean, default=False)
     has_nyc_account = db.Column(db.Boolean, default=False)

app/report/forms.py

@@ -3,9 +3,12 @@
 
     :synopsis: Defines forms used for report statistics.
 """
-from flask_wtf import Form
-from wtforms import SelectField
+from datetime import date
+
 from flask_login import current_user
+from flask_wtf import Form
+from wtforms import DateField, SelectField, SubmitField
+from wtforms.validators import DataRequired
 
 from app.lib.db_utils import get_agency_choices
 
@@ -31,3 +34,26 @@ def __init__(self):
                                    key=lambda x: x[1])
             for user_agency in user_agencies:
                 self.agency.choices.insert(1, self.agency.choices.pop(self.agency.choices.index(user_agency)))
+
+
+class AcknowledgmentForm(Form):
+    """Form to generate a report with acknowledgment data."""
+    date_from = DateField('Date From (required)', format='%m/%d/%Y', validators=[DataRequired()])
+    date_to = DateField('Date To (required)', format='%m/%d/%Y', validators=[DataRequired()])
+    submit_field = SubmitField('Generate Report')
+
+    def validate(self):
+        if not super().validate():
+            return False
+        is_valid = True
+        for field in [self.date_from, self.date_to]:
+            if field.data > date.today():
+                field.errors.append('The {} cannot be greater than today.'.format(field.label.text))
+                is_valid = False
+        if self.date_to.data < self.date_from.data:
+            field.errors.append('Date To cannot be before Date From.')
+            is_valid = False
+        if self.date_from.data == self.date_to.data:
+            field.errors.append('The dates cannot be the same.')
+            is_valid = False
+        return is_valid