updates ui templating;

redirects to all orgs page on load; udpates harvest source page with feedback changes; updates fixtures to have better jobs data; fixes broken tests; lint
GSA · Feb 11, 2025 · 3e64623 · 3e64623 · github-actions · Feb 11, 2025
1 parent bccea3f
commit 3e64623
Show file tree

Hide file tree

Showing 20 changed files with 400 additions and 409 deletions.
diff --git a/app/routes.py b/app/routes.py
@@ -5,6 +5,7 @@
 import secrets
 import time
 import uuid
+from datetime import timedelta
 from functools import wraps
 from io import StringIO
 
@@ -28,7 +29,7 @@
 
 from database.interface import HarvesterDBInterface
 from harvester.lib.load_manager import LoadManager
-from harvester.utils.general_utils import convert_to_int, is_it_true
+from harvester.utils.general_utils import convert_to_int, get_datetime, is_it_true
 
 from . import htmx
 from .forms import HarvestSourceForm, OrganizationForm
@@ -309,7 +310,7 @@ def make_new_org_contract(form):
 # Routes
 @mod.route("/", methods=["GET"])
 def index():
-    return render_template("index.html")
+    return redirect(url_for("harvest.view_organizations"))
 
 
 ## Organizations
@@ -336,7 +337,7 @@ def add_organization():
                 flash(f"Added new organization with ID: {org.id}")
             else:
                 flash("Failed to add organization.")
-            return redirect("/")
+            return redirect(url_for("harvest.view_organizations"))
         elif form.errors:
             flash(form.errors)
             return redirect(url_for("harvest.add_organization"))
@@ -458,7 +459,7 @@ def add_harvest_source():
                 flash(f"Updated source with ID: {source.id}. {job_message}")
             else:
                 flash("Failed to add harvest source.")
-            return redirect("/")
+            return redirect(url_for("harvest.view_harvest_sources"))
         elif form.errors:
             flash(form.errors)
             return redirect(url_for("harvest.add_harvest_source"))
@@ -474,28 +475,33 @@ def add_harvest_source():
 @mod.route("/harvest_source/<source_id>", methods=["GET"])
 def view_harvest_source_data(source_id: str):
     source = db.get_harvest_source(source_id)
-    records_count = db.get_harvest_records_by_source(
-        count=True,
+    records = db.get_latest_harvest_records_by_source(
         source_id=source.id,
     )
-    ckan_records_count = db.get_harvest_records_by_source(
-        count=True,
-        source_id=source.id,
-        facets="ckan_id is not null",
-    )
-    error_records_count = db.get_harvest_records_by_source(
-        count=True,
-        source_id=source.id,
-        facets="status = 'error'",
-    )
+    summary_data = {
+        "records_count": len(records),
+        "last_job_errors": "N/A",
+        "last_job_finished": "N/A",
+        "next_job_scheduled": "N/A",
+    }
+
     # TODO: wire in paginated jobs htmx refresh ui & route
     jobs = db.pget_harvest_jobs(
         paginate=False, facets=f"harvest_source_id = '{source.id}'"
     )
-    next_job = "N/A"
+    if len(jobs):
+        last_job = jobs[len(jobs) - 1]
+        last_job_error_count = db.get_harvest_record_errors_by_job(
+            count=True,
+            job_id=last_job.id,
+        )
+        summary_data["last_job_errors"] = last_job_error_count
+        summary_data["last_job_finished"] = last_job.date_finished
+
     future_jobs = db.get_new_harvest_jobs_by_source_in_future(source.id)
     if len(future_jobs):
-        next_job = future_jobs[0].date_created
+        summary_data["next_job_scheduled"] = future_jobs[0].date_created
+
     chartdata = {
         "labels": [job.date_finished for job in jobs],
         "datasets": [
@@ -526,14 +532,10 @@ def view_harvest_source_data(source_id: str):
         ],
     }
     data = {
-        "harvest_source": source,
-        "harvest_source_dict": db._to_dict(source),
-        "total_records": records_count,
-        "records_with_ckan_id": ckan_records_count,
-        "records_with_error": error_records_count,
+        "harvest_source": db._to_dict(source),
+        "summary_data": summary_data,
         "harvest_jobs": jobs,
         "chart": chartdata,
-        "next_job": next_job,
     }
     return render_template("view_source_data.html", data=data)
 
@@ -753,12 +755,15 @@ def download_harvest_errors_by_job(job_id, error_type):
                     [
                         error.id,
                         identifier,
-                        json.loads(source_raw).get("title", None) if source_raw else None,
+                        (
+                            json.loads(source_raw).get("title", None)
+                            if source_raw
+                            else None
+                        ),
                         error.harvest_record_id,
                         error.type,
                         error.message,
-                        error.date_created
-
+                        error.date_created,
                     ]
                     for error, identifier, source_raw in db.get_harvest_record_errors_by_job(
                         job_id, paginate=False
@@ -772,7 +777,7 @@ def download_harvest_errors_by_job(job_id, error_type):
                         "harvest_record_id",
                         "record_error_type",
                         "message",
-                        "date_created"
+                        "date_created",
                     ]
                 ]
 
@@ -790,15 +795,15 @@ def download_harvest_errors_by_job(job_id, error_type):
         return "Please provide correct job_id"
 
 
-## Harvest Record
-### Get record
+# Records
+## Get record
 @mod.route("/harvest_record/<record_id>", methods=["GET"])
 def get_harvest_record(record_id):
     record = db.get_harvest_record(record_id)
     return db._to_dict(record) if record else ("Not Found", 404)
 
 
-### Get records
+## Get records
 @mod.route("/harvest_records/", methods=["GET"])
 def get_harvest_records():
     job_id = request.args.get("harvest_job_id")
@@ -826,6 +831,7 @@ def get_harvest_records():
         return db._to_dict(records)
 
 
+## Get records source raw
 @mod.route("/harvest_record/<record_id>/raw", methods=["GET"])
 def get_harvest_record_raw(record_id=None):
     record = db.get_harvest_record(record_id)
@@ -839,7 +845,7 @@ def get_harvest_record_raw(record_id=None):
         return {"error": "Not Found"}, 404
 
 
-### Add record
+## Add record
 @mod.route("/harvest_record/add", methods=["POST", "GET"])
 def add_harvest_record():
     if request.is_json:
@@ -876,13 +882,50 @@ def get_harvest_error(error_id: str = None) -> dict:
         return db._to_dict(errors)
 
 
-## Test interface, will remove later
-## TODO: remove with completion of https://github.com/GSA/data.gov/issues/4741
-@mod.route("/get_data_sources", methods=["GET"])
-def get_data_sources():
-    source = db.get_all_harvest_sources()
-    org = db.get_all_organizations()
-    return render_template("get_data_sources.html", sources=source, organizations=org)
+@mod.route("/metrics/", methods=["GET"])
+def view_metrics():
+    """Render index page with recent harvest jobs."""
+    # Get current time in UTC
+    current_time = get_datetime()
+    start_time = current_time - timedelta(hours=24)
+
+    # Format timestamps for SQL query
+    time_filter = f"date_created >= '{start_time.isoformat()}'"
+
+    # Get recent jobs with pagination
+    page = convert_to_int(request.args.get("page", 1))  # Start from page 1
+    per_page = convert_to_int(request.args.get("per_page", 10))
+
+    jobs = db.pget_harvest_jobs(
+        facets=time_filter,
+        page=page - 1,  # Convert to 0-based for DB
+        per_page=per_page,
+        paginate=True,
+    )
+
+    total = db.pget_harvest_jobs(facets=time_filter, count=True)
+
+    # Create pagination object matching the existing pattern
+    page_count = (total + per_page - 1) // per_page
+    pagination = {
+        "current": page,
+        "page_count": page_count,
+        "count": total,
+        "per_page": per_page,
+        "page_label": "Page",
+        "previous": {"label": "Previous", "page": page - 1 if page > 1 else None},
+        "next": {"label": "Next", "page": page + 1 if page < page_count else None},
+        "last_item": {"label": "Last page", "page": page_count},
+    }
+
+    return render_template(
+        "metrics_dashboard.html",
+        jobs=jobs,
+        pagination=pagination,
+        current_time=current_time,
+        window_start=start_time,
+        data={"htmx_vars": {}},  # Required for pagination template
+    )
 
 
 def register_routes(app):

diff --git a/app/static/_scss/_uswds-theme-custom-styles.scss b/app/static/_scss/_uswds-theme-custom-styles.scss
@@ -28,8 +28,10 @@ i.e.
 
 ul.menu {
   padding-inline-start: 0;
+
   li {
     display: inline-block;
+
     &::after {
       content: '  |'
     }
@@ -45,11 +47,13 @@ ul.menu {
 .align-middle {
   vertical-align: middle;
   top: -1px;
+
   &.check {
-      color: green;
+    color: green;
   }
+
   &.cancel {
-      color: red;
+    color: red;
   }
 }
 
@@ -68,7 +72,7 @@ ul.menu {
 }
 
 .container:first-child {
-  display: flex; 
+  display: flex;
   justify-content: space-between;
 }
 

diff --git a/app/templates/base.html b/app/templates/base.html
@@ -1,62 +1,68 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
-    <meta charset="UTF  -8">
-    {% block title %}
-        <title>{{action}} {{data_type}}</title>
-    {% endblock %}
-    <link rel="stylesheet" href="{{ url_for('static', filename='assets/uswds/css/styles.css') }}">
-    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
-    <script src="{{ url_for('static', filename='assets/uswds/js/uswds-init.js') }}"></script>
-    <script src="{{ url_for('static', filename='assets/htmx/htmx.min.js') }}"></script>
-    {% block script_head %}
-    {% endblock %}
+  <meta charset="UTF  -8">
+  {% block title %}
+  <title>{{action}} {{data_type}}</title>
+  {% endblock %}
+  <link rel="stylesheet" href="{{ url_for('static', filename='assets/uswds/css/styles.css') }}">
+  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+  <script src="{{ url_for('static', filename='assets/uswds/js/uswds-init.js') }}"></script>
+  <script src="{{ url_for('static', filename='assets/htmx/htmx.min.js') }}"></script>
+  {% block script_head %}
+  {% endblock %}
 </head>
+
 <body>
   {% include '/snippets/banner.html' %}
-    <header>
-      <div class="container">
-        <strong>
-          <nav>
-            <ul class="menu">
-              <li><a href="{{ url_for('harvest.index') }}">Home</a></li>
-              <li><a href="{{ url_for('harvest.view_organizations') }}">Orgs</a></li>
-              <li><a href="{{ url_for('harvest.view_harvest_sources') }}">Sources</a></li>
-              {% if session['user'] %}
-                <li>{{ session['user'] }}</li>
-                <li><a href="{{ url_for('harvest.logout') }}">Logout</a></li>
-              {% else %}
-                <li><a href="{{ url_for('harvest.login') }}">Login</a></li>
-              {% endif %}
-            </ul>
-          </nav>
-        </strong>
-        <a href="javascript:void(0)" class="icon-glossary js-glossary-toggle">
-          <svg aria-hidden="true" width=" 14" height="16" viewBox="0 0 14 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-            <path fill-rule="evenodd" clip-rule="evenodd" d="M13.9583 2.99612C13.9583 2.25376 13.5274 1.631 12.9074 1.25803C12.2875 0.885127 11.4706 0.755653 10.6759 0.999207L7.41667 1.99802L4.15746 0.999208C3.36272 0.755654 2.54584 0.885127 1.92596 1.25803C1.30597 1.631 0.875 2.25376 0.875 2.99612V11.6299C0.875 12.5372 1.54014 13.3298 2.50921 13.6268L7.41667 15.1307L12.3241 13.6268C13.2932 13.3298 13.9583 12.5372 13.9583 11.6299V2.99612ZM11.1429 2.17693C11.4882 2.07112 11.8416 2.12807 12.107 2.28774C12.3723 2.44735 12.5417 2.7033 12.5417 2.99612V11.6299C12.5417 11.9855 12.2789 12.3198 11.8571 12.4491L8.125 13.5928V3.10179L11.1429 2.17693ZM2.29167 2.99612C2.29167 2.7033 2.46099 2.44735 2.7263 2.28774C2.99172 2.12808 3.34516 2.07112 3.69046 2.17694L6.70833 3.10179V13.5928L2.97621 12.4491C2.55441 12.3198 2.29167 11.9855 2.29167 11.6299V2.99612Z" />
-          </svg>
-          Glossary</a>
-      </div>
-    </header>
+  <header>
+    <div class="container">
+      <strong>
+        <nav>
+          <ul class="menu">
+            <li><a href="{{ url_for('harvest.view_organizations') }}">Organizations</a></li>
+            <li><a href="{{ url_for('harvest.view_harvest_sources') }}">Harvest Sources</a></li>
+            <li><a href="{{ url_for('harvest.view_metrics') }}">Metrics</a></li>
+            {% if session['user'] %}
+            <li>{{ session['user'] }}</li>
+            <li><a href="{{ url_for('harvest.logout') }}">Logout</a></li>
+            {% else %}
+            <li><a href="{{ url_for('harvest.login') }}">Login</a></li>
+            {% endif %}
+          </ul>
+        </nav>
+      </strong>
+      <a href="javascript:void(0)" class="icon-glossary js-glossary-toggle">
+        <svg aria-hidden="true" width=" 14" height="16" viewBox="0 0 14 16" fill="none"
+          xmlns="http://www.w3.org/2000/svg">
+          <path fill-rule="evenodd" clip-rule="evenodd"
+            d="M13.9583 2.99612C13.9583 2.25376 13.5274 1.631 12.9074 1.25803C12.2875 0.885127 11.4706 0.755653 10.6759 0.999207L7.41667 1.99802L4.15746 0.999208C3.36272 0.755654 2.54584 0.885127 1.92596 1.25803C1.30597 1.631 0.875 2.25376 0.875 2.99612V11.6299C0.875 12.5372 1.54014 13.3298 2.50921 13.6268L7.41667 15.1307L12.3241 13.6268C13.2932 13.3298 13.9583 12.5372 13.9583 11.6299V2.99612ZM11.1429 2.17693C11.4882 2.07112 11.8416 2.12807 12.107 2.28774C12.3723 2.44735 12.5417 2.7033 12.5417 2.99612V11.6299C12.5417 11.9855 12.2789 12.3198 11.8571 12.4491L8.125 13.5928V3.10179L11.1429 2.17693ZM2.29167 2.99612C2.29167 2.7033 2.46099 2.44735 2.7263 2.28774C2.99172 2.12808 3.34516 2.07112 3.69046 2.17694L6.70833 3.10179V13.5928L2.97621 12.4491C2.55441 12.3198 2.29167 11.9855 2.29167 11.6299V2.99612Z" />
+        </svg>
+        Glossary</a>
+    </div>
+  </header>
 
-    {% include 'glossary.html' %}
-
-    <div class="container my-3">
-        {% for message in get_flashed_messages() %}
-        <div class="alert alert-warning">
-            <button type="button" class="close" data-dismiss="alert">&times;</button>
-            {{ message }}
-        </div>
-        {% endfor %}
+  {% include 'glossary.html' %}
 
-        <div id="content">{% block content %}{% endblock %}</div>
+  <div class="container my-3">
+    {% for message in get_flashed_messages() %}
+    <div class="alert alert-warning">
+      <button type="button" class="close" data-dismiss="alert">&times;</button>
+      {{ message }}
     </div>
-    {% include '/snippets/footer.html' %}
-    <script src="https://code.jquery.com/jquery-3.7.1.min.js" integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo=" crossorigin="anonymous"></script>
-    <script src="{{ url_for('static', filename='assets/uswds/js/uswds.js') }}"></script>
-    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.bundle.min.js"></script>
-    <script src="{{ url_for('static', filename='assets/js/bundle.js') }}"></script>
-    {% block scripts %}
-    {% endblock %}
+    {% endfor %}
+
+    <div id="content">{% block content %}{% endblock %}</div>
+  </div>
+  {% include '/snippets/footer.html' %}
+  <script src="https://code.jquery.com/jquery-3.7.1.min.js"
+    integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo=" crossorigin="anonymous"></script>
+  <script src="{{ url_for('static', filename='assets/uswds/js/uswds.js') }}"></script>
+  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.bundle.min.js"></script>
+  <script src="{{ url_for('static', filename='assets/js/bundle.js') }}"></script>
+  {% block scripts %}
+  {% endblock %}
 </body>
+
 </html>
Title	Tests	Time
Unit tests	39	1.035s ⏱️
Integration Tests	73	4.55s ⏱️
Functional Tests	2	6.59s ⏱️