Skip to content

Commit

Permalink
fixes issues with metrics pagination;
Browse files Browse the repository at this point in the history 
cleans up metrics route code;
fixes issue with pagination numbers in pagination module;
makes pagination per_page count a kwarg;
removes sort functionality from tables as it breaks with htmx updates and is not super useful
  • Loading branch information
@btylerburton
btylerburton committed Feb 12, 2025
1 parent 8eadb39 commit b054910
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 137 deletions.
8 changes: 5 additions & 3 deletions app/paginate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@


class Pagination:
def __init__(self, current: int = 1, count: int = 1):
def __init__(
self, current: int = 1, count: int = 1, per_page=PAGINATE_ENTRIES_PER_PAGE
):
self.current = current
self.count = count
self.page_count = math.ceil(count / PAGINATE_ENTRIES_PER_PAGE)
self.per_page = PAGINATE_ENTRIES_PER_PAGE
self.page_count = math.ceil(count / per_page)
self.per_page = per_page

def to_dict(self):
return {
Expand Down
76 changes: 39 additions & 37 deletions app/routes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -652,9 +652,8 @@ def get_harvest_job(job_id=None):
pagination = Pagination(count=record_error_count)

if htmx:
page = request.args.get("page")
db_page = int(page) - 1
record_errors = db.get_harvest_record_errors_by_job(job_id, page=db_page)
page = request.args.get("page", 1, type=convert_to_int)
record_errors = db.get_harvest_record_errors_by_job(job_id, page=page - 1)
record_errors_dict = [
{
"error": db._to_dict(row.HarvestRecordError),
Expand Down Expand Up @@ -686,7 +685,6 @@ def get_harvest_job(job_id=None):
}
for row in record_errors
]

if request.args.get("type") and request.args.get("type") == "json":
return db._to_dict(job) if job else ("Not Found", 404)
else:
Expand Down Expand Up @@ -885,47 +883,51 @@ def get_harvest_error(error_id: str = None) -> dict:
@mod.route("/metrics/", methods=["GET"])
def view_metrics():
"""Render index page with recent harvest jobs."""
# Get current time in UTC
current_time = get_datetime()
start_time = current_time - timedelta(hours=24)

# Format timestamps for SQL query
time_filter = f"date_created >= '{start_time.isoformat()}'"

# Get recent jobs with pagination
page = convert_to_int(request.args.get("page", 1)) # Start from page 1
per_page = convert_to_int(request.args.get("per_page", 10))

jobs = db.pget_harvest_jobs(
job_count = db.pget_harvest_jobs(
facets=time_filter,
page=page - 1, # Convert to 0-based for DB
per_page=per_page,
paginate=True,
count=True,
)

total = db.pget_harvest_jobs(facets=time_filter, count=True)

# Create pagination object matching the existing pattern
page_count = (total + per_page - 1) // per_page
pagination = {
"current": page,
"page_count": page_count,
"count": total,
"per_page": per_page,
"page_label": "Page",
"previous": {"label": "Previous", "page": page - 1 if page > 1 else None},
"next": {"label": "Next", "page": page + 1 if page < page_count else None},
"last_item": {"label": "Last page", "page": page_count},
htmx_vars = {
"target_div": "#paginated__harvest-jobs",
"endpoint_url": "/metrics",
}
pagination = Pagination(count=job_count, per_page=10)

return render_template(
"metrics_dashboard.html",
jobs=jobs,
pagination=pagination,
current_time=current_time,
window_start=start_time,
data={"htmx_vars": {}}, # Required for pagination template
)
if htmx:
page = request.args.get("page", 1, type=convert_to_int)
jobs = db.pget_harvest_jobs(facets=time_filter, page=page - 1, per_page=10)
data = {
"jobs": jobs,
"htmx_vars": htmx_vars,
}
pagination.update_current(page)
return render_block(
"metrics_dashboard.html",
"htmx_paginated",
data=data,
pagination=pagination.to_dict(),
)
else:
jobs = db.pget_harvest_jobs(
facets=time_filter,
page=request.args.get("page", type=convert_to_int),
per_page=10,
)
data = {
"htmx_vars": htmx_vars,
"jobs": jobs,
"current_time": current_time,
"window_start": start_time,
}
return render_template(
"metrics_dashboard.html",
pagination=pagination.to_dict(),
data=data,
)


def register_routes(app):
Expand Down
2 changes: 1 addition & 1 deletion app/templates/components/pagination/pagination_numbers.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
{# Page numbers #}
{# List all items if less than 7 #}
{% if pagination.page_count <= 7 %}
{% for item in range(1, pagination.page_count) %}
{% for item in range(1, pagination.page_count + 1) %}
{{ pagination_button(item, pager_button_opts, data.htmx_vars) }}
{% endfor %}
{# User is at the start of a long dataset #}
Expand Down
138 changes: 71 additions & 67 deletions app/templates/metrics_dashboard.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,77 +9,81 @@
<div class="section my-3">
<h2>Recent Harvest Jobs</h2>
<p class="text-muted">
Showing jobs from {{ window_start.strftime('%Y-%m-%d %H:%M:%S UTC') }}
to {{ current_time.strftime('%Y-%m-%d %H:%M:%S UTC') }}
Showing jobs from {{ data.window_start.strftime('%Y-%m-%d %H:%M:%S UTC') }}
to {{ data.current_time.strftime('%Y-%m-%d %H:%M:%S UTC') }}
</p>

{% if jobs %}
<div class="usa-table-container--scrollable" tabindex="0">
<table class="usa-table usa-table--striped">
<caption>Recent Harvest Jobs</caption>
<thead>
<tr>
<th data-sortable scope="col" role="columnheader">Source ID</th>
<th data-sortable scope="col" role="columnheader">Status</th>
<th data-sortable scope="col" role="columnheader">Created</th>
<th data-sortable scope="col" role="columnheader">Records Added</th>
<th data-sortable scope="col" role="columnheader">Records Updated</th>
<th data-sortable scope="col" role="columnheader">Records Deleted</th>
<th data-sortable scope="col" role="columnheader">Records Errored</th>
<th scope="col" role="columnheader">Actions</th>
</tr>
</thead>
<tbody>
{% for job in jobs %}
<tr>
<td data-sort-value="{{ job.harvest_source_id }}">
<a
href="{{ url_for('harvest.view_harvest_source_data', source_id=job.harvest_source_id) }}">
{{ job.harvest_source_id[:8] }}...
</a>
</td>
<td data-sort-value="{{ job.status }}">
<span class="usa-tag {% if job.status == 'error' %}usa-tag--error
{% elif job.status == 'in_progress' %}usa-tag--warning
{% elif job.status == 'complete' %}usa-tag--success
{% else %}usa-tag--base{% endif %}">
{{ job.status }}
</span>
</td>
<td data-sort-value="{{ job.date_created }}">{{ job.date_created.strftime('%Y-%m-%d %H:%M:%S')
}}</td>
<td data-sort-value="{{ job.records_added or 0 }}">{{ job.records_added or 0 }}</td>
<td data-sort-value="{{ job.records_updated or 0 }}">{{ job.records_updated or 0 }}</td>
<td data-sort-value="{{ job.records_deleted or 0 }}">{{ job.records_deleted or 0 }}</td>
<td data-sort-value="{{ job.records_errored or 0 }}">{{ job.records_errored or 0 }}</td>
<td>
<ul class="usa-button-group">
<li class="usa-button-group__item">
<a href="{{ url_for('harvest.get_harvest_job', job_id=job.id) }}"
class="usa-button usa-button--outline">
View
</a>
</li>
{% if job.status == 'in_progress' %}
<li class="usa-button-group__item">
<a href="{{ url_for('harvest.cancel_harvest_job', job_id=job.id) }}"
class="usa-button usa-button--secondary">
Cancel
</a>
</li>
{% endif %}
</ul>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="usa-sr-only usa-table__announcement-region" aria-live="polite"></div>
{% if data.jobs %}
{% block htmx_paginated %}
<div id="paginated__harvest-jobs">
<div class="usa-table-container--scrollable" tabindex="0">
<table class="usa-table usa-table--striped">
<caption>Recent Harvest Jobs</caption>
<thead>
<tr>
<th scope="col">Source ID</th>
<th scope="col">Status</th>
<th scope="col">Created</th>
<th scope="col">Records Added</th>
<th scope="col">Records Updated</th>
<th scope="col">Records Deleted</th>
<th scope="col">Records Errored</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody id="paginated_harvest_jobs">
{% for job in data.jobs %}
<tr>
<td data-sort-value="{{ job.harvest_source_id }}">
<a
href="{{ url_for('harvest.view_harvest_source_data', source_id=job.harvest_source_id) }}">
{{ job.harvest_source_id[:8] }}...
</a>
</td>
<td>
<span class="usa-tag {% if job.status == 'error' %}usa-tag--error
{% elif job.status == 'in_progress' %}usa-tag--warning
{% elif job.status == 'complete' %}usa-tag--success
{% else %}usa-tag--base{% endif %}">
{{ job.status }}
</span>
</td>
<td>{{ job.date_created.strftime('%Y-%m-%d %H:%M:%S')
}}</td>
<td>{{ job.records_added}}</td>
<td>{{ job.records_updated}}</td>
<td>{{ job.records_deleted}}</td>
<td>{{ job.records_errored}}</td>
<td>
<ul class="usa-button-group">
<li class="usa-button-group__item">
<a href="{{ url_for('harvest.get_harvest_job', job_id=job.id) }}"
class="usa-button usa-button--outline">
View
</a>
</li>
{% if job.status == 'in_progress' %}
<li class="usa-button-group__item">
<a href="{{ url_for('harvest.cancel_harvest_job', job_id=job.id) }}"
class="usa-button usa-button--secondary">
Cancel
</a>
</li>
{% endif %}
</ul>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="usa-sr-only usa-table__announcement-region" aria-live="polite"></div>
</div>
{% if pagination.count > jobs|count %}
{% include '/components/pagination/pagination.html' %}
{% endif %}
</div>
{% endblock %}

{% if pagination.page_count > 1 %}
{% include '/components/pagination/pagination.html' %}
{% endif %}

{% else %}
<div class="usa-alert usa-alert--info">
Expand Down
57 changes: 30 additions & 27 deletions app/templates/view_job_data.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,27 +44,28 @@ <h2>Job Error Table</h2>
No job errors found
{% else %}
{% set error_type = "job" %}
<a href="{{ url_for('harvest.download_harvest_errors_by_job', job_id=data.harvest_job_id, error_type=error_type)}}">
<a
href="{{ url_for('harvest.download_harvest_errors_by_job', job_id=data.harvest_job_id, error_type=error_type)}}">
<button class="btn btn-primary">download job errors as .csv</button>
</a>
<div class="usa-table-container--scrollable" tabindex="0">
<table class="usa-table usa-table--striped">
<caption> Harvest Job Errors for {{data.harvest_job_id}} </caption>
<thead>
<tr>
<th data-sortable scope="col" role="columnheader">Date Created</th>
<th data-sortable scope="col" role="columnheader">Id</th>
<th data-sortable scope="col" role="columnheader">Type</th>
<th data-sortable scope="col" role="columnheader">Message</th>
<th scope="col">Date Created</th>
<th scope="col">Id</th>
<th scope="col">Type</th>
<th scope="col">Message</th>
</tr>
</thead>
<tbody>
{% for errors in data.harvest_job.errors %}
<tr>
<td data-sort-value={errors.date_created}> {{errors.date_created}}</td>
<td data-sort-value={errors.id}>{{errors.id}}</td>
<td data-sort-value={errors.type}>{{errors.type}}</td>
<td data-sort-value={errors.message}>{{errors.message}}</td>
<td> {{errors.date_created}}</td>
<td>{{errors.id}}</td>
<td>{{errors.type}}</td>
<td>{{errors.message}}</td>
</tr>
{% endfor %}
</tbody>
Expand All @@ -77,40 +78,42 @@ <h2>Job Error Table</h2>
<div class="section">
<h2>Record Error Details</h2>
{% if not data.record_errors %}
<p>No record errors found</p>
<p>No record errors found</p>
{% else %}
{% set error_type = "record" %}
<a href="{{ url_for('harvest.download_harvest_errors_by_job', job_id=data.harvest_job_id, error_type=error_type)}}">
<a
href="{{ url_for('harvest.download_harvest_errors_by_job', job_id=data.harvest_job_id, error_type=error_type)}}">
<button class="btn btn-primary">download record errors as .csv</button>
</a>
<div class="usa-table-container--scrollable" tabindex="0">
{% block record_errors_table %}
<div id="error_results_pagination">
<div class="error-list">
{% for record_error in data.record_errors %}
<div class="error-block">
<h3>{{ record_error.error.id }}</h3>
<p><strong>Identifier:</strong> {{ record_error.identifier }}</p>
<p><strong>Title:</strong> {{ record_error.title }}</p>
<p><strong>Harvest Record ID:</strong>
<a href="{{ url_for('harvest.get_harvest_record', record_id=record_error.error.harvest_record_id) }}">
{{ record_error.error.harvest_record_id }}
</a>
</p>
<p><strong>Error Message:</strong> {{ record_error.error.message }}</p>
<p><strong>Type:</strong> {{ record_error.error.type }}</p>
<p><strong>Date Created:</strong> {{ record_error.error.date_created }}</p>
</div>
<div class="error-block">
<h3>{{ record_error.error.id }}</h3>
<p><strong>Identifier:</strong> {{ record_error.identifier }}</p>
<p><strong>Title:</strong> {{ record_error.title }}</p>
<p><strong>Harvest Record ID:</strong>
<a
href="{{ url_for('harvest.get_harvest_record', record_id=record_error.error.harvest_record_id) }}">
{{ record_error.error.harvest_record_id }}
</a>
</p>
<p><strong>Error Message:</strong> {{ record_error.error.message }}</p>
<p><strong>Type:</strong> {{ record_error.error.type }}</p>
<p><strong>Date Created:</strong> {{ record_error.error.date_created }}</p>
</div>
{% endfor %}
</div>
{% if pagination.count > data.record_errors|count %}
{% include '/components/pagination/pagination.html' %}
{% include '/components/pagination/pagination.html' %}
{%endif%}
</div>

{% endblock %}
<div class="usa-sr-only usa-table__announcement-region" aria-live="polite"></div>
</div>
<div class="usa-sr-only usa-table__announcement-region" aria-live="polite"></div>
</div>
{% endif %}
</div>
{% endif %}
Expand Down
3 changes: 1 addition & 2 deletions tests/unit/test_pagination.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ def test_update_current(self):
pagination.update_current(12)
assert pagination.current == 12

@patch("app.paginate.PAGINATE_ENTRIES_PER_PAGE", 7)
def test_change_pagination_val(self):
pagination = Pagination(count=40)
pagination = Pagination(count=40, per_page=7)
assert pagination.count == 40
assert pagination.page_count == 6

1 comment on commit b054910

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Title Coverage Tests Skipped Failures Errors Time
Unit tests Coverage 39 0 💤 0 ❌ 0 🔥 1.01s ⏱️
Integration Tests Coverage 73 0 💤 0 ❌ 0 🔥 4.6s ⏱️
Functional Tests Coverage 2 0 💤 0 ❌ 0 🔥 7.988s ⏱️

Please sign in to comment.