Improve handling of published/unpublished data in downloads (#956)

* wip - team level downloads

* WIP refactor animal with get_published_only pattern

* update the rest of the animal viewset

* change permissions to team member or higher instead of can_edit

* lint

* update epi api and rename serializer

* update epi tests

* update downloads page

* add checkbox for inclusion of unpublished data

* remove checkbox from unsupported exports

* restore checkbox to some downloads

* remove unused template

* update in vitro and epimeta apis

* update ROB exports

* fixes from review

* only add the unpublished flag if the checkbox exists

* whitespace edits

---------

Co-authored-by: Andy Shapiro <shapiromatron@gmail.com>

hawc/apps/animal/api.py

@@ -3,7 +3,7 @@
 from django.db.models import Q
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
-from rest_framework.exceptions import NotAcceptable, PermissionDenied
+from rest_framework.exceptions import NotAcceptable
 from rest_framework.response import Response
 
 from ..assessment.api import (
@@ -13,9 +13,10 @@
     DoseUnitsViewSet,
 )
 from ..assessment.constants import AssessmentViewSetPermissions
+from ..common.api.utils import get_published_only
 from ..common.helper import FlatExport, cacheable
 from ..common.renderers import PandasRenderers
-from ..common.serializers import HeatmapQuerySerializer, UnusedSerializer
+from ..common.serializers import ExportQuerySerializer, UnusedSerializer
 from ..common.views import create_object_log
 from . import exports, models, serializers
 from .actions.model_metadata import AnimalMetadata
@@ -26,11 +27,9 @@ class AnimalAssessmentViewSet(BaseAssessmentViewSet):
     model = models.Assessment
     serializer_class = UnusedSerializer
 
-    def get_endpoint_queryset(self):
-        perms = self.assessment.user_permissions(self.request.user)
-        if not perms["edit"]:
-            return models.Endpoint.objects.published(self.assessment)
-        return models.Endpoint.objects.get_qs(self.assessment)
+    def get_endpoint_queryset(self, request):
+        published_only = get_published_only(self.assessment, request)
+        return models.Endpoint.objects.get_qs(self.assessment).published_only(published_only)
 
     @action(
         detail=True,
@@ -41,10 +40,13 @@ def get_endpoint_queryset(self):
     def full_export(self, request, pk):
         """
         Retrieve complete animal data
+
+        By default only shows data from published studies. If the query param `unpublished=true`
+        is present then results from all studies are shown.
         """
         self.assessment = self.get_object()
         exporter = exports.EndpointGroupFlatComplete(
-            self.get_endpoint_queryset(),
+            self.get_endpoint_queryset(request),
             filename=f"{self.assessment}-bioassay-complete",
             assessment=self.assessment,
         )
@@ -59,10 +61,13 @@ def full_export(self, request, pk):
     def endpoint_export(self, request, pk):
         """
         Retrieve endpoint animal data
+
+        By default only shows data from published studies. If the query param `unpublished=true`
+        is present then results from all studies are shown.
         """
         self.assessment = self.get_object()
         exporter = exports.EndpointSummary(
-            self.get_endpoint_queryset(),
+            self.get_endpoint_queryset(request),
             filename=f"{self.assessment}-bioassay-summary",
             assessment=self.assessment,
         )
@@ -82,15 +87,13 @@ def study_heatmap(self, request, pk):
         is present then results from all studies are shown.
         """
         self.assessment = self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-bioassay-study-heatmap-pub-{unpublished}"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-bioassay-study-heatmap-unpublished-{not published_only}"
 
         def func() -> pd.DataFrame:
-            return models.Endpoint.heatmap_study_df(self.assessment, published_only=not unpublished)
+            return models.Endpoint.heatmap_study_df(self.assessment, published_only=published_only)
 
         df = cacheable(func, key)
         return FlatExport.api_response(df=df, filename=f"bio-study-heatmap-{self.assessment.id}")
@@ -109,15 +112,13 @@ def endpoint_heatmap(self, request, pk):
         is present then results from all studies are shown.
         """
         self.assessment = self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-bioassay-endpoint-heatmap-unpublished-{unpublished}"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-bioassay-endpoint-heatmap-unpublished-{not published_only}"
 
         def df_func() -> pd.DataFrame:
-            return models.Endpoint.heatmap_df(self.assessment.id, published_only=not unpublished)
+            return models.Endpoint.heatmap_df(self.assessment.id, published_only=published_only)
 
         df = cacheable(df_func, key)
         return FlatExport.api_response(df=df, filename=f"bio-endpoint-heatmap-{self.assessment.id}")
@@ -136,15 +137,13 @@ def endpoint_doses_heatmap(self, request, pk):
         is present then results from all studies are shown.
         """
         self.assessment = self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-bioassay-endpoint-doses-heatmap-unpublished-{unpublished}"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-bioassay-endpoint-doses-heatmap-unpublished-{not published_only}"
 
         def df_func() -> pd.DataFrame:
-            return models.Endpoint.heatmap_doses_df(self.assessment, published_only=not unpublished)
+            return models.Endpoint.heatmap_doses_df(self.assessment, published_only=published_only)
 
         df = cacheable(df_func, key)
         return FlatExport.api_response(
@@ -158,16 +157,14 @@ def df_func() -> pd.DataFrame:
     )
     def endpoints(self, request, pk):
         self.assessment = self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-bioassay-endpoint-list"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-bioassay-endpoint-list-unpublished-{not published_only}"
 
         def df_func() -> pd.DataFrame:
             return models.Endpoint.objects.endpoint_df(
-                self.assessment, published_only=not unpublished
+                self.assessment, published_only=published_only
             )
 
         df = cacheable(df_func, key)

hawc/apps/animal/managers.py

@@ -178,16 +178,18 @@ def selector(self) -> QuerySet:
             .only("id")  # https://code.djangoproject.com/ticket/30052
         )
 
+    def published_only(self, published_only=True):
+        return (
+            self.filter(animal_group__experiment__study__published=True) if published_only else self
+        )
+
 
 class EndpointManager(BaseManager):
     assessment_relation = "assessment"
 
     def get_queryset(self):
         return EndpointQuerySet(self.model, using=self._db)
 
-    def published(self, assessment_id=None):
-        return self.get_qs(assessment_id).filter(animal_group__experiment__study__published=True)
-
     def optimized_qs(self, **filters):
         return (
             self.filter(**filters)

hawc/apps/assessment/templates/assessment/assessment_downloads.html

@@ -20,12 +20,12 @@ <h3 class="card-header">Literature Review</h3>
           {% else %}
           <li class="list-group-item">
             {% url 'lit:api:assessment-reference-export' assessment.pk as the_url %}
-            {% include "assessment/fragments/downloads_select.html" with link=the_url format="xlsx" name='references' button_text="Reference Export" help_text="All references and tags." %}
+            {% include "assessment/fragments/downloads_select.html" with link=the_url format="xlsx" name='references' button_text="Reference Export" help_text="All references and tags." allow_unpublished=False %}
           </li>
           {% if obj_perms.edit and assessment.literature_settings.conflict_resolution %}
           <li class="list-group-item">
             {% url 'lit:api:assessment-user-tag-export' assessment.pk as the_url %}
-            {% include "assessment/fragments/downloads_select.html" with link=the_url format="xlsx" name='references-usertags' button_text='User Tag Reference Export' help_text="All references and tags, including user tags. Team members only." %}
+            {% include "assessment/fragments/downloads_select.html" with link=the_url format="xlsx" name='references-usertags' button_text='User Tag Reference Export' help_text="All references and tags, including user tags. Team members only." allow_unpublished=False %}
           </li>
           {% endif %}
           {% endif %}

hawc/apps/assessment/templates/assessment/fragments/downloads_select.html

@@ -1,6 +1,6 @@
 <div class="input-group my-2" style="width: fit-content;" role="group" aria-label="Downlink link button group with format select">
     <div class="input-group-prepend">
-        <a class="btn btn-primary" style="line-height: 1.5rem;" type="button" id="{{name}}-url" href="{{link}}?format={{format}}"><i class="fa fa-download" aria-hidden="true"></i>&nbsp;{{button_text}}</a>
+        <a class="btn btn-primary" style="line-height: 1.5rem;" type="button" id="{{name}}-url" href="{{link}}?format={{format}}{% if allow_unpublished %}&unpublished=false{% endif %}"><i class="fa fa-download" aria-hidden="true"></i>&nbsp;{{button_text}}</a>
     </div>
     <select id="{{name}}" class="custom-select" required>
         <option selected value="xlsx">xlsx</option>
@@ -10,5 +10,13 @@
         <option value="html">html</option>
     </select>
 </div>
+{% if allow_unpublished %}
+<div class="form-check">
+    <input class="form-check-input unpublished-checkbox" type="checkbox" data-url="{{name}}-url">
+    <label class="form-check-label" for="{{name}}-checkbox">
+        Include unpublished data
+    </label>
+</div>
+{% endif %}
 <p class="text-muted">{{help_text}}</p>
 <p><small><i class="text-muted" id="{{name}}-fmt-text">Excel Spreadsheet</i></small></p>

hawc/apps/assessment/templates/assessment/fragments/downloads_select_js.html

@@ -1,15 +1,25 @@
 <script type="text/javascript">
-$(document).ready(function(){
-  $('{{name_list}}').on('change', function () {
-    var format = $(this).val(),
+  $(document).ready(function () {
+    $('{{name_list}}').on('change', function () {
+      var format = $(this).val(),
         url = $(`#${this.id}-url`).attr("href");
-    if (format && url) {
-      var format_idx = url.indexOf("format="),
+      if (format && url) {
+        var format_idx = url.indexOf("format="),
           help_text = format === "xlsx" ? "Excel Spreadsheet" : `${format.toUpperCase()} file`;
-      $(`#${this.id}-url`).attr("href", url.replace(/format=(\w+)/, `format=${format}`));
-      $(`#${this.id}-fmt-text`).text(help_text)
-    }
-    return false;
+        $(`#${this.id}-url`).attr("href", url.replace(/format=(\w+)/, `format=${format}`));
+        $(`#${this.id}-fmt-text`).text(help_text)
+      }
+      return false;
+    });
+    $(`.unpublished-checkbox`).on('change', function () {
+      var unpublished = $(this).prop('checked'),
+        url = $(`#${this.dataset.url}`).attr("href");
+      if (url) {
+        $(`#${this.dataset.url}`).attr(
+          "href", url.replace(/unpublished=(\w+)/, `unpublished=${unpublished}`
+          ));
+      }
+      return false;
+    });
   });
-});
 </script>

hawc/apps/assessment/views.py

@@ -361,6 +361,9 @@ def get_context_data(self, **kwargs):
         kwargs.update(
             EpiVersion=constants.EpiVersion,
         )
+        kwargs["allow_unpublished"] = self.assessment.user_is_team_member_or_higher(
+            self.request.user
+        )
         return super().get_context_data(**kwargs)
 
 

hawc/apps/common/api/utils.py

@@ -15,7 +15,7 @@ def get_published_only(assessment: Assessment, request: Request) -> bool:
         request (Request): a DRF request instance
     """
     unpublished_requested = request.query_params.get("unpublished", "").lower() == "true"
-    can_edit = assessment.user_can_edit_object(request.user)
+    can_edit = assessment.user_is_team_member_or_higher(request.user)
     if unpublished_requested and not can_edit:
         raise PermissionDenied("You must be part of the team to view unpublished data")
     return not (can_edit and unpublished_requested)

hawc/apps/common/serializers.py

@@ -67,7 +67,11 @@ class UnusedSerializer(serializers.Serializer):
     pass
 
 
-class HeatmapQuerySerializer(serializers.Serializer):
+class ExportQuerySerializer(serializers.Serializer):
+    """
+    Serializer for exports that may or may not include unpublished data.
+    """
+
     unpublished = serializers.BooleanField(default=False)
 
 

hawc/apps/epi/api.py

@@ -15,10 +15,10 @@
 from ..assessment.constants import AssessmentViewSetPermissions
 from ..assessment.models import Assessment, DSSTox
 from ..assessment.serializers import AssessmentSerializer
-from ..common.api import ReadWriteSerializerMixin
+from ..common.api import ReadWriteSerializerMixin, get_published_only
 from ..common.helper import FlatExport, cacheable
 from ..common.renderers import PandasRenderers
-from ..common.serializers import HeatmapQuerySerializer, UnusedSerializer
+from ..common.serializers import ExportQuerySerializer, UnusedSerializer
 from . import exports, models, serializers
 from .actions.model_metadata import EpiAssessmentMetadata
 
@@ -27,12 +27,6 @@ class EpiAssessmentViewSet(BaseAssessmentViewSet):
     model = Assessment
     serializer_class = UnusedSerializer
 
-    def get_outcome_queryset(self):
-        perms = self.assessment.user_permissions(self.request.user)
-        if not perms["edit"]:
-            return models.Outcome.objects.published(self.assessment)
-        return models.Outcome.objects.get_qs(self.assessment)
-
     @action(
         detail=True,
         url_path="export",
@@ -42,11 +36,19 @@ def get_outcome_queryset(self):
     def export(self, request, pk):
         """
         Retrieve epidemiology data for assessment.
+
+        By default only shows data from published studies. If the query param `unpublished=true`
+        is present then results from all studies are shown.
         """
         self.get_object()
-        exporter = exports.OutcomeComplete(
-            self.get_outcome_queryset(), filename=f"{self.assessment}-epi"
-        )
+        ser = ExportQuerySerializer(data=request.query_params)
+        ser.is_valid(raise_exception=True)
+        published_only = get_published_only(self.assessment, request)
+        if published_only:
+            qs = models.Outcome.objects.published(self.assessment)
+        else:
+            qs = models.Outcome.objects.get_qs(self.assessment)
+        exporter = exports.OutcomeComplete(qs, filename=f"{self.assessment}-epi")
         return Response(exporter.build_export())
 
     @action(
@@ -63,14 +65,12 @@ def study_heatmap(self, request, pk):
         is present then results from all studies are shown.
         """
         self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-epi-study-heatmap-pub-{unpublished}"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-epi-study-heatmap-unpub-{not published_only}"
         df = cacheable(
-            lambda: models.Result.heatmap_study_df(self.assessment, published_only=not unpublished),
+            lambda: models.Result.heatmap_study_df(self.assessment, published_only=published_only),
             key,
         )
         return FlatExport.api_response(df=df, filename=f"epi-study-heatmap-{self.assessment.id}")
@@ -89,14 +89,12 @@ def result_heatmap(self, request, pk):
         is present then results from all studies are shown.
         """
         self.get_object()
-        ser = HeatmapQuerySerializer(data=request.query_params)
+        ser = ExportQuerySerializer(data=request.query_params)
         ser.is_valid(raise_exception=True)
-        unpublished = ser.data["unpublished"]
-        if unpublished and not self.assessment.user_is_reviewer_or_higher(self.request.user):
-            raise PermissionDenied("You must be part of the team to view unpublished data")
-        key = f"assessment-{self.assessment.id}-epi-result-heatmap-pub-{unpublished}"
+        published_only = get_published_only(self.assessment, request)
+        key = f"assessment-{self.assessment.id}-epi-result-heatmap-unpub-{not published_only}"
         df = cacheable(
-            lambda: models.Result.heatmap_df(self.assessment.id, published_only=not unpublished),
+            lambda: models.Result.heatmap_df(self.assessment.id, published_only=published_only),
             key,
         )
         return FlatExport.api_response(df=df, filename=f"epi-result-heatmap-{self.assessment.id}")