Merge pull request #2038 from kobotoolbox/1851-export-validation-stat…

…uses

Add _validation_status to exports; closes #1851

dependencies/pip/dev_requirements.txt

@@ -5,7 +5,7 @@
 #    pip-compile --output-file dependencies/pip/dev_requirements.txt dependencies/pip/dev_requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@b9f00374d65391dd456fc117d5321461b7acdc42#egg=formpack
 amqp==2.1.4
 anyjson==0.3.3
 argparse==1.4.0           # via unittest2

dependencies/pip/external_services.txt

@@ -5,7 +5,7 @@
 #    pip-compile --output-file dependencies/pip/external_services.txt dependencies/pip/external_services.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@b9f00374d65391dd456fc117d5321461b7acdc42#egg=formpack
 amqp==2.3.2
 anyjson==0.3.3
 argparse==1.4.0           # via unittest2

dependencies/pip/requirements.in

@@ -2,7 +2,7 @@
 # https://github.com/bndr/pipreqs is a handy utility, too.
 
 # Formpack
--e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@b9f00374d65391dd456fc117d5321461b7acdc42#egg=formpack
 
 # More up-to-date version of django-digest than PyPI seems to have.
 # Also, python-digest is an unlisted dependency thereof.

dependencies/pip/requirements.txt

@@ -2,10 +2,10 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements.txt requirements.in
+#    pip-compile --output-file dependencies/pip/requirements.txt dependencies/pip/requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@b9f00374d65391dd456fc117d5321461b7acdc42#egg=formpack
 amqp==2.3.2
 anyjson==0.3.3
 argparse==1.4.0           # via unittest2

jsapp/js/components/formEditors.es6

@@ -159,6 +159,25 @@ export class ProjectDownloads extends React.Component {
     dataInterface.getAssetExports(this.props.asset.uid).done((data)=>{
       if (data.count > 0) {
         data.results.reverse();
+        data.results.map(result => {
+          switch(result.data.lang) {
+            case '_default':
+            case null: // The value of `formpack.constants.UNTRANSLATED`,
+                       // which shouldn't be revealed here, but just in case...
+              result.data.langDescription = t('Default');
+              break;
+            case '_xml':
+            case false: // `formpack.constants.UNSPECIFIED_TRANSLATION`
+              // Exports previously used `xml` (no underscore) for this, which
+              // works so long as the form has no language called `xml`. In
+              // reality, we shouldn't bank on that:
+              // https://en.wikipedia.org/wiki/Malaysian_Sign_Language
+              result.data.langDescription = t('XML');
+              break;
+            default:
+              result.data.langDescription = result.data.lang;
+          }
+        });
         this.setState({exports: data.results});
 
         // Start a polling Interval if there is at least one export is not yet complete
@@ -232,7 +251,7 @@ export class ProjectDownloads extends React.Component {
                           <label htmlFor='lang'>{t('Value and header format')}</label>
                           <select name='lang' value={this.state.lang}
                               onChange={this.langChange}>
-                            <option value='xml'>{t('XML values and headers')}</option>
+                            <option value='_xml'>{t('XML values and headers')}</option>
                             { translations.length < 2 &&
                               <option value='_default'>{t('Labels')}</option>
                             }
@@ -320,7 +339,7 @@ export class ProjectDownloads extends React.Component {
                           {formatTime(item.date_created)}
                         </bem.FormView__label>
                         <bem.FormView__label m='lang'>
-                        {item.data.lang === '_default' ? t('Default') : item.data.lang}
+                          {item.data.langDescription}
                         </bem.FormView__label>
                         <bem.FormView__label m='include-groups'>
                           {item.data.hierarchy_in_labels === 'false' ? t('No') : t('Yes')}

kpi/models/import_export_task.py

@@ -22,15 +22,17 @@
 import formpack.constants
 from pyxform import xls2json_backends
 from formpack.utils.string import ellipsize
+from formpack.schema.fields import ValidationStatusCopyField
+
+from kpi.utils.log import logging
 from kobo.apps.reports.report_data import build_formpack
 
 from ..fields import KpiUidField
 from ..models import Collection, Asset
-from ..deployment_backends.mock_backend import MockDeploymentBackend
 from ..zip_importer import HttpContentParse
 from ..model_utils import create_assets, _load_library_content, \
                           remove_string_prefix
-from kpi.utils.log import logging
+from ..deployment_backends.mock_backend import MockDeploymentBackend
 
 
 def utcnow(*args, **kwargs):
@@ -342,9 +344,10 @@ class ExportTask(ImportExportTask):
     `data` attribute to a dictionary with the following keys:
     * `type`: required; `xls` or `csv`
     * `source`: required; URL of a deployed `Asset`
-    * `lang`: optional; `xml` for XML names or the name of the language to be
-              used for labels. Leave unset, or use `_default` or `None`, for
-              labels in the default language
+    * `lang`: optional; the name of the translation to be used for headers and
+              response values. Specify `_xml` to use question and choice names
+              instead of labels. Leave unset, or use `_default` for labels in
+              the default language
     * `hierarchy_in_labels`: optional; when `true`, include the labels for all
                              ancestor groups in each field label, separated by
                              `group_sep`. Defaults to `False`
@@ -375,7 +378,21 @@ class ExportTask(ImportExportTask):
     last_submission_time = models.DateTimeField(null=True)
     result = PrivateFileField(upload_to=export_upload_to, max_length=380)
 
-    COPY_FIELDS = ('_id', '_uuid', '_submission_time')
+    COPY_FIELDS = (
+        '_id',
+        '_uuid',
+        '_submission_time',
+        ValidationStatusCopyField,
+    )
+
+    # It's not very nice to ask our API users to submit `null` or `false`,
+    # so replace friendlier language strings with the constants that formpack
+    # expects
+    API_LANGUAGE_TO_FORMPACK_LANGUAGE = {
+        '_default': formpack.constants.UNTRANSLATED,
+        '_xml': formpack.constants.UNSPECIFIED_TRANSLATION,
+    }
+
     TIMESTAMP_KEY = '_submission_time'
     # Above 244 seems to cause 'Download error' in Chrome 64/Linux
     MAXIMUM_FILENAME_LENGTH = 240
@@ -445,8 +462,12 @@ def _build_export_options(self, pack):
         group_sep = self.data.get('group_sep', '/')
         translations = pack.available_translations
         lang = self.data.get('lang', None) or next(iter(translations), None)
-        if lang == '_default':
-            lang = formpack.constants.UNTRANSLATED
+        try:
+            # If applicable, substitute the constants that formpack expects for
+            # friendlier language strings used by the API
+            lang = self.API_LANGUAGE_TO_FORMPACK_LANGUAGE[lang]
+        except KeyError:
+            pass
         tag_cols_for_header = self.data.get('tag_cols_for_header', ['hxl'])
 
         return {

kpi/tests/test_api_assets.py

@@ -548,8 +548,8 @@ def test_owner_can_create_export(self):
             result_content = result_response.content
         self.assertEqual(result_response.status_code, status.HTTP_200_OK)
         expected_content = ''.join([
-            '"q1";"_id";"_uuid";"_submission_time";"_index"\r\n',
-            '"¿Qué tal?";"";"";"";"1"\r\n',
+            '"q1";"_id";"_uuid";"_submission_time";"_validation_status";"_index"\r\n',
+            '"¿Qué tal?";"";"";"";"";"1"\r\n',
         ])
         self.assertEqual(result_content, expected_content)
         return detail_response

kpi/tests/test_mock_data_conflicting_version_exports.py

@@ -67,7 +67,7 @@ def test_csv_export(self):
         export_task.data = {
             'source': reverse('asset-detail', args=[self.asset.uid]),
             'type': 'csv',
-            'lang': 'xml'
+            'lang': '_xml'
         }
         messages = defaultdict(list)
         export_task._run_task(messages)