Merge pull request #1997 from kobotoolbox/1607-spss-export

Integrate with formpack SPSS label export feature
kobotoolbox · Sep 19, 2018 · ba0257b · ba0257b
2 parents 6156c6c + 28a09d5
commit ba0257b
Show file tree

Hide file tree

Showing 8 changed files with 157 additions and 53 deletions.
diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt
@@ -5,7 +5,7 @@
 #    pip-compile --output-file dependencies/pip/dev_requirements.txt dependencies/pip/dev_requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@d87364f951eb17f321957469a189f8dcb3eab5d1#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@ab8104cbf01f86f9ee29e75d7de41ba2ed1a07c8#egg=formpack
 -e git+https://github.com/kobotoolbox/pyxform.git@2.018.19#egg=pyxform
 amqp==2.1.4
 anyjson==0.3.3
@@ -61,7 +61,7 @@ jsonschema==2.6.0
 kombu==4.0.2
 lxml==4.2.1
 markdown==2.6.8
-mock==2.0.0               # via responses
+mock==2.0.0
 ndg-httpsclient==0.4.2
 oauthlib==1.1.2
 paramiko==2.1.1           # via fabric

diff --git a/dependencies/pip/external_services.txt b/dependencies/pip/external_services.txt
@@ -5,7 +5,7 @@
 #    pip-compile --output-file dependencies/pip/external_services.txt dependencies/pip/external_services.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@d87364f951eb17f321957469a189f8dcb3eab5d1#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@ab8104cbf01f86f9ee29e75d7de41ba2ed1a07c8#egg=formpack
 -e git+https://github.com/kobotoolbox/pyxform.git@2.018.19#egg=pyxform
 amqp==1.4.9
 anyjson==0.3.3
@@ -61,7 +61,7 @@ jsonschema==2.6.0
 kombu==3.0.35
 lxml==4.2.1
 markdown==2.6.6
-mock==2.0.0               # via responses
+mock==2.0.0
 ndg-httpsclient==0.4.2
 newrelic==2.84.0.64
 oauthlib==1.0.3

diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in
@@ -5,7 +5,7 @@
 -e git+https://github.com/kobotoolbox/pyxform.git@2.018.19#egg=pyxform
 
 # Formpack
--e git+https://github.com/kobotoolbox/formpack.git@d87364f951eb17f321957469a189f8dcb3eab5d1#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@ab8104cbf01f86f9ee29e75d7de41ba2ed1a07c8#egg=formpack
 
 # More up-to-date version of django-digest than PyPI seems to have.
 # Also, python-digest is an unlisted dependency thereof.
@@ -51,6 +51,7 @@ gunicorn
 jsonfield
 kombu
 lxml
+mock
 oauthlib
 psycopg2
 pymongo

diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt
@@ -5,7 +5,7 @@
 #    pip-compile --output-file dependencies/pip/requirements.txt dependencies/pip/requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
--e git+https://github.com/kobotoolbox/formpack.git@d87364f951eb17f321957469a189f8dcb3eab5d1#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@ab8104cbf01f86f9ee29e75d7de41ba2ed1a07c8#egg=formpack
 -e git+https://github.com/kobotoolbox/pyxform.git@2.018.19#egg=pyxform
 amqp==1.4.9
 anyjson==0.3.3
@@ -60,7 +60,7 @@ jsonschema==2.6.0
 kombu==3.0.35
 lxml==4.2.1
 markdown==2.6.6
-mock==2.0.0               # via responses
+mock==2.0.0
 ndg-httpsclient==0.4.2
 oauthlib==1.0.3
 path.py==11.0.1

diff --git a/jsapp/js/components/formEditors.es6 b/jsapp/js/components/formEditors.es6
@@ -872,7 +872,7 @@ export class ProjectDownloads extends React.Component {
       let url = this.props.asset.deployment__data_download_links[
         this.state.type
       ];
-      if (this.state.type == 'xls' || this.state.type == 'csv') {
+      if (['xls', 'csv', 'spss_labels'].includes(this.state.type)) {
         url = `${dataInterface.rootUrl}/exports/`; // TODO: have the backend pass the URL in the asset
         let postData = {
           source: this.props.asset.url,
@@ -1020,42 +1020,44 @@ export class ProjectDownloads extends React.Component {
                         <option value='spss_labels'>{t('SPSS Labels')}</option>
                       </select>
                     </bem.FormModal__item>
-                  , this.state.type == 'xls' || this.state.type == 'csv' ? [
-                      <bem.FormModal__item key={'x'} m='export-format'>
-                        <label htmlFor='lang'>{t('Value and header format')}</label>
-                        <select name='lang' value={this.state.lang}
-                            onChange={this.langChange}>
-                          <option value='xml'>{t('XML values and headers')}</option>
-                          { translations.length < 2 &&
-                            <option value='_default'>{t('Labels')}</option>
-                          }
-                          {
-                            translations && translations.map((t, i) => {
-                              if (t) {
-                                return <option value={t} key={i}>{t}</option>;
-                              }
-                            })
-                          }
-                        </select>
-                      </bem.FormModal__item>,
-                      <bem.FormModal__item key={'h'} m='export-group-headers'>
-                        <input type='checkbox' id='hierarchy_in_labels'
-                          value={this.state.hierInLabels}
-                          onChange={this.hierInLabelsChange}
-                        />
-                        <label htmlFor='hierarchy_in_labels'>
-                          {t('Include groups in headers')}
-                        </label>
-                      </bem.FormModal__item>,
-                      this.state.hierInLabels ?
-                        <bem.FormModal__item key={'g'}>
-                          <label htmlFor='group_sep'>{t('Group separator')}</label>
-                          <input type='text' name='group_sep'
-                            value={this.state.groupSep}
-                            onChange={this.groupSepChange}
+                  , ['xls', 'csv', 'spss_labels'].includes(this.state.type) ? [
+                      ['xls', 'csv'].includes(this.state.type) ? [
+                        <bem.FormModal__item key={'x'} m='export-format'>
+                          <label htmlFor='lang'>{t('Value and header format')}</label>
+                          <select name='lang' value={this.state.lang}
+                              onChange={this.langChange}>
+                            <option value='xml'>{t('XML values and headers')}</option>
+                            { translations.length < 2 &&
+                              <option value='_default'>{t('Labels')}</option>
+                            }
+                            {
+                              translations && translations.map((t, i) => {
+                                if (t) {
+                                  return <option value={t} key={i}>{t}</option>;
+                                }
+                              })
+                            }
+                          </select>
+                        </bem.FormModal__item>,
+                        <bem.FormModal__item key={'h'} m='export-group-headers'>
+                          <input type='checkbox' id='hierarchy_in_labels'
+                            value={this.state.hierInLabels}
+                            onChange={this.hierInLabelsChange}
                           />
-                        </bem.FormModal__item>
-                      : null,
+                          <label htmlFor='hierarchy_in_labels'>
+                            {t('Include groups in headers')}
+                          </label>
+                        </bem.FormModal__item>,
+                        this.state.hierInLabels ?
+                          <bem.FormModal__item key={'g'}>
+                            <label htmlFor='group_sep'>{t('Group separator')}</label>
+                            <input type='text' name='group_sep'
+                              value={this.state.groupSep}
+                              onChange={this.groupSepChange}
+                            />
+                          </bem.FormModal__item>
+                        : null,
+                      ] : null,
                       dvcount > 1 ?
                         <bem.FormModal__item key={'v'} m='export-fields-from-all-versions'>
                           <input type='checkbox' id='fields_from_all_versions'
@@ -1106,7 +1108,7 @@ export class ProjectDownloads extends React.Component {
                       <bem.FormView__group m='items' key={item.uid}
                         className={timediff < 45 ? 'recent' : ''}>
                         <bem.FormView__label m='type'>
-                          {item.data.type}
+                          {item.data.type == 'spss_labels' ? 'spss' : item.data.type}
                         </bem.FormView__label>
                         <bem.FormView__label m='date'>
                           {formatTime(item.date_created)}

diff --git a/kpi/deployment_backends/kobocat_backend.py b/kpi/deployment_backends/kobocat_backend.py
@@ -403,7 +403,6 @@ def get_data_download_links(self):
             # For GET requests that return files directly
             'xls': u'/'.join((reports_base_url, 'export.xlsx')),
             'csv': u'/'.join((reports_base_url, 'export.csv')),
-            'spss_labels': u'/'.join((forms_base_url, 'spss_labels.zip')),
         }
         return links
 

diff --git a/kpi/models/import_export_task.py b/kpi/models/import_export_task.py
@@ -32,6 +32,14 @@
                           remove_string_prefix
 
 
+def utcnow(*args, **kwargs):
+    '''
+    Stupid, and exists only to facilitate mocking during unit testing.
+    If you know of a better way, please remove this.
+    '''
+    return datetime.datetime.utcnow()
+
+
 def _resolve_url_to_asset_or_collection(item_path):
     if item_path.startswith(('http', 'https')):
         item_path = urlparse.urlparse(item_path).path
@@ -377,14 +385,25 @@ def _fields_from_all_versions(self):
             'fields_from_all_versions', 'true'
         ).lower() == 'true'
 
-    def _build_export_filename(self, export, extension):
+    def _build_export_filename(self, export, export_type):
         '''
         Internal method to build the export filename based on the export title
         (which should be set when calling the `FormPack()` constructor),
         whether the latest or all versions are included, the label language,
-        the current date and time, and the given `extension`
+        the current date and time, and the appropriate extension for the given
+        `export_type`
         '''
-        if export.lang == formpack.constants.UNTRANSLATED:
+
+        if export_type == 'xls':
+            extension = 'xlsx'
+        elif export_type == 'spss_labels':
+            extension = 'zip'
+        else:
+            extension = export_type
+
+        if export_type == 'spss_labels':
+            lang = 'SPSS Labels'
+        elif export.lang == formpack.constants.UNTRANSLATED:
             lang = 'labels'
         else:
             lang = export.lang
@@ -399,7 +418,7 @@ def _build_export_filename(self, export, extension):
             u'{{title}} - {version} - {{lang}} - {date:%Y-%m-%d-%H-%M-%S}'
             u'.{ext}'.format(
                 version=version,
-                date=datetime.datetime.utcnow(),
+                date=utcnow(),
                 ext=extension
             )
         )
@@ -490,9 +509,9 @@ def _run_task(self, messages):
             raise Exception('the source must be deployed prior to export')
 
         export_type = self.data.get('type', '').lower()
-        if export_type not in ('xls', 'csv'):
+        if export_type not in ('xls', 'csv', 'spss_labels'):
             raise NotImplementedError(
-                'only `xls` and `csv` are valid export types')
+                'only `xls`, `csv`, and `spss_labels` are valid export types')
 
         # Take this opportunity to do some housekeeping
         self.log_and_mark_stuck_as_errored(self.user, source_url)
@@ -514,8 +533,7 @@ def _run_task(self, messages):
 
         options = self._build_export_options(pack)
         export = pack.export(**options)
-        extension = 'xlsx' if export_type == 'xls' else export_type
-        filename = self._build_export_filename(export, extension)
+        filename = self._build_export_filename(export, export_type)
         self.result.save(filename, ContentFile(''))
         # FileField files are opened read-only by default and must be
         # closed and reopened to allow writing
@@ -545,6 +563,8 @@ def _run_task(self, messages):
                             break
                     '''
                     output_file.write(xlsx_output_file.read())
+            elif export_type == 'spss_labels':
+                export.to_spss_labels(output_file)
 
         # Restore the FileField to its typical state
         self.result.open('rb')

diff --git a/kpi/tests/test_mock_data_exports.py b/kpi/tests/test_mock_data_exports.py
@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 import os
+import mock
 import xlrd
+import zipfile
 import datetime
 import unittest
 from collections import defaultdict
@@ -331,6 +333,86 @@ def test_xls_export_english_labels(self):
             self.assertEqual(result_row, expected_row)
             row_index += 1
 
+    def test_export_spss_labels(self):
+        export_task = ExportTask()
+        export_task.user = self.user
+        export_task.data = {
+            'source': reverse('asset-detail', args=[self.asset.uid]),
+            'type': 'spss_labels',
+        }
+        messages = defaultdict(list)
+        # Set the current date and time artificially to generate a predictable
+        # file name for the export
+        utcnow = datetime.datetime.utcnow()
+        with mock.patch('kpi.models.import_export_task.utcnow') as mock_utcnow:
+            mock_utcnow.return_value = utcnow
+            export_task._run_task(messages)
+        self.assertFalse(messages)
+        self.assertEqual(
+            os.path.split(export_task.result.name)[-1],
+            'Identificaci\xf3n de animales - all versions - SPSS Labels - '
+            '{date:%Y-%m-%d-%H-%M-%S}.zip'.format(date=utcnow)
+        )
+        expected_file_names_and_content_lines = {
+            'Identificaci\xf3n de animales - Spanish - SPSS labels.sps': [
+                '\ufeffVARIABLE LABELS',
+                " start 'start'",
+                " /end 'end'",
+                " /What_kind_of_symmetry_do_you_have '\xbfQu\xe9 tipo de simetr\xeda tiene?'",
+                " /What_kind_of_symmetry_do_you_have_spherical '\xbfQu\xe9 tipo de simetr\xeda tiene? :: Esf\xe9rico'",
+                " /What_kind_of_symmetry_do_you_have_radial '\xbfQu\xe9 tipo de simetr\xeda tiene? :: Radial'",
+                " /What_kind_of_symmetry_do_you_have_bilateral '\xbfQu\xe9 tipo de simetr\xeda tiene? :: Bilateral'",
+                " /How_many_segments_does_your_body_have '\xbfCu\xe1ntos segmentos tiene tu cuerpo?'",
+                " /Do_you_have_body_flu_intracellular_space '\xbfTienes fluidos corporales que ocupan espacio intracelular?'",
+                " /Do_you_descend_from_unicellular_organism '\xbfDesciende de un organismo unicelular ancestral?'",
+                " /_id '_id'",
+                " /_uuid '_uuid'",
+                " /_submission_time '_submission_time'",
+                ' .',
+                'VALUE LABELS',
+                ' Do_you_have_body_flu_intracellular_space',
+                " 'yes' 'S\xed'",
+                " 'yes__and_some_' 'S\xed, y alg\xfan espacio extracelular'",
+                " 'no___unsure' 'No / Inseguro'",
+                ' /Do_you_descend_from_unicellular_organism',
+                " 'yes' 'S\xed'",
+                " 'no' 'No'",
+                ' .'
+            ],
+            'Identificaci\xf3n de animales - English - SPSS labels.sps': [
+                '\ufeffVARIABLE LABELS',
+                " start 'start'",
+                " /end 'end'",
+                " /What_kind_of_symmetry_do_you_have 'What kind of symmetry do you have?'",
+                " /What_kind_of_symmetry_do_you_have_spherical 'What kind of symmetry do you have? :: Spherical'",
+                " /What_kind_of_symmetry_do_you_have_radial 'What kind of symmetry do you have? :: Radial'",
+                " /What_kind_of_symmetry_do_you_have_bilateral 'What kind of symmetry do you have? :: Bilateral'",
+                " /How_many_segments_does_your_body_have 'How many segments does your body have?'",
+                " /Do_you_have_body_flu_intracellular_space 'Do you have body fluids that occupy intracellular space?'",
+                " /Do_you_descend_from_unicellular_organism 'Do you descend from an ancestral unicellular organism?'",
+                " /_id '_id'",
+                " /_uuid '_uuid'",
+                " /_submission_time '_submission_time'",
+                ' .',
+                'VALUE LABELS',
+                ' Do_you_have_body_flu_intracellular_space',
+                " 'yes' 'Yes'",
+                " 'yes__and_some_' 'Yes, and some extracellular space'",
+                " 'no___unsure' 'No / Unsure'",
+                ' /Do_you_descend_from_unicellular_organism',
+                " 'yes' 'Yes'",
+                " 'no' 'No'",
+                ' .'
+            ],
+        }
+        result_zip = zipfile.ZipFile(export_task.result, 'r')
+        for name, content_lines in expected_file_names_and_content_lines.items():
+            self.assertEqual(
+                # we have `unicode_literals` but the rest of the app doesn't
+                result_zip.open(name, 'r').read().decode('utf-8'),
+                '\r\n'.join(content_lines)
+            )
+
     def test_remove_excess_exports(self):
         task_data = {
             'source': reverse('asset-detail', args=[self.asset.uid]),