Replace smart quotes in form

pyxform/tests/test_expected_output/attribute_columns_test.xml

@@ -327,7 +327,7 @@
             <value>-</value>
           </text>
           <text id="/attribute_columns_test/everything/repeat_test/group_test/select_multiple_test:jr:constraintMsg">
-            <value>對不起 <output value=" /attribute_columns_test/everything/my_name "/>，你可以不選擇“是”和“否”。</value>
+            <value>對不起 <output value=" /attribute_columns_test/everything/my_name "/>，你可以不選擇"是"和"否"。</value>
           </text>
           <text id="/attribute_columns_test/skip_to_end:label">
             <value>-</value>

pyxform/tests/test_expected_output/flat_xlsform_test.xml

@@ -300,7 +300,7 @@
             <value>-</value>
           </text>
           <text id="/flat_xlsform_test/select_multiple_test:jr:constraintMsg">
-            <value>對不起 <output value=" /flat_xlsform_test/my_name "/>，你可以不選擇“是”和“否”。</value>
+            <value>對不起 <output value=" /flat_xlsform_test/my_name "/>，你可以不選擇"是"和"否"。</value>
           </text>
           <text id="/flat_xlsform_test/autocomplete_chars_test:label">
             <value>-</value>

pyxform/tests/test_expected_output/xlsform_spec_test.xml

@@ -333,7 +333,7 @@
             <value>-</value>
           </text>
           <text id="/xlsform_spec_test/everything/repeat_test/group_test/select_multiple_test:jr:constraintMsg">
-            <value>對不起 <output value=" /xlsform_spec_test/everything/my_name "/>，你可以不選擇“是”和“否”。</value>
+            <value>對不起 <output value=" /xlsform_spec_test/everything/my_name "/>，你可以不選擇"是"和"否"。</value>
           </text>
           <text id="/xlsform_spec_test/everything/number_label:label">
             <value>-</value>

pyxform/tests/test_output/flat_xlsform_test.xml

@@ -300,7 +300,7 @@
             <value>-</value>
           </text>
           <text id="/flat_xlsform_test/select_multiple_test:jr:constraintMsg">
-            <value>對不起 <output value=" /flat_xlsform_test/my_name "/>，你可以不選擇“是”和“否”。</value>
+            <value>對不起 <output value=" /flat_xlsform_test/my_name "/>，你可以不選擇&quot;是&quot;和&quot;否&quot;。</value>
           </text>
           <text id="/flat_xlsform_test/autocomplete_chars_test:label">
             <value>-</value>

pyxform/tests/test_output/xlsform_spec_test.xml

@@ -333,7 +333,7 @@
             <value>-</value>
           </text>
           <text id="/xlsform_spec_test/everything/repeat_test/group_test/select_multiple_test:jr:constraintMsg">
-            <value>對不起 <output value=" /xlsform_spec_test/everything/my_name "/>，你可以不選擇“是”和“否”。</value>
+            <value>對不起 <output value=" /xlsform_spec_test/everything/my_name "/>，你可以不選擇&quot;是&quot;和&quot;否&quot;。</value>
           </text>
           <text id="/xlsform_spec_test/everything/number_label:label">
             <value>-</value>

pyxform/tests_v1/test_unicode_rtl.py

@@ -4,7 +4,7 @@
 from pyxform.tests_v1.pyxform_test_case import PyxformTestCase
 
 
-class UnicodeRtl(PyxformTestCase):
+class UnicodeStrings(PyxformTestCase):
     def test_unicode_snowman(self):
         self.assertPyxformXform(
             md="""
@@ -17,3 +17,44 @@ def test_unicode_snowman(self):
                 '<label>☃</label>',
             ],
         )
+
+    def test_smart_quotes(self):
+        self.assertPyxformXform(
+            ss_structure={
+                'survey': [
+                    {'type': 'select_one xyz',
+                     'name': 'smart_single_quoted',
+                     'label': u'''
+                     ‘single-quoted’
+                     '''.strip()},
+                    {'type': 'text',
+                     'name': 'smart_double_quoted',
+                     'relevant': 'selected(${smart_single_quoted}, ‘xxx’)',
+                     'label': u'''
+                     “double-quoted”
+                     '''.strip()},
+                ],
+                'choices': [
+                    {'list_name': 'xyz',
+                     'name': 'xxx',
+                     'label': '‘Xxx’'},
+                    {'list_name': 'xyz',
+                     'name': 'yyy',
+                     'label': '“Yyy”'},
+                ],
+                'settings': [{'version': 'q(‘-’)p'}],
+            },
+            errored=False,
+            validate=False,
+            name="quoth",
+            xml__contains=[
+                "'single-quoted",
+                '"double-quoted"',
+                "selected( /quoth/smart_single_quoted , 'xxx')",
+                "<label>'Xxx'</label>",
+                '<label>"Yyy"</label>',
+                '''
+                version="q('-')p"
+                '''.strip(),
+            ],
+        )

pyxform/xls2json.py

@@ -1,7 +1,7 @@
 """
 A Python script to convert excel files into JSON.
 """
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 import json
 import re
 import sys
@@ -12,6 +12,13 @@
 from pyxform.xls2json_backends import xls_to_dict, csv_to_dict
 from pyxform.utils import is_valid_xml_tag, unicode, basestring
 
+SMART_QUOTES = {
+    '\u2018': "'",
+    '\u2019': "'",
+    '\u201c': '"',
+    '\u201d': '"',
+}
+
 
 def print_pyobj_to_json(pyobj, path=None):
     """
@@ -66,6 +73,17 @@ def list_to_nested_dict(lst):
         return lst[0]
 
 
+def replace_smart_quotes_in_dict(_d):
+    for key, value in _d.items():
+        _changed = False
+        for smart_quote, dumb_quote in SMART_QUOTES.items():
+            if smart_quote in value:
+                value = value.replace(smart_quote, dumb_quote)
+                _changed = True
+        if _changed:
+            _d[key] = value
+
+
 def dealias_and_group_headers(dict_array, header_aliases, use_double_colons,
                               default_language=u"default", ignore_case=False):
     """
@@ -139,6 +157,7 @@ def clean_text_values(dict_array):
     Note that the keys don't get cleaned, which could be an issue.
     """
     for row in dict_array:
+        replace_smart_quotes_in_dict(row)
         for key, value in row.items():
             if isinstance(value, basestring):
                 row[key] = re.sub(r"( )+", " ", value.strip())
@@ -292,6 +311,7 @@ def workbook_to_json(
         workbook_dict.get(constants.SETTINGS, []),
         aliases.settings_header, use_double_colons)
     settings = settings_sheet[0] if len(settings_sheet) > 0 else {}
+    replace_smart_quotes_in_dict(settings)
 
     default_language = settings.get(
         constants.DEFAULT_LANGUAGE, default_language)
@@ -337,6 +357,9 @@ def workbook_to_json(
         use_double_colons, default_language)
 
     choices_sheet = workbook_dict.get(constants.CHOICES, [])
+    for choice_item in choices_sheet:
+        replace_smart_quotes_in_dict(choice_item)
+
     choices_sheet = dealias_and_group_headers(
         choices_sheet, aliases.list_header, use_double_colons,
         default_language)