Fix attempt for unicode bug #1137

pip/basecommand.py

@@ -7,6 +7,7 @@
 import time
 import optparse
 
+from pip._vendor import six
 from pip import cmdoptions
 from pip.locations import running_under_virtualenv
 from pip.log import logger
@@ -25,6 +26,28 @@
 __all__ = ['Command']
 
 
+# On Python 2: converts unicode objects to UTF-8, leaves bytestrings untouched
+# On Python 3: converts bytestrings to unicode, leaves str untouched
+# (we probably won't run into bytestrings on Python 3 that much -- they are no
+# longer the default type for literals)
+def to_native_str_type(s):
+    if isinstance(s, str):
+        # unicode for PY3 or bytes for PY2 -- ok
+        return s
+    elif isinstance(s, bytes):
+        # bytes and str != bytes
+        # convert to unicode for PY3
+        return s.decode('utf-8', 'replace')
+    elif isinstance(s, six.text_type):
+        # unicode and unicode != str
+        # converts to bytes for PY2
+        return s.encode('utf-8')
+    else:
+        # not a string at all
+        raise TypeError("to_utf8() expected unicode or bytes, got %r" %
+                        type(s))
+
+
 class Command(object):
     name = None
     usage = None
@@ -165,7 +188,7 @@ def main(self, args):
             exit = UNKNOWN_ERROR
         if store_log:
             log_file_fn = options.log_file
-            text = '\n'.join(complete_log)
+            text = '\n'.join(to_native_str_type(l) for l in complete_log)
             try:
                 log_file_fp = open_logfile(log_file_fn, 'w')
             except IOError:

tests/unit/test_basecommand.py

@@ -1,5 +1,5 @@
 import os
-from pip.basecommand import Command
+from pip.basecommand import Command, to_native_str_type
 from pip.log import logger
 
 
@@ -17,6 +17,15 @@ def run(self, options, args):
             raise SystemExit(1)
 
 
+class FakeCommandWithUnicode(FakeCommand):
+    name = 'fake_unicode'
+    summary = name
+
+    def run(self, options, args):
+        logger.info(b"bytes here \xE9")
+        logger.info(b"unicode here \xC3\xA9".decode('utf-8'))
+
+
 class Test_basecommand_logging(object):
     """
     Test `pip.basecommand.Command` setting up logging consumers based on
@@ -83,3 +92,18 @@ def test_verbose_quiet(self):
         cmd.main(['fake', '-vqq'])
         console_level = logger.consumers[0][0]
         assert console_level == logger.WARN
+
+    def test_unicode_messages(self, tmpdir):
+        """
+        Tests that logging bytestrings and unicode objects don't break logging
+        """
+        cmd = FakeCommandWithUnicode()
+        log_path = tmpdir.join('log')
+        cmd.main(['fake_unicode', '--log', log_path])
+
+
+def test_to_native_str_type():
+    some_bytes = b"test\xE9 et approuv\xC3\xE9"
+    some_unicode = b"test\xE9 et approuv\xE9".decode('iso-8859-15')
+    assert isinstance(to_native_str_type(some_bytes), str)
+    assert isinstance(to_native_str_type(some_unicode), str)