Fix pip when using Py2 on Windows with non-ASCII hostname or username…

… (originally pypa#3970 from @pekkaklarck)

CHANGES.txt

@@ -41,6 +41,9 @@
 
 * Added pip completion support for fish shell.
 
+* Fix problems on Windows on Python 2 when username or hostname contains
+  non-ASCII characters (:issue:`3463`, :pull:`3970`).
+
 * Use git fetch --tags to fetch tags in addition to everything else that
   is normally fetched; this is necessary in case a git requirement url
   points to a tag or commit that is not on a branch (:pull:`3791`)

pip/utils/appdirs.py

@@ -8,6 +8,7 @@
 import sys
 
 from pip.compat import WINDOWS, expanduser
+from pip._vendor.six import PY2, text_type
 
 
 def user_cache_dir(appname):
@@ -35,6 +36,11 @@ def user_cache_dir(appname):
         # Get the base path
         path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
 
+        # When using Python 2, return paths as bytes on Windows like we do on
+        # other operating systems. See helper function docs for more details.
+        if PY2 and isinstance(path, text_type):
+            path = _win_path_to_bytes(path)
+
         # Add our app name and Cache directory to it
         path = os.path.join(path, appname, "Cache")
     elif sys.platform == "darwin":
@@ -222,3 +228,21 @@ def _get_win_folder_with_ctypes(csidl_name):
         _get_win_folder = _get_win_folder_with_ctypes
     except ImportError:
         _get_win_folder = _get_win_folder_from_registry
+
+
+def _win_path_to_bytes(path):
+    """Encode Windows paths to bytes. Only used on Python 2.
+
+    Motivation is to be consistent with other operating systems where paths
+    are also returned as bytes. This avoids problems mixing bytes and Unicode
+    elsewhere in the codebase. For more details and discussion see
+    <https://github.com/pypa/pip/issues/3463>.
+
+    If encoding using ASCII and MBCS fails, return the original Unicode path.
+    """
+    for encoding in ('ASCII', 'MBCS'):
+        try:
+            return path.encode(encoding)
+        except (UnicodeEncodeError, LookupError):
+            pass
+    return path