From b3d5209b26322329d7e4ba1fd1b1457f86b44a8a Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Tue, 6 Feb 2018 10:55:37 +0900 Subject: [PATCH] Address a comment. --- python/pyspark/serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py index 1469a44dc1caf..45f8290e5c17d 100644 --- a/python/pyspark/serializers.py +++ b/python/pyspark/serializers.py @@ -232,7 +232,8 @@ def create_array(s, t): return pa.Array.from_pandas(s, mask=mask).cast(t, safe=False) elif t is not None and pa.types.is_string(t) and sys.version < '3': # TODO: need decode before converting to Arrow in Python 2 - return pa.Array.from_pandas(s.str.decode('utf-8'), mask=mask, type=t) + return pa.Array.from_pandas(s.apply( + lambda v: v.decode("utf-8") if isinstance(v, str) else v), mask=mask, type=t) return pa.Array.from_pandas(s, mask=mask, type=t) arrs = [create_array(s, t) for s, t in series]