From 0c96ae9c24c52ba2c8d6cae6d97d194a33e7245c Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Fri, 11 Jan 2019 15:04:53 -0500 Subject: [PATCH 1/8] Try to coerce all objects to Numpy arrays. --- _plotly_utils/basevalidators.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 388732ec159..2a23e97a8f2 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -350,7 +350,14 @@ def validate_coerce(self, v): elif is_simple_array(v): v = to_scalar_or_list(v) else: - self.raise_invalid_val(v) + # Try to coerce 'v' into an array. Useful if 'v' is a list-like object, such as a + # PyTorch tensor, an xarray Dataframe, etc that knows how to turn itself into a Numpy array. + v_array = np.array(v) + # If np.array returns a scalar object, then it failed to coerce it to a list-like object. + if v_array.shape == (): + self.raise_invalid_val(v) + else: + return self.validate_coerce(v_array) return v From fdef64cc97f427e18ccdd80f8ae0ac66d0fd3f97 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Mon, 14 Jan 2019 12:28:21 -0500 Subject: [PATCH 2/8] Move conversion logic to copy_to_readonly_numpy_array --- _plotly_utils/basevalidators.py | 36 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 2a23e97a8f2..942e583429c 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -101,16 +101,19 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): else: # DatetimeIndex v = v.to_pydatetime() - if not isinstance(v, np.ndarray): - # v is not homogenous array - v_list = [to_scalar_or_list(e) for e in v] + # v has its own logic on how to convert itself into a numpy array + if is_numpy_convertable(v): + new_v = np.array(v) + else: + # v is not homogenous array + v_list = [to_scalar_or_list(e) for e in v] - # Lookup dtype for requested kind, if any - dtype = kind_default_dtypes.get(first_kind, None) + # Lookup dtype for requested kind, if any + dtype = kind_default_dtypes.get(first_kind, None) - # construct new array from list - new_v = np.array(v_list, order='C', dtype=dtype) + # construct new array from list + new_v = np.array(v_list, order='C', dtype=dtype) elif v.dtype.kind in numeric_kinds: # v is a homogenous numeric array if kind and v.dtype.kind not in kind: @@ -148,11 +151,19 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): return new_v +def is_numpy_convertable(v): + """ + Return whether a value is meaningfully convertable to a numpy array + via 'numpy.array' + """ + return hasattr(v, '__array__') or hasattr(v, '__array_interface__') + + def is_homogeneous_array(v): """ Return whether a value is considered to be a homogeneous array """ - return ((np and isinstance(v, np.ndarray)) or + return ((np and (isinstance(v, np.ndarray) or is_numpy_convertable(v))) or (pd and isinstance(v, (pd.Series, pd.Index)))) @@ -350,14 +361,7 @@ def validate_coerce(self, v): elif is_simple_array(v): v = to_scalar_or_list(v) else: - # Try to coerce 'v' into an array. Useful if 'v' is a list-like object, such as a - # PyTorch tensor, an xarray Dataframe, etc that knows how to turn itself into a Numpy array. - v_array = np.array(v) - # If np.array returns a scalar object, then it failed to coerce it to a list-like object. - if v_array.shape == (): - self.raise_invalid_val(v) - else: - return self.validate_coerce(v_array) + self.raise_invalid_val(v) return v From 51f13afa0095be86727ef0f1067ce9bc44736a7d Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 15 Jan 2019 14:59:58 -0500 Subject: [PATCH 3/8] Don't convert numpy scalars to arrays. --- _plotly_utils/basevalidators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 942e583429c..b2981b8b492 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -156,13 +156,15 @@ def is_numpy_convertable(v): Return whether a value is meaningfully convertable to a numpy array via 'numpy.array' """ + if np.isscalar(v): # Scalar types like numpy.float64 shouldn't count as arrays + return False return hasattr(v, '__array__') or hasattr(v, '__array_interface__') def is_homogeneous_array(v): """ Return whether a value is considered to be a homogeneous array - """ + """ return ((np and (isinstance(v, np.ndarray) or is_numpy_convertable(v))) or (pd and isinstance(v, (pd.Series, pd.Index)))) From fe089b818f2a03d8d328a3d756cbb90ef35a6477 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 15 Jan 2019 21:00:11 -0500 Subject: [PATCH 4/8] Update tests. --- .circleci/create_conda_optional_env.sh | 2 +- _plotly_utils/basevalidators.py | 30 +++-- .../tests/validators/test_xarray_input.py | 126 ++++++++++++++++++ optional-requirements.txt | 2 +- tox.ini | 3 +- 5 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 _plotly_utils/tests/validators/test_xarray_input.py diff --git a/.circleci/create_conda_optional_env.sh b/.circleci/create_conda_optional_env.sh index 81a4f990e2b..d26a6cbdfb3 100755 --- a/.circleci/create_conda_optional_env.sh +++ b/.circleci/create_conda_optional_env.sh @@ -16,7 +16,7 @@ if [ ! -d $HOME/miniconda/envs/circle_optional ]; then # Create environment # PYTHON_VERSION=3.6 $HOME/miniconda/bin/conda create -n circle_optional --yes python=$PYTHON_VERSION \ -requests six pytz retrying psutil pandas decorator pytest mock nose poppler +requests six pytz retrying psutil pandas decorator pytest mock nose poppler xarray # Install orca into environment $HOME/miniconda/bin/conda install --yes -n circle_optional -c plotly plotly-orca diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index b2981b8b492..38f4abbb66d 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -43,12 +43,16 @@ def fullmatch(regex, string, flags=0): # Utility functions # ----------------- def to_scalar_or_list(v): + if np.isscalar(v) and hasattr(v, 'item'): + return np.asscalar(v) if isinstance(v, (list, tuple)): return [to_scalar_or_list(e) for e in v] elif np and isinstance(v, np.ndarray): return [to_scalar_or_list(e) for e in v] elif pd and isinstance(v, (pd.Series, pd.Index)): return [to_scalar_or_list(e) for e in v] + elif is_numpy_convertable(v): + return to_scalar_or_list(np.array(v)) else: return v @@ -104,7 +108,7 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): if not isinstance(v, np.ndarray): # v has its own logic on how to convert itself into a numpy array if is_numpy_convertable(v): - new_v = np.array(v) + return copy_to_readonly_numpy_array(np.array(v), kind=kind, force_numeric=force_numeric) else: # v is not homogenous array v_list = [to_scalar_or_list(e) for e in v] @@ -156,8 +160,6 @@ def is_numpy_convertable(v): Return whether a value is meaningfully convertable to a numpy array via 'numpy.array' """ - if np.isscalar(v): # Scalar types like numpy.float64 shouldn't count as arrays - return False return hasattr(v, '__array__') or hasattr(v, '__array_interface__') @@ -165,8 +167,17 @@ def is_homogeneous_array(v): """ Return whether a value is considered to be a homogeneous array """ - return ((np and (isinstance(v, np.ndarray) or is_numpy_convertable(v))) or - (pd and isinstance(v, (pd.Series, pd.Index)))) + if ((np and isinstance(v, np.ndarray) or + (pd and isinstance(v, (pd.Series, pd.Index))))): + return True + if is_numpy_convertable(v): + v_numpy = np.array(v) + # v is essentially a scalar and so shouldn't count as an array + if v_numpy.shape == (): + return False + else: + return True + return False def is_simple_array(v): @@ -936,7 +947,6 @@ def validate_coerce(self, v): # Pass None through pass elif self.array_ok and is_array(v): - # If strict, make sure all elements are strings. if self.strict: invalid_els = [e for e in v if not isinstance(e, string_types)] @@ -1110,13 +1120,12 @@ def validate_coerce(self, v, should_raise=True): # Pass None through pass elif self.array_ok and is_homogeneous_array(v): - - v_array = copy_to_readonly_numpy_array(v) + v = copy_to_readonly_numpy_array(v) if (self.numbers_allowed() and - v_array.dtype.kind in ['u', 'i', 'f']): + v.dtype.kind in ['u', 'i', 'f']): + pass # Numbers are allowed and we have an array of numbers. # All good - v = v_array else: validated_v = [ self.validate_coerce(e, should_raise=False) @@ -1570,7 +1579,6 @@ def validate_coerce(self, v): # Pass None through pass elif self.array_ok and is_array(v): - # Coerce individual strings validated_v = [self.vc_scalar(e) for e in v] diff --git a/_plotly_utils/tests/validators/test_xarray_input.py b/_plotly_utils/tests/validators/test_xarray_input.py new file mode 100644 index 00000000000..18ddea9c375 --- /dev/null +++ b/_plotly_utils/tests/validators/test_xarray_input.py @@ -0,0 +1,126 @@ +import pytest +import numpy as np +import xarray +import datetime +from _plotly_utils.basevalidators import (NumberValidator, + IntegerValidator, + DataArrayValidator, + ColorValidator) + + +@pytest.fixture +def data_array_validator(request): + return DataArrayValidator('prop', 'parent') + + +@pytest.fixture +def integer_validator(request): + return IntegerValidator('prop', 'parent', array_ok=True) + + +@pytest.fixture +def number_validator(request): + return NumberValidator('prop', 'parent', array_ok=True) + + +@pytest.fixture +def color_validator(request): + return ColorValidator('prop', 'parent', array_ok=True, colorscale_path='') + + +@pytest.fixture( + params=['int8', 'int16', 'int32', 'int64', + 'uint8', 'uint16', 'uint32', 'uint64', + 'float16', 'float32', 'float64']) +def numeric_dtype(request): + return request.param + + +@pytest.fixture( + params=[xarray.DataArray]) +def xarray_type(request): + return request.param + + +@pytest.fixture +def numeric_xarray(request, xarray_type, numeric_dtype): + return xarray_type(np.arange(10, dtype=numeric_dtype)) + + +@pytest.fixture +def color_object_xarray(request, xarray_type): + return xarray_type(['blue', 'green', 'red']*3) + + +def test_numeric_validator_numeric_xarray(number_validator, numeric_xarray): + res = number_validator.validate_coerce(numeric_xarray) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + assert res.dtype == numeric_xarray.dtype + + # Check values + np.testing.assert_array_equal(res, numeric_xarray) + + +def test_integer_validator_numeric_xarray(integer_validator, numeric_xarray): + res = integer_validator.validate_coerce(numeric_xarray) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + if numeric_xarray.dtype.kind in ('u', 'i'): + # Integer and unsigned integer dtype unchanged + assert res.dtype == numeric_xarray.dtype + else: + # Float datatypes converted to default integer type of int32 + assert res.dtype == 'int32' + + # Check values + np.testing.assert_array_equal(res, numeric_xarray) + + +def test_data_array_validator(data_array_validator, + numeric_xarray): + res = data_array_validator.validate_coerce(numeric_xarray) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + assert res.dtype == numeric_xarray.dtype + + # Check values + np.testing.assert_array_equal(res, numeric_xarray) + + +def test_color_validator_numeric(color_validator, + numeric_xarray): + res = color_validator.validate_coerce(numeric_xarray) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + assert res.dtype == numeric_xarray.dtype + + # Check values + np.testing.assert_array_equal(res, numeric_xarray) + + +def test_color_validator_object(color_validator, + color_object_xarray): + + res = color_validator.validate_coerce(color_object_xarray) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + assert res.dtype == 'object' + + # Check values + np.testing.assert_array_equal(res, color_object_xarray) diff --git a/optional-requirements.txt b/optional-requirements.txt index 4c6843877ff..f037d594dc6 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -17,7 +17,7 @@ mock==2.0.0 nose==1.3.3 pytest==3.5.1 backports.tempfile==1.0 - +xarray ## orca ## psutil diff --git a/tox.ini b/tox.ini index aaa92637ccf..08a447701d4 100644 --- a/tox.ini +++ b/tox.ini @@ -72,6 +72,7 @@ deps= optional: pyshp==1.2.10 optional: pillow==5.2.0 optional: matplotlib==2.2.3 + optional: xarray==0.11.2 ; CORE ENVIRONMENTS [testenv:py27-core] @@ -177,4 +178,4 @@ commands= basepython={env:PLOTLY_TOX_PYTHON_37:} commands= python --version - nosetests {posargs} -x plotly/tests/test_plot_ly \ No newline at end of file + nosetests {posargs} -x plotly/tests/test_plot_ly From 78dce4059cd1e0aa078a98a94353b23af953c70c Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 15 Jan 2019 21:03:19 -0500 Subject: [PATCH 5/8] Tweaks --- _plotly_utils/basevalidators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 38f4abbb66d..2774dc89600 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -947,6 +947,7 @@ def validate_coerce(self, v): # Pass None through pass elif self.array_ok and is_array(v): + # If strict, make sure all elements are strings. if self.strict: invalid_els = [e for e in v if not isinstance(e, string_types)] @@ -1123,9 +1124,9 @@ def validate_coerce(self, v, should_raise=True): v = copy_to_readonly_numpy_array(v) if (self.numbers_allowed() and v.dtype.kind in ['u', 'i', 'f']): - pass # Numbers are allowed and we have an array of numbers. # All good + pass else: validated_v = [ self.validate_coerce(e, should_raise=False) @@ -1579,6 +1580,7 @@ def validate_coerce(self, v): # Pass None through pass elif self.array_ok and is_array(v): + # Coerce individual strings validated_v = [self.vc_scalar(e) for e in v] From 322f3a7d7347675dc5e65dd8c8278d0f7cab22a5 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 15 Jan 2019 21:07:39 -0500 Subject: [PATCH 6/8] Check for numpy --- _plotly_utils/basevalidators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 2774dc89600..685dd2ffb09 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -43,7 +43,7 @@ def fullmatch(regex, string, flags=0): # Utility functions # ----------------- def to_scalar_or_list(v): - if np.isscalar(v) and hasattr(v, 'item'): + if np and np.isscalar(v) and hasattr(v, 'item'): return np.asscalar(v) if isinstance(v, (list, tuple)): return [to_scalar_or_list(e) for e in v] From ef0706314ca4b9219a2abc50301318482239446f Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 15 Jan 2019 21:22:35 -0500 Subject: [PATCH 7/8] Downgrade xarray version --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 08a447701d4..511f1973ff3 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,7 @@ deps= optional: pyshp==1.2.10 optional: pillow==5.2.0 optional: matplotlib==2.2.3 - optional: xarray==0.11.2 + optional: xarray==0.10.9 ; CORE ENVIRONMENTS [testenv:py27-core] From affd1151d1e994081539dc3e0109e8a879c9a951 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Wed, 16 Jan 2019 13:46:48 -0500 Subject: [PATCH 8/8] Add comment. --- _plotly_utils/basevalidators.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 685dd2ffb09..c24cb00f133 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -43,6 +43,16 @@ def fullmatch(regex, string, flags=0): # Utility functions # ----------------- def to_scalar_or_list(v): + # Handle the case where 'v' is a non-native scalar-like type, + # such as numpy.float32. Without this case, the object might be + # considered numpy-convertable and therefore promoted to a + # 0-dimensional array, but we instead want it converted to a + # Python native scalar type ('float' in the example above). + # We explicitly check if is has the 'item' method, which conventionally + # converts these types to native scalars. This guards against 'v' already being + # a Python native scalar type since `numpy.isscalar` would return + # True but `numpy.asscalar` will (oddly) raise an error is called with a + # a native Python scalar object. if np and np.isscalar(v) and hasattr(v, 'item'): return np.asscalar(v) if isinstance(v, (list, tuple)):