Try to coerce all array-compatible objects to Numpy arrays. (#1393)

During validation, check for the presence of `__array_interface__` or `__array__` protocol methods and, if present, convert objects to numpy arrays.

Author: malmaud

.circleci/create_conda_optional_env.sh

@@ -16,7 +16,7 @@ if [ ! -d $HOME/miniconda/envs/circle_optional ]; then
     # Create environment
     # PYTHON_VERSION=3.6
     $HOME/miniconda/bin/conda create -n circle_optional --yes python=$PYTHON_VERSION \
-requests six pytz retrying psutil pandas decorator pytest mock nose poppler
+requests six pytz retrying psutil pandas decorator pytest mock nose poppler xarray
 
     # Install orca into environment
     $HOME/miniconda/bin/conda install --yes -n circle_optional -c plotly plotly-orca

_plotly_utils/basevalidators.py

@@ -43,12 +43,26 @@ def fullmatch(regex, string, flags=0):
 # Utility functions
 # -----------------
 def to_scalar_or_list(v):
+    # Handle the case where 'v' is a non-native scalar-like type,
+    # such as numpy.float32. Without this case, the object might be
+    # considered numpy-convertable and therefore promoted to a
+    # 0-dimensional array, but we instead want it converted to a
+    # Python native scalar type ('float' in the example above).
+    # We explicitly check if is has the 'item' method, which conventionally
+    # converts these types to native scalars. This guards against 'v' already being
+    # a Python native scalar type since  `numpy.isscalar` would return
+    # True but `numpy.asscalar` will (oddly) raise an error is called with a
+    # a native Python scalar object.
+    if np and np.isscalar(v) and hasattr(v, 'item'):
+        return np.asscalar(v)
     if isinstance(v, (list, tuple)):
         return [to_scalar_or_list(e) for e in v]
     elif np and isinstance(v, np.ndarray):
         return [to_scalar_or_list(e) for e in v]
     elif pd and isinstance(v, (pd.Series, pd.Index)):
         return [to_scalar_or_list(e) for e in v]
+    elif is_numpy_convertable(v):
+        return to_scalar_or_list(np.array(v))
     else:
         return v
 
@@ -101,16 +115,19 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
             else:
                 # DatetimeIndex
                 v = v.to_pydatetime()
-
     if not isinstance(v, np.ndarray):
-        # v is not homogenous array
-        v_list = [to_scalar_or_list(e) for e in v]
+        # v has its own logic on how to convert itself into a numpy array
+        if is_numpy_convertable(v):
+            return copy_to_readonly_numpy_array(np.array(v), kind=kind, force_numeric=force_numeric)
+        else:
+            # v is not homogenous array
+            v_list = [to_scalar_or_list(e) for e in v]
 
-        # Lookup dtype for requested kind, if any
-        dtype = kind_default_dtypes.get(first_kind, None)
+            # Lookup dtype for requested kind, if any
+            dtype = kind_default_dtypes.get(first_kind, None)
 
-        # construct new array from list
-        new_v = np.array(v_list, order='C', dtype=dtype)
+            # construct new array from list
+            new_v = np.array(v_list, order='C', dtype=dtype)
     elif v.dtype.kind in numeric_kinds:
         # v is a homogenous numeric array
         if kind and v.dtype.kind not in kind:
@@ -148,12 +165,29 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     return new_v
 
 
+def is_numpy_convertable(v):
+    """
+    Return whether a value is meaningfully convertable to a numpy array
+    via 'numpy.array'
+    """
+    return hasattr(v, '__array__') or hasattr(v, '__array_interface__')
+
+
 def is_homogeneous_array(v):
     """
     Return whether a value is considered to be a homogeneous array
-    """
-    return ((np and isinstance(v, np.ndarray)) or
-            (pd and isinstance(v, (pd.Series, pd.Index))))
+    """    
+    if ((np and isinstance(v, np.ndarray) or
+        (pd and isinstance(v, (pd.Series, pd.Index))))):
+            return True
+    if is_numpy_convertable(v):
+        v_numpy = np.array(v)
+        # v is essentially a scalar and so shouldn't count as an array
+        if v_numpy.shape == ():
+            return False
+        else:
+            return True
+    return False
 
 
 def is_simple_array(v):
@@ -1097,13 +1131,12 @@ def validate_coerce(self, v, should_raise=True):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-
-            v_array = copy_to_readonly_numpy_array(v)
+            v = copy_to_readonly_numpy_array(v)
             if (self.numbers_allowed() and
-                    v_array.dtype.kind in ['u', 'i', 'f']):
+                    v.dtype.kind in ['u', 'i', 'f']):
                 # Numbers are allowed and we have an array of numbers.
                 # All good
-                v = v_array
+                pass
             else:
                 validated_v = [
                     self.validate_coerce(e, should_raise=False)

_plotly_utils/tests/validators/test_xarray_input.py

@@ -0,0 +1,126 @@
+import pytest
+import numpy as np
+import xarray
+import datetime
+from _plotly_utils.basevalidators import (NumberValidator,
+                                          IntegerValidator,
+                                          DataArrayValidator,
+                                          ColorValidator)
+
+
+@pytest.fixture
+def data_array_validator(request):
+    return DataArrayValidator('prop', 'parent')
+
+
+@pytest.fixture
+def integer_validator(request):
+    return IntegerValidator('prop', 'parent', array_ok=True)
+
+
+@pytest.fixture
+def number_validator(request):
+    return NumberValidator('prop', 'parent', array_ok=True)
+
+
+@pytest.fixture
+def color_validator(request):
+    return ColorValidator('prop', 'parent', array_ok=True, colorscale_path='')
+
+
+@pytest.fixture(
+    params=['int8', 'int16', 'int32', 'int64',
+            'uint8', 'uint16', 'uint32', 'uint64',
+            'float16', 'float32', 'float64'])
+def numeric_dtype(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[xarray.DataArray])
+def xarray_type(request):
+    return request.param
+
+
+@pytest.fixture
+def numeric_xarray(request, xarray_type, numeric_dtype):
+    return xarray_type(np.arange(10, dtype=numeric_dtype))
+
+
+@pytest.fixture
+def color_object_xarray(request, xarray_type):
+    return xarray_type(['blue', 'green', 'red']*3)
+
+
+def test_numeric_validator_numeric_xarray(number_validator, numeric_xarray):
+    res = number_validator.validate_coerce(numeric_xarray)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_xarray.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_xarray)
+
+
+def test_integer_validator_numeric_xarray(integer_validator, numeric_xarray):
+    res = integer_validator.validate_coerce(numeric_xarray)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    if numeric_xarray.dtype.kind in ('u', 'i'):
+        # Integer and unsigned integer dtype unchanged
+        assert res.dtype == numeric_xarray.dtype
+    else:
+        # Float datatypes converted to default integer type of int32
+        assert res.dtype == 'int32'
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_xarray)
+
+
+def test_data_array_validator(data_array_validator,
+                              numeric_xarray):
+    res = data_array_validator.validate_coerce(numeric_xarray)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_xarray.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_xarray)
+
+
+def test_color_validator_numeric(color_validator,
+                                 numeric_xarray):
+    res = color_validator.validate_coerce(numeric_xarray)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == numeric_xarray.dtype
+
+    # Check values
+    np.testing.assert_array_equal(res, numeric_xarray)
+
+
+def test_color_validator_object(color_validator,
+                                color_object_xarray):
+
+    res = color_validator.validate_coerce(color_object_xarray)
+
+    # Check type
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == 'object'
+
+    # Check values
+    np.testing.assert_array_equal(res, color_object_xarray)

optional-requirements.txt

@@ -17,7 +17,7 @@ mock==2.0.0
 nose==1.3.3
 pytest==3.5.1
 backports.tempfile==1.0
-
+xarray
 ## orca ##
 psutil
 

tox.ini

@@ -72,6 +72,7 @@ deps=
     optional: pyshp==1.2.10
     optional: pillow==5.2.0
     optional: matplotlib==2.2.3
+    optional: xarray==0.10.9
 
 ; CORE ENVIRONMENTS
 [testenv:py27-core]
@@ -177,4 +178,4 @@ commands=
 basepython={env:PLOTLY_TOX_PYTHON_37:}
 commands=
     python --version
-    nosetests {posargs} -x plotly/tests/test_plot_ly
+    nosetests {posargs} -x plotly/tests/test_plot_ly