ARROW-4637: [Python] Conditionally import pandas symbols if they are used. Do not require pandas as a test dependency

Warning: hold your nose for this one =) I think this can be made cleaner, but I just wanted to triage all the problems and make sure we don't introduce a hard dependency on pandas again

Also resolves ARROW-4794: make pandas an optional dependency

Author: Wes McKinney <[email protected]>

Closes #3893 from wesm/ARROW-4637 and squashes the following commits:

3c353b6 <Wes McKinney> do not override orc global mark in test_orc.py
59fdf8a <Wes McKinney> Tweak pyarrow._orc import to see if it fixes MSVC
747c893 <Wes McKinney> Address Python 2.7 unicode interaction issue with Cython
4d94b2e <Wes McKinney> Add benchmark from ARROW-4629
f7bf774 <Wes McKinney> Cythonize pandas API shim for better performance
4a2549a <Wes McKinney> Remove TF testing from travis_script_manylinux1.sh
5061c6e <Wes McKinney> Do not require pandas to run unit tests
385cfe5 <Wes McKinney> Finish pandas API shim; do not eagerly import pandas, add to CI
804587a <Wes McKinney> add import test script
bb0240e <Wes McKinney> Begin to refactor to make references to pandas more centralized and lazy

Author: wesm

ci/travis_script_manylinux.sh

@@ -59,10 +59,9 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
   conda activate $CONDA_ENV_DIR
 
   # install the produced wheels
-  pip install tensorflow
   pip install dist/*.whl
 
-  # Test optional dependencies and the presence of tensorflow
+  # Test optional dependencies
   python check_imports.py
 
   # Install test dependencies and run pyarrow tests

ci/travis_script_python.sh

@@ -166,6 +166,9 @@ python -c "import pyarrow.parquet"
 python -c "import pyarrow.plasma"
 python -c "import pyarrow.orc"
 
+# Ensure we do eagerly import pandas (or other expensive imports)
+python < scripts/test_imports.py
+
 echo "PLASMA_VALGRIND: $PLASMA_VALGRIND"
 
 # Set up huge pages for plasma test

docs/source/developers/python.rst

@@ -360,8 +360,6 @@ Now, we build and install Arrow C++ libraries
    set ARROW_HOME=C:\thirdparty
    cmake -G "Visual Studio 14 2015 Win64" ^
          -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^
-         -DCMAKE_BUILD_TYPE=Release ^
-         -DARROW_BUILD_TESTS=on ^
          -DARROW_CXXFLAGS="/WX /MP" ^
          -DARROW_GANDIVA=on ^
          -DARROW_PARQUET=on ^
@@ -380,7 +378,9 @@ Now, we can build pyarrow:
 .. code-block:: shell
 
    cd python
-   python setup.py build_ext --inplace --with-parquet
+   set PYARROW_WITH_GANDIVA=1
+   set PYARROW_WITH_PARQUET=1
+   python setup.py build_ext --inplace
 
 Then run the unit tests with:
 

python/benchmarks/convert_pandas.py

@@ -105,3 +105,17 @@ def time_serialize_pandas(self):
 
     def time_deserialize_pandas(self):
         pa.deserialize_pandas(self.serialized)
+
+
+class TableFromPandasMicroperformance(object):
+    # ARROW-4629
+
+    def setup(self):
+        ser = pd.Series(range(10000))
+        df = pd.DataFrame({col: ser.copy(deep=True) for col in range(100)})
+        # Simulate a real dataset by converting some columns to strings
+        self.df = df.astype({col: str for col in range(50)})
+
+    def time_Table_from_pandas(self):
+        for _ in range(50):
+            pa.Table.from_pandas(self.df, nthreads=1)

python/pyarrow/array.pxi

@@ -15,11 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.compat import HAVE_PANDAS
-
-if HAVE_PANDAS:
-    import pyarrow.pandas_compat as pdcompat
-
 
 cdef _sequence_to_array(object sequence, object mask, object size,
                         DataType type, CMemoryPool* pool, c_bool from_pandas):
@@ -46,12 +41,10 @@ cdef _sequence_to_array(object sequence, object mask, object size,
         return pyarrow_wrap_chunked_array(out)
 
 
-cdef _is_array_like(obj):
-    try:
-        import pandas
-        return isinstance(obj, (np.ndarray, pd.Series, pd.Index, Categorical))
-    except ImportError:
-        return isinstance(obj, np.ndarray)
+cdef inline _is_array_like(obj):
+    if isinstance(obj, np.ndarray):
+        return True
+    return pandas_api._have_pandas_internal() and pandas_api.is_array_like(obj)
 
 
 def _ndarray_to_arrow_type(object values, DataType type):
@@ -163,15 +156,15 @@ def array(object obj, type=None, mask=None, size=None, bint from_pandas=False,
 
         values = get_series_values(obj)
 
-        if isinstance(values, Categorical):
+        if pandas_api.is_categorical(values):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, ordered=values.ordered,
                 from_pandas=True, safe=safe,
                 memory_pool=memory_pool)
         else:
-            if HAVE_PANDAS:
-                values, type = pdcompat.get_datetimetz_type(
+            if pandas_api.have_pandas:
+                values, type = pandas_api.compat.get_datetimetz_type(
                     values, obj.dtype, type)
             return _ndarray_to_array(values, mask, type, from_pandas, safe,
                                      pool)
@@ -852,9 +845,10 @@ cdef wrap_array_output(PyObject* output):
     cdef object obj = PyObject_to_object(output)
 
     if isinstance(obj, dict):
-        return Categorical(obj['indices'],
-                           categories=obj['dictionary'],
-                           ordered=obj['ordered'], fastpath=True)
+        return pandas_api.categorical_type(obj['indices'],
+                                           categories=obj['dictionary'],
+                                           ordered=obj['ordered'],
+                                           fastpath=True)
     else:
         return obj
 
@@ -1385,11 +1379,11 @@ cdef dict _array_classes = {
 
 
 cdef object get_series_values(object obj):
-    if isinstance(obj, PandasSeries):
+    if pandas_api.is_series(obj):
         result = obj.values
     elif isinstance(obj, np.ndarray):
         result = obj
     else:
-        result = PandasSeries(obj).values
+        result = pandas_api.make_series(obj).values
 
     return result

python/pyarrow/compat.py

@@ -17,7 +17,6 @@
 
 # flake8: noqa
 
-from distutils.version import LooseVersion
 import itertools
 
 import numpy as np
@@ -31,38 +30,6 @@
 PY26 = sys.version_info[:2] == (2, 6)
 PY2 = sys.version_info[0] == 2
 
-try:
-    import pandas as pd
-    pdver = LooseVersion(pd.__version__)
-    if pdver >= '0.20.0':
-        from pandas.api.types import DatetimeTZDtype
-        pdapi = pd.api.types
-    elif pdver >= '0.19.0':
-        from pandas.types.dtypes import DatetimeTZDtype
-        pdapi = pd.api.types
-    else:
-        from pandas.types.dtypes import DatetimeTZDtype
-        pdapi = pd.core.common
-
-    PandasSeries = pd.Series
-    Categorical = pd.Categorical
-    HAVE_PANDAS = True
-except:
-    HAVE_PANDAS = False
-    class DatetimeTZDtype(object):
-        pass
-
-    class ClassPlaceholder(object):
-
-        def __init__(self, *args, **kwargs):
-            raise NotImplementedError
-
-    class PandasSeries(ClassPlaceholder):
-        pass
-
-    class Categorical(ClassPlaceholder):
-        pass
-
 
 if PY26:
     import unittest2 as unittest

python/pyarrow/feather.py

@@ -15,31 +15,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from distutils.version import LooseVersion
 import os
 
 import six
-import pandas as pd
 
-from pyarrow.compat import pdapi
+from pyarrow.pandas_compat import _pandas_api  # noqa
 from pyarrow.lib import FeatherError  # noqa
 from pyarrow.lib import Table, concat_tables
 import pyarrow.lib as ext
 
 
-try:
-    infer_dtype = pdapi.infer_dtype
-except AttributeError:
-    infer_dtype = pd.lib.infer_dtype
-
-
-if LooseVersion(pd.__version__) < '0.17.0':
-    raise ImportError("feather requires pandas >= 0.17.0")
+def _check_pandas_version():
+    if _pandas_api.loose_version < '0.17.0':
+        raise ImportError("feather requires pandas >= 0.17.0")
 
 
 class FeatherReader(ext.FeatherReader):
 
     def __init__(self, source):
+        _check_pandas_version()
         self.source = source
         self.open(source)
 
@@ -80,12 +74,13 @@ def check_chunked_overflow(col):
 class FeatherWriter(object):
 
     def __init__(self, dest):
+        _check_pandas_version()
         self.dest = dest
         self.writer = ext.FeatherWriter()
         self.writer.open(dest)
 
     def write(self, df):
-        if isinstance(df, pd.SparseDataFrame):
+        if isinstance(df, _pandas_api.pd.SparseDataFrame):
             df = df.to_dense()
 
         if not df.columns.is_unique:
@@ -114,6 +109,7 @@ class FeatherDataset(object):
         Check that individual file schemas are all the same / compatible
     """
     def __init__(self, path_or_paths, validate_schema=True):
+        _check_pandas_version()
         self.paths = path_or_paths
         self.validate_schema = validate_schema
 

python/pyarrow/lib.pyx

@@ -19,13 +19,15 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from collections import OrderedDict
 import datetime
 import decimal as _pydecimal
+import json
 import multiprocessing
 import numpy as np
 import os
 import six
-from pyarrow.compat import frombytes, tobytes, PandasSeries, Categorical
+from pyarrow.compat import frombytes, tobytes
 
 from cython.operator cimport dereference as deref
 from pyarrow.includes.libarrow cimport *
@@ -90,6 +92,9 @@ Type_MAP = _Type_MAP
 UnionMode_SPARSE = _UnionMode_SPARSE
 UnionMode_DENSE = _UnionMode_DENSE
 
+# pandas API shim
+include "pandas-shim.pxi"
+
 # Exception types
 include "error.pxi"
 

python/pyarrow/orc.py

@@ -18,9 +18,9 @@
 from itertools import count
 from numbers import Integral
 
-from pyarrow import _orc
 from pyarrow import types
 from pyarrow.lib import Schema
+import pyarrow._orc as _orc
 
 
 def _is_map(typ):