implement libmissing; untangles _libs dependencies

pandas/_libs/algos.pyx

@@ -32,8 +32,7 @@ from libc.math cimport sqrt, fabs
 # this is our util.pxd
 from util cimport numeric, get_nat
 
-cimport lib
-from pandas._libs import lib
+import missing
 
 cdef int64_t iNaT = get_nat()
 

pandas/_libs/algos_rank_helper.pxi.in

@@ -83,7 +83,7 @@ def rank_1d_{{dtype}}(object in_arr, ties_method='average', ascending=True,
         nan_value = {{neg_nan_value}}
 
     {{if dtype == 'object'}}
-    mask = lib.isnaobj(values)
+    mask = missing.isnaobj(values)
     {{elif dtype == 'float64'}}
     mask = np.isnan(values)
     {{elif dtype == 'int64'}}
@@ -259,7 +259,7 @@ def rank_2d_{{dtype}}(object in_arr, axis=0, ties_method='average',
         nan_value = {{neg_nan_value}}
 
     {{if dtype == 'object'}}
-    mask = lib.isnaobj2d(values)
+    mask = missing.isnaobj2d(values)
     {{elif dtype == 'float64'}}
     mask = np.isnan(values)
     {{elif dtype == 'int64'}}

pandas/_libs/hashtable.pyx

@@ -42,7 +42,7 @@ cdef extern from "numpy/npy_math.h":
 cimport cython
 cimport numpy as cnp
 
-from pandas._libs.lib import checknull
+from missing cimport checknull
 
 cnp.import_array()
 cnp.import_ufunc()

pandas/_libs/hashtable_class_helper.pxi.in

@@ -4,7 +4,7 @@ Template for each `dtype` helper function for hashtable
 WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 """
 
-from lib cimport is_null_datetimelike
+from missing cimport is_null_datetimelike
 
 
 #----------------------------------------------------------------------

pandas/_libs/lib.pxd

@@ -1,4 +1,3 @@
 # prototypes for sharing
 
-cdef bint is_null_datetimelike(v)
 cpdef bint is_period(val)

pandas/_libs/lib.pyx

@@ -53,9 +53,9 @@ PyDateTime_IMPORT
 
 from tslibs.np_datetime cimport get_timedelta64_value, get_datetime64_value
 
-from tslib cimport _check_all_nulls
 from tslib import NaT, Timestamp, Timedelta, array_to_datetime
 from interval import Interval
+from missing cimport checknull
 
 cdef int64_t NPY_NAT = util.get_nat()
 
@@ -112,54 +112,6 @@ def memory_usage_of_objects(ndarray[object, ndim=1] arr):
 
 
 # ----------------------------------------------------------------------
-# isnull / notnull related
-
-cdef double INF = <double> np.inf
-cdef double NEGINF = -INF
-
-
-cpdef bint checknull(object val):
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val  # and val != INF and val != NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is NaT:
-        return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif is_array(val):
-        return False
-    else:
-        return _checknull(val)
-
-
-cpdef bint checknull_old(object val):
-    if util.is_float_object(val) or util.is_complex_object(val):
-        return val != val or val == INF or val == NEGINF
-    elif util.is_datetime64_object(val):
-        return get_datetime64_value(val) == NPY_NAT
-    elif val is NaT:
-        return True
-    elif util.is_timedelta64_object(val):
-        return get_timedelta64_value(val) == NPY_NAT
-    elif is_array(val):
-        return False
-    else:
-        return _checknull(val)
-
-
-cpdef bint isposinf_scalar(object val):
-    if util.is_float_object(val) and val == INF:
-        return True
-    else:
-        return False
-
-
-cpdef bint isneginf_scalar(object val):
-    if util.is_float_object(val) and val == NEGINF:
-        return True
-    else:
-        return False
 
 
 cpdef bint isscalar(object val):
@@ -212,78 +164,6 @@ def item_from_zerodim(object val):
     return util.unbox_if_zerodim(val)
 
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj(ndarray arr):
-    cdef Py_ssize_t i, n
-    cdef object val
-    cdef ndarray[uint8_t] result
-
-    assert arr.ndim == 1, "'arr' must be 1-D."
-
-    n = len(arr)
-    result = np.empty(n, dtype=np.uint8)
-    for i from 0 <= i < n:
-        val = arr[i]
-        result[i] = _check_all_nulls(val)
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj_old(ndarray arr):
-    cdef Py_ssize_t i, n
-    cdef object val
-    cdef ndarray[uint8_t] result
-
-    assert arr.ndim == 1, "'arr' must be 1-D."
-
-    n = len(arr)
-    result = np.zeros(n, dtype=np.uint8)
-    for i from 0 <= i < n:
-        val = arr[i]
-        result[i] = val is NaT or util._checknull_old(val)
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj2d(ndarray arr):
-    cdef Py_ssize_t i, j, n, m
-    cdef object val
-    cdef ndarray[uint8_t, ndim=2] result
-
-    assert arr.ndim == 2, "'arr' must be 2-D."
-
-    n, m = (<object> arr).shape
-    result = np.zeros((n, m), dtype=np.uint8)
-    for i from 0 <= i < n:
-        for j from 0 <= j < m:
-            val = arr[i, j]
-            if checknull(val):
-                result[i, j] = 1
-    return result.view(np.bool_)
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def isnaobj2d_old(ndarray arr):
-    cdef Py_ssize_t i, j, n, m
-    cdef object val
-    cdef ndarray[uint8_t, ndim=2] result
-
-    assert arr.ndim == 2, "'arr' must be 2-D."
-
-    n, m = (<object> arr).shape
-    result = np.zeros((n, m), dtype=np.uint8)
-    for i from 0 <= i < n:
-        for j from 0 <= j < m:
-            val = arr[i, j]
-            if checknull_old(val):
-                result[i, j] = 1
-    return result.view(np.bool_)
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cpdef ndarray[object] list_to_object_array(list obj):

pandas/_libs/missing.pxd

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# cython: profile=False
+
+cdef bint is_null_datetimelike(v)
+cpdef bint checknull(object val)
+cpdef bint checknull_old(object val)

pandas/_libs/missing.pyx

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+# cython: profile=False
+
+from cpython cimport PyFloat_Check, PyComplex_Check
+
+cimport cython
+from cython cimport Py_ssize_t
+
+import numpy as np
+cimport numpy as np
+from numpy cimport ndarray, int64_t, uint8_t
+np.import_array()
+
+cimport util
+
+from tslibs.np_datetime cimport get_timedelta64_value, get_datetime64_value
+from tslibs.nattype import NaT, iNaT
+
+cdef double INF = <double> np.inf
+cdef double NEGINF = -INF
+
+cdef int64_t NPY_NAT = util.get_nat()
+
+
+cdef inline bint is_null_datetimelike(v):
+    # determine if we have a null for a timedelta/datetime (or integer
+    # versions)
+    if util._checknull(v):
+        return True
+    elif v is NaT:
+        return True
+    elif util.is_timedelta64_object(v):
+        return v.view('int64') == iNaT
+    elif util.is_datetime64_object(v):
+        return v.view('int64') == iNaT
+    elif util.is_integer_object(v):
+        return v == iNaT
+    return False
+
+
+cdef inline bint _check_all_nulls(object val):
+    """ utility to check if a value is any type of null """
+    cdef bint res
+    if PyFloat_Check(val) or PyComplex_Check(val):
+        res = val != val
+    elif val is NaT:
+        res = 1
+    elif val is None:
+        res = 1
+    elif util.is_datetime64_object(val):
+        res = get_datetime64_value(val) == NPY_NAT
+    elif util.is_timedelta64_object(val):
+        res = get_timedelta64_value(val) == NPY_NAT
+    else:
+        res = 0
+    return res
+
+
+cpdef bint checknull(object val):
+    if util.is_float_object(val) or util.is_complex_object(val):
+        return val != val  # and val != INF and val != NEGINF
+    elif util.is_datetime64_object(val):
+        return get_datetime64_value(val) == NPY_NAT
+    elif val is NaT:
+        return True
+    elif util.is_timedelta64_object(val):
+        return get_timedelta64_value(val) == NPY_NAT
+    elif util.is_array(val):
+        return False
+    else:
+        return util._checknull(val)
+
+
+cpdef bint checknull_old(object val):
+    if util.is_float_object(val) or util.is_complex_object(val):
+        return val != val or val == INF or val == NEGINF
+    elif util.is_datetime64_object(val):
+        return get_datetime64_value(val) == NPY_NAT
+    elif val is NaT:
+        return True
+    elif util.is_timedelta64_object(val):
+        return get_timedelta64_value(val) == NPY_NAT
+    elif util.is_array(val):
+        return False
+    else:
+        return util._checknull(val)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def isnaobj(ndarray arr):
+    cdef:
+        Py_ssize_t i, n
+        object val
+        ndarray[uint8_t] result
+
+    assert arr.ndim == 1, "'arr' must be 1-D."
+
+    n = len(arr)
+    result = np.empty(n, dtype=np.uint8)
+    for i from 0 <= i < n:
+        val = arr[i]
+        result[i] = _check_all_nulls(val)
+    return result.view(np.bool_)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def isnaobj_old(ndarray arr):
+    cdef:
+        Py_ssize_t i, n
+        object val
+        ndarray[uint8_t] result
+
+    assert arr.ndim == 1, "'arr' must be 1-D."
+
+    n = len(arr)
+    result = np.zeros(n, dtype=np.uint8)
+    for i from 0 <= i < n:
+        val = arr[i]
+        result[i] = val is NaT or util._checknull_old(val)
+    return result.view(np.bool_)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def isnaobj2d(ndarray arr):
+    cdef:
+        Py_ssize_t i, j, n, m
+        object val
+        ndarray[uint8_t, ndim=2] result
+
+    assert arr.ndim == 2, "'arr' must be 2-D."
+
+    n, m = (<object> arr).shape
+    result = np.zeros((n, m), dtype=np.uint8)
+    for i from 0 <= i < n:
+        for j from 0 <= j < m:
+            val = arr[i, j]
+            if checknull(val):
+                result[i, j] = 1
+    return result.view(np.bool_)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def isnaobj2d_old(ndarray arr):
+    cdef:
+        Py_ssize_t i, j, n, m
+        object val
+        ndarray[uint8_t, ndim=2] result
+
+    assert arr.ndim == 2, "'arr' must be 2-D."
+
+    n, m = (<object> arr).shape
+    result = np.zeros((n, m), dtype=np.uint8)
+    for i from 0 <= i < n:
+        for j from 0 <= j < m:
+            val = arr[i, j]
+            if checknull_old(val):
+                result[i, j] = 1
+    return result.view(np.bool_)
+
+
+cpdef bint isposinf_scalar(object val):
+    if util.is_float_object(val) and val == INF:
+        return True
+    else:
+        return False
+
+
+cpdef bint isneginf_scalar(object val):
+    if util.is_float_object(val) and val == NEGINF:
+        return True
+    else:
+        return False

pandas/_libs/period.pyx

@@ -25,7 +25,7 @@ from tslibs.np_datetime cimport (pandas_datetimestruct,
 cimport util
 from util cimport is_period_object, is_string_object, INT32_MIN
 
-from lib cimport is_null_datetimelike
+from missing cimport is_null_datetimelike
 from pandas._libs.tslib import Timestamp
 from tslibs.timezones cimport (
     is_utc, is_tzlocal, get_utcoffset, get_dst_info)