Skip to content

Commit

Permalink
release GIL on median
Browse files Browse the repository at this point in the history 
release GIL on is_lexsorted / fix memory leak
release GIL on nancorr
  • Loading branch information
@jreback
jreback committed Mar 22, 2017
1 parent ce28bb5 commit 4e2bfec
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 104 deletions.
30 changes: 2 additions & 28 deletions pandas/_libs/algos.pxd
View file
Original file line number Diff line number Diff line change
@@ -1,39 +1,13 @@
from util cimport numeric
from numpy cimport float64_t, double_t

cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k)
cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k) nogil

cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil except -1:
cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
cdef numeric t

# cython doesn't allow pointer dereference so use array syntax
t = a[0]
a[0] = b[0]
b[0] = t
return 0


cdef inline kth_smallest_c(float64_t* a, Py_ssize_t k, Py_ssize_t n):
cdef:
Py_ssize_t i, j, l, m
double_t x, t

l = 0
m = n -1
while (l<m):
x = a[k]
i = l
j = m

while 1:
while a[i] < x: i += 1
while x < a[j]: j -= 1
if i <= j:
swap(&a[i], &a[j])
i += 1; j -= 1

if i > j: break

if j < k: l = i
if k < i: m = j
return a[k]
140 changes: 76 additions & 64 deletions pandas/_libs/algos.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ class NegInfinity(object):
__ge__ = lambda self, other: self is other



@cython.wraparound(False)
@cython.boundscheck(False)
def is_lexsorted(list list_of_arrays):
Expand All @@ -105,28 +104,31 @@ def is_lexsorted(list list_of_arrays):
Py_ssize_t n, nlevels
int64_t k, cur, pre
ndarray arr
bint result = True

nlevels = len(list_of_arrays)
n = len(list_of_arrays[0])

cdef int64_t **vecs = <int64_t**> malloc(nlevels * sizeof(int64_t*))
for i from 0 <= i < nlevels:
for i in range(nlevels):
arr = list_of_arrays[i]
vecs[i] = <int64_t*> arr.data

# Assume uniqueness??
for i from 1 <= i < n:
for k from 0 <= k < nlevels:
cur = vecs[k][i]
pre = vecs[k][i -1]
if cur == pre:
continue
elif cur > pre:
break
else:
return False
with nogil:
for i in range(n):
for k in range(nlevels):
cur = vecs[k][i]
pre = vecs[k][i -1]
if cur == pre:
continue
elif cur > pre:
break
else:
result = False
break
free(vecs)
return True
return result


@cython.boundscheck(False)
Expand Down Expand Up @@ -159,15 +161,15 @@ def groupsort_indexer(ndarray[int64_t] index, Py_ssize_t ngroups):
with nogil:

# count group sizes, location 0 for NA
for i from 0 <= i < n:
for i in range(n):
counts[index[i] + 1] += 1

# mark the start of each contiguous group of like-indexed data
for i from 1 <= i < ngroups + 1:
for i in range(1, ngroups + 1):
where[i] = where[i - 1] + counts[i - 1]

# this is our indexer
for i from 0 <= i < n:
for i in range(n):
label = index[i] + 1
result[where[label]] = i
where[label] += 1
Expand All @@ -177,10 +179,11 @@ def groupsort_indexer(ndarray[int64_t] index, Py_ssize_t ngroups):

@cython.boundscheck(False)
@cython.wraparound(False)
cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k):
cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k) nogil:
cdef:
Py_ssize_t i, j, l, m, n = a.size
Py_ssize_t i, j, l, m, n = a.shape[0]
numeric x

with nogil:
l = 0
m = n - 1
Expand All @@ -201,7 +204,7 @@ cpdef numeric kth_smallest(numeric[:] a, Py_ssize_t k):

if j < k: l = i
if k < i: m = j
return a[k]
return a[k]


cpdef numeric median(numeric[:] arr):
Expand All @@ -224,6 +227,8 @@ cpdef numeric median(numeric[:] arr):

# -------------- Min, Max subsequence

@cython.boundscheck(False)
@cython.wraparound(False)
def max_subseq(ndarray[double_t] arr):
cdef:
Py_ssize_t i=0, s=0, e=0, T, n
Expand All @@ -238,21 +243,24 @@ def max_subseq(ndarray[double_t] arr):
S = m
T = 0

for i in range(1, n):
# S = max { S + A[i], A[i] )
if (S > 0):
S = S + arr[i]
else:
S = arr[i]
T = i
if S > m:
s = T
e = i
m = S
with nogil:
for i in range(1, n):
# S = max { S + A[i], A[i] )
if (S > 0):
S = S + arr[i]
else:
S = arr[i]
T = i
if S > m:
s = T
e = i
m = S

return (s, e, m)


@cython.boundscheck(False)
@cython.wraparound(False)
def min_subseq(ndarray[double_t] arr):
cdef:
Py_ssize_t s, e
Expand All @@ -268,9 +276,10 @@ def min_subseq(ndarray[double_t] arr):

@cython.boundscheck(False)
@cython.wraparound(False)
def nancorr(ndarray[float64_t, ndim=2] mat, cov=False, minp=None):
def nancorr(ndarray[float64_t, ndim=2] mat, bint cov=0, minp=None):
cdef:
Py_ssize_t i, j, xi, yi, N, K
bint minpv
ndarray[float64_t, ndim=2] result
ndarray[uint8_t, ndim=2] mask
int64_t nobs = 0
Expand All @@ -279,46 +288,49 @@ def nancorr(ndarray[float64_t, ndim=2] mat, cov=False, minp=None):
N, K = (<object> mat).shape

if minp is None:
minp = 1
minpv = 1
else:
minpv = <int>minp

result = np.empty((K, K), dtype=np.float64)
mask = np.isfinite(mat).view(np.uint8)

for xi in range(K):
for yi in range(xi + 1):
nobs = sumxx = sumyy = sumx = sumy = 0
for i in range(N):
if mask[i, xi] and mask[i, yi]:
vx = mat[i, xi]
vy = mat[i, yi]
nobs += 1
sumx += vx
sumy += vy

if nobs < minp:
result[xi, yi] = result[yi, xi] = np.NaN
else:
meanx = sumx / nobs
meany = sumy / nobs

# now the cov numerator
sumx = 0

with nogil:
for xi in range(K):
for yi in range(xi + 1):
nobs = sumxx = sumyy = sumx = sumy = 0
for i in range(N):
if mask[i, xi] and mask[i, yi]:
vx = mat[i, xi] - meanx
vy = mat[i, yi] - meany
vx = mat[i, xi]
vy = mat[i, yi]
nobs += 1
sumx += vx
sumy += vy

if nobs < minpv:
result[xi, yi] = result[yi, xi] = NaN
else:
meanx = sumx / nobs
meany = sumy / nobs

sumx += vx * vy
sumxx += vx * vx
sumyy += vy * vy
# now the cov numerator
sumx = 0

divisor = (nobs - 1.0) if cov else sqrt(sumxx * sumyy)
for i in range(N):
if mask[i, xi] and mask[i, yi]:
vx = mat[i, xi] - meanx
vy = mat[i, yi] - meany

if divisor != 0:
result[xi, yi] = result[yi, xi] = sumx / divisor
else:
result[xi, yi] = result[yi, xi] = np.NaN
sumx += vx * vy
sumxx += vx * vx
sumyy += vy * vy

divisor = (nobs - 1.0) if cov else sqrt(sumxx * sumyy)

if divisor != 0:
result[xi, yi] = result[yi, xi] = sumx / divisor
else:
result[xi, yi] = result[yi, xi] = NaN

return result

Expand Down Expand Up @@ -351,7 +363,7 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1):
nobs += 1

if nobs < minp:
result[xi, yi] = result[yi, xi] = np.NaN
result[xi, yi] = result[yi, xi] = NaN
else:
maskedx = np.empty(nobs, dtype=np.float64)
maskedy = np.empty(nobs, dtype=np.float64)
Expand Down Expand Up @@ -382,7 +394,7 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1):
if divisor != 0:
result[xi, yi] = result[yi, xi] = sumx / divisor
else:
result[xi, yi] = result[yi, xi] = np.NaN
result[xi, yi] = result[yi, xi] = NaN

return result

Expand Down
32 changes: 30 additions & 2 deletions pandas/_libs/groupby.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ from numpy cimport (int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
from libc.stdlib cimport malloc, free

from util cimport numeric, get_nat
from algos cimport kth_smallest, kth_smallest_c
from algos cimport swap
from algos import take_2d_axis1_float64_float64, groupsort_indexer

cdef int64_t iNaT = get_nat()
Expand Down Expand Up @@ -219,7 +219,7 @@ def group_last_bin_object(ndarray[object, ndim=2] out,
out[i, j] = resx[i, j]


cdef inline float64_t _median_linear(float64_t* a, int n):
cdef inline float64_t _median_linear(float64_t* a, int n) nogil:
cdef int i, j, na_count = 0
cdef float64_t result
cdef float64_t* tmp
Expand Down Expand Up @@ -259,5 +259,33 @@ cdef inline float64_t _median_linear(float64_t* a, int n):
return result


cdef inline float64_t kth_smallest_c(float64_t* a,
Py_ssize_t k,
Py_ssize_t n) nogil:
cdef:
Py_ssize_t i, j, l, m
double_t x, t

l = 0
m = n -1
while (l<m):
x = a[k]
i = l
j = m

while 1:
while a[i] < x: i += 1
while x < a[j]: j -= 1
if i <= j:
swap(&a[i], &a[j])
i += 1; j -= 1

if i > j: break

if j < k: l = i
if k < i: m = j
return a[k]


# generated from template
include "groupby_helper.pxi"
18 changes: 11 additions & 7 deletions pandas/_libs/groupby_helper.pxi.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,8 @@ def group_cummax_{{name}}(ndarray[{{dest_type2}}, ndim=2] out,
#----------------------------------------------------------------------


@cython.boundscheck(False)
@cython.wraparound(False)
def group_median_float64(ndarray[float64_t, ndim=2] out,
ndarray[int64_t] counts,
ndarray[float64_t, ndim=2] values,
Expand All @@ -704,13 +706,15 @@ def group_median_float64(ndarray[float64_t, ndim=2] out,

take_2d_axis1_float64_float64(values.T, indexer, out=data)

for i in range(K):
# exclude NA group
ptr += _counts[0]
for j in range(ngroups):
size = _counts[j + 1]
out[j, i] = _median_linear(ptr, size)
ptr += size
with nogil:

for i in range(K):
# exclude NA group
ptr += _counts[0]
for j in range(ngroups):
size = _counts[j + 1]
out[j, i] = _median_linear(ptr, size)
ptr += size


@cython.boundscheck(False)
Expand Down
Loading

0 comments on commit 4e2bfec

Please sign in to comment.