8000 POC Use fused types more, tempita less by jbrockmendel · Pull Request #22432 · pandas-dev/pandas · GitHub
[go: up one dir, main page]
Skip to content

POC Use fused types more, tempita less #22432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

POC Use fused types more, tempita less #22432

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9f5aed3
use fused types for group_cummin and group_cummax
jbrockmendel Aug 19, 2018
97278bc
use fused type for unstack
jbrockmendel Aug 19, 2018
7912d71
use templates
jbrockmendel Aug 19, 2018
3e1f79a
fuse more
jbrockmendel Aug 19, 2018
5a4e7fc
Merge branch 'master' of https://github.com/pandas-dev/pandas into cy…
jbrockmendel Aug 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
use fused type for unstack
  • Loading branch information
@jbrockmendel
jbrockmendel committed Aug 19, 2018
commit 97278bccdd335e1161bae8cbbcd96eff2ea48008
66 changes: 43 additions & 23 deletions pandas/_libs/algos_common_helper.pxi.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,49 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
# 1-d template
#----------------------------------------------------------------------

ctypedef fused algos_t:
float64_t
float32_t
object
int32_t
int64_t
uint64_t
uint8_t


@cython.wraparound(False)
@cython.boundscheck(False)
cpdef map_indices(ndarray[algos_t] index):
"""
Produce a dict mapping the values of the input array to their respective
locations.

Example:
array(['hi', 'there']) --> {'hi' : 0 , 'there' : 1}

Better to do this with Cython because of the enormous speed boost.
"""
cdef:
Py_ssize_t i, length
dict result = {}

length = len(index)

for i in range(length):
result[index[i]] = i

return result


map_indices_float64 = map_indices["float64_t"]
map_indices_float32 = map_indices["float32_t"]
map_indices_object = map_indices["object"]
map_indices_int32 = map_indices["int32_t"]
map_indices_int64 = map_indices["int64_t"]
map_indices_uint64 = map_indices["uint64_t"]
map_indices_uint8 = map_indices["uint8_t"]


{{py:

# name, c_type, dtype, can_hold_na, nogil
Expand All @@ -43,29 +86,6 @@ def get_dispatch(dtypes):
in get_dispatch(dtypes)}}


@cython.wraparound(False)
@cython.boundscheck(False)
cpdef map_indices_{{name}}(ndarray[{{c_type}}] index):
"""
Produce a dict mapping the values of the input array to their respective
locations.

Example:
array(['hi', 'there']) --> {'hi' : 0 , 'there' : 1}

Better to do this with Cython because of the enormous speed boost.
"""
cdef Py_ssize_t i, length
cdef dict result = {}

length = len(index)

for i in range(length):
result[index[i]] = i

return result


@cython.boundscheck(False)
@cython.wraparound(False)
def pad_{{name}}(ndarray[{{c_type}}] old, ndarray[{{c_type}}] new,
Expand Down
85 changes: 50 additions & 35 deletions pandas/_libs/reshape_helper.pxi.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,28 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
# reshape
# ----------------------------------------------------------------------

{{py:

# name, c_type
dtypes = [('uint8', 'uint8_t'),
('uint16', 'uint16_t'),
('uint32', 'uint32_t'),
('uint64', 'uint64_t'),
('int8', 'int8_t'),
('int16', 'int16_t'),
('int32', 'int32_t'),
('int64', 'int64_t'),
('float32', 'float32_t'),
('float64', 'float64_t'),
('object', 'object')]
}}

{{for dtype, c_type in dtypes}}

ctypedef fused reshape_t:
uint8_t
uint16_t
uint32_t
uint64_t
int8_t
int16_t
int32_t
int64_t
float32_t
float64_t
object

@cython.wraparound(False)
@cython.boundscheck(False)
def unstack_{{dtype}}(ndarray[{{c_type}}, ndim=2] values,
ndarray[uint8_t, ndim=1] mask,
Py_ssize_t stride,
Py_ssize_t length,
Py_ssize_t width,
ndarray[{{c_type}}, ndim=2] new_values,
ndarray[uint8_t, ndim=2] new_mask):
def unstack(ndarray[reshape_t, ndim=2] values,
ndarray[uint8_t, ndim=1] mask,
Py_ssize_t stride,
Py_ssize_t length,
Py_ssize_t width,
ndarray[reshape_t, ndim=2] new_values,
ndarray[uint8_t, ndim=2] new_mask):
"""
transform long sorted_values to wide new_values

Expand All @@ -50,23 +44,33 @@ def unstack_{{dtype}}(ndarray[{{c_type}}, ndim=2] values,
result array
new_mask : boolean ndarray
result mask

"""

cdef:
Py_ssize_t i, j, w, nulls, s, offset

{{if dtype == 'object'}}
if True:
{{else}}
with nogil:
{{endif}}
if reshape_t is not object:
with nogil:
for i in range(stride):
nulls = 0

for i in range(stride):
for j in range(length):
for w in range(width):

offset = j * width + w

if mask[offset]:
s = i * width + w
new_values[j, s] = values[offset - nulls, i]
new_mask[j, s] = 1
else:
nulls += 1

else:
# identical to above version, but "with nogil" is not available
for i in range(stride):
nulls = 0
for j in range(length):

for j in range(length):
for w in range(width):

offset = j * width + w
Expand All @@ -78,4 +82,15 @@ def unstack_{{dtype}}(ndarray[{{c_type}}, ndim=2] values,
else:
nulls += 1

{{endfor}}

unstack_uint8 = unstack["uint8_t"]
unstack_uint16 = unstack["uint16_t"]
unstack_uint32 = unstack["uint32_t"]
unstack_uint64 = unstack["uint64_t"]
unstack_int8 = unstack["int8_t"]
unstack_int16 = unstack["int16_t"]
unstack_int32 = unstack["int32_t"]
unstack_int64 = unstack["int64_t"]
unstack_float32 = unstack["float32_t"]
unstack_float64 = unstack["float64_t"]
unstack_object = unstack["object"]
0