Skip to content

Commit

Permalink
iterblocks: groupby_value (2nd step)
Browse files Browse the repository at this point in the history
  • Loading branch information
Francesc Elies Henar committed Mar 10, 2015
1 parent dcb01e7 commit af1eaf6
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 32 deletions.
24 changes: 8 additions & 16 deletions bquery/ctable_ext.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -772,9 +772,8 @@ cdef sum_int64(carray ca_input, carray ca_factor,
@cython.boundscheck(False)
cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_ssize_t skip_key):
cdef:
chunk factor_chunk
Py_ssize_t in_buffer_len
Py_ssize_t factor_chunk_nr, factor_chunk_len, factor_chunk_row
Py_ssize_t in_buffer_len, factor_buffer_len
Py_ssize_t factor_chunk_nr, factor_chunk_row
Py_ssize_t current_index, i, factor_total_chunks

ndarray in_buffer
Expand All @@ -784,16 +783,13 @@ cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_s
count = 0
ret = 0
reverse = {}
iter_ca_factor = bz.iterblocks(ca_factor)


factor_chunk_len = ca_factor.chunklen
factor_total_chunks = ca_factor.nchunks
factor_chunk_nr = 0
factor_buffer = np.empty(factor_chunk_len, dtype='int64')
if factor_total_chunks > 0:
factor_chunk = ca_factor.chunks[factor_chunk_nr]
factor_chunk._getitem(0, factor_chunk_len, factor_buffer.data)
else:
factor_buffer = ca_factor.leftover_array
factor_buffer = iter_ca_factor.next()
factor_buffer_len = len(factor_buffer)
factor_chunk_row = 0
out_buffer = np.zeros(nr_groups, dtype=ca_input.dtype)

Expand All @@ -803,13 +799,9 @@ cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_s
for i in range(in_buffer_len):

# go to next factor buffer if necessary
if factor_chunk_row == factor_chunk_len:
if factor_chunk_row == factor_buffer_len:
factor_chunk_nr += 1
if factor_chunk_nr < factor_total_chunks:
factor_chunk = ca_factor.chunks[factor_chunk_nr]
factor_chunk._getitem(0, factor_chunk_len, factor_buffer.data)
else:
factor_buffer = ca_factor.leftover_array
factor_buffer = iter_ca_factor.next()
factor_chunk_row = 0

# retrieve index
Expand Down
24 changes: 8 additions & 16 deletions bquery/templates/ctable_ext.template.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,9 +417,8 @@ cdef sum_{{ sum_type }}(carray ca_input, carray ca_factor,
@cython.boundscheck(False)
cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_ssize_t skip_key):
cdef:
chunk factor_chunk
Py_ssize_t in_buffer_len
Py_ssize_t factor_chunk_nr, factor_chunk_len, factor_chunk_row
Py_ssize_t in_buffer_len, factor_buffer_len
Py_ssize_t factor_chunk_nr, factor_chunk_row
Py_ssize_t current_index, i, factor_total_chunks

ndarray in_buffer
Expand All @@ -429,16 +428,13 @@ cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_s
count = 0
ret = 0
reverse = {}
iter_ca_factor = bz.iterblocks(ca_factor)


factor_chunk_len = ca_factor.chunklen
factor_total_chunks = ca_factor.nchunks
factor_chunk_nr = 0
factor_buffer = np.empty(factor_chunk_len, dtype='int64')
if factor_total_chunks > 0:
factor_chunk = ca_factor.chunks[factor_chunk_nr]
factor_chunk._getitem(0, factor_chunk_len, factor_buffer.data)
else:
factor_buffer = ca_factor.leftover_array
factor_buffer = iter_ca_factor.next()
factor_buffer_len = len(factor_buffer)
factor_chunk_row = 0
out_buffer = np.zeros(nr_groups, dtype=ca_input.dtype)

Expand All @@ -448,13 +444,9 @@ cdef groupby_value(carray ca_input, carray ca_factor, Py_ssize_t nr_groups, Py_s
for i in range(in_buffer_len):

# go to next factor buffer if necessary
if factor_chunk_row == factor_chunk_len:
if factor_chunk_row == factor_buffer_len:
factor_chunk_nr += 1
if factor_chunk_nr < factor_total_chunks:
factor_chunk = ca_factor.chunks[factor_chunk_nr]
factor_chunk._getitem(0, factor_chunk_len, factor_buffer.data)
else:
factor_buffer = ca_factor.leftover_array
factor_buffer = iter_ca_factor.next()
factor_chunk_row = 0

# retrieve index
Expand Down

0 comments on commit af1eaf6

Please sign in to comment.