introduce chunk._getitem_r(...), a thread-safe version of chunk._getitem(...)

bcolz/carray_ext.pxd

@@ -9,6 +9,9 @@ cdef class chunk:
     cdef object atom, constant, dobject
 
     cdef void _getitem(self, int start, int stop, char *dest)
+    cdef void _getitem_r(self, int start, int stop, char *dest, char *data, 
+                         int nbytes, int atomsize, int itemsize, 
+                         int isconstant, char *constant, int constant_nbytes) nogil
     cdef compress_data(self, char *data, size_t itemsize, size_t nbytes,
                        object cparams)
     cdef compress_arrdata(self, ndarray array, int itemsize,

bcolz/carray_ext.pyx

@@ -27,6 +27,7 @@ from numpy cimport (ndarray,
                     npy_intp,
                     )
 import cython
+from libc.stdint cimport uintptr_t
 
 import bcolz
 from bcolz import utils, attrs, array2string
@@ -464,30 +465,70 @@ cdef class chunk:
 
     cdef void _getitem(self, int start, int stop, char *dest):
         """Read data from `start` to `stop` and return it as a numpy array."""
-        cdef int ret, bsize, blen, nitems, nstart
-        cdef ndarray constants
+        cdef int ret, bsize, blen, nitems, nstart, constant_nbytes
+        cdef char * constant_ptr
+
+        constant_ptr = NULL
+        constant_nbytes = 0
+
+        if self.constant is not None:
+            constant_ptr = <char*><uintptr_t>self.constant \
+                .__array_interface__['data'][0]
+            constant_nbytes = self.constant.nbytes
+
+        with nogil:
+            self._getitem_r(start, stop, dest, 
+                            data=self.data, 
+                            nbytes=self.nbytes, 
+                            atomsize=self.atomsize, 
+                            itemsize=self.itemsize, 
+                            isconstant=self.isconstant, 
+                            constant=constant_ptr,
+                            constant_nbytes=constant_nbytes)
+
+    cdef void _getitem_r(self, int start, int stop, char *dest, char *data, 
+                         int nbytes, int atomsize, int itemsize, 
+                         int isconstant, char *constant_ptr, int constant_nbytes) nogil:
+        """Thread-safe version of _getitem(...)."""
+        cdef:
+            int ret, bsize, blen, nitems, nstart
+            char * start_ptr
+            char * current_ptr
+            char * end_ptr
 
         blen = stop - start
-        bsize = blen * self.atomsize
-        nitems = cython.cdiv(bsize, self.itemsize)
-        nstart = cython.cdiv(start * self.atomsize, self.itemsize)
+        bsize = blen * atomsize
+        nitems = cython.cdiv(bsize, itemsize)
+        nstart = cython.cdiv(start * atomsize, itemsize)
 
-        if self.isconstant:
-            # The chunk is made of constants
-            constants = np.ndarray(shape=(blen,), dtype=self.dtype,
-                                   buffer=self.constant, strides=(0,)).copy()
-            memcpy(dest, constants.data, bsize)
+        if isconstant:
+            if constant_nbytes == 1:
+                memset(dest, constant_ptr[0], bsize)
+            else:
+                # multi-byte constant: create repeating pattern of the constant
+                # by copying increasingly longer repeated sequences of the 
+                # constant until the entire destination buffer is filled
+                memcpy(dest, constant_ptr, constant_nbytes)
+                start_ptr = dest
+                current_ptr = dest + constant_nbytes
+                end_ptr = start_ptr + bsize
+                while constant_nbytes < end_ptr - current_ptr:
+                    memcpy(current_ptr, start_ptr, constant_nbytes)
+                    current_ptr += constant_nbytes
+                    constant_nbytes *= 2
+                memcpy(current_ptr, start_ptr, end_ptr-current_ptr);
             return
 
         # Fill dest with uncompressed data
-        with nogil:
-            if bsize == self.nbytes:
-                ret = blosc_decompress(self.data, dest, bsize)
-            else:
-                ret = blosc_getitem(self.data, nstart, nitems, dest)
+        if bsize == nbytes:
+            ret = blosc_decompress(data, dest, bsize)
+        else:
+            ret = blosc_getitem(data, nstart, nitems, dest)
+
         if ret < 0:
-            raise RuntimeError(
-                "fatal error during Blosc decompression: %d" % ret)
+            with gil:
+                raise RuntimeError(
+                    "fatal error during Blosc decompression: %d" % ret)
 
     def __getitem__(self, object key):
         """__getitem__(self, key) -> values."""

bcolz/definitions.pxd

@@ -24,8 +24,8 @@ cdef extern from "string.h":
     char *strncpy(char *dest, char *src, size_t n)
     int strcmp(char *s1, char *s2)
     char *strdup(char *s)
-    void *memcpy(void *dest, void *src, size_t n)
-    void *memset(void *s, int c, size_t n)
+    void *memcpy(void *dest, void *src, size_t n) nogil
+    void *memset(void *s, int c, size_t n) nogil
 
 cdef extern from "time.h":
     ctypedef int time_t