Skip to content

Commit

Permalink
Make IOStream methods available in Python
Browse files Browse the repository at this point in the history
  • Loading branch information
phoerious committed Mar 6, 2023
1 parent c615424 commit eb569b7
Show file tree
Hide file tree
Showing 4 changed files with 235 additions and 166 deletions.
54 changes: 13 additions & 41 deletions fastwarc/fastwarc/stream_io.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cython
from libc.stdio cimport FILE
from libc.string cimport memcpy
from libcpp.string cimport string
Expand All @@ -23,62 +23,34 @@ from resiliparse_inc.string_view cimport string_view


cdef class IOStream:
cdef size_t read(self, char* out, size_t size) except -1
cdef size_t write(self, const char* data, size_t size) except -1
cdef size_t tell(self) except -1
cdef void seek(self, size_t offset) except *
cdef void flush(self) except *
cdef void close(self) except *
cdef size_t read_(self, char* out, size_t size) except -1
cdef size_t write_(self, const char* data, size_t size) except -1
cpdef size_t tell(self) except -1
cpdef void seek(self, size_t offset) except *
cpdef void flush(self) except *
cpdef void close(self) except *


# noinspection PyAttributeOutsideInit
cdef class PythonIOStreamAdapter(IOStream):
cdef object py_stream

cdef inline size_t read(self, char* out, size_t size) except -1:
cdef bytes data = self.py_stream.read(size)
size = min(<size_t>len(data), size)
memcpy(out, <char*>data, size * sizeof(char))
return size

cdef inline size_t write(self, const char* data, size_t size) except -1:
return self.py_stream.write(data[:size])

cdef inline size_t tell(self) except -1:
return self.py_stream.tell()

cdef inline void seek(self, size_t offset) except *:
self.py_stream.seek(offset)

cdef inline void flush(self) except *:
self.py_stream.flush()

cdef inline void close(self) except *:
if self.py_stream is None:
return
try:
self.py_stream.close()
except ValueError:
# Ignore if already closed
pass


cdef class BytesIOStream(IOStream):
cdef string buffer
cdef size_t pos

cdef string getvalue(self)
cpdef string getvalue(self)


cdef class FileStream(IOStream):
cdef FILE* fp

cdef void open(self, const char* path, char* mode=*) except *
cdef void open_(self, const char* path, char* mode=*) except *


cdef class CompressingStream(IOStream):
cdef size_t begin_member(self)
cdef size_t end_member(self)
cpdef size_t begin_member(self)
cpdef size_t end_member(self)


cdef extern from * nogil:
Expand Down Expand Up @@ -107,7 +79,7 @@ cdef class GZipStream(CompressingStream):
cdef bint member_started
cdef int compression_level

cdef void prepopulate(self, bint deflate, const string& initial_data)
cpdef void prepopulate(self, bint deflate, const string& initial_data)
cdef void _init_z_stream(self, bint zlib) nogil
cdef void _free_z_stream(self) nogil
cdef void _reinit_z_stream(self, bint deflate) nogil
Expand All @@ -125,7 +97,7 @@ cdef class LZ4Stream(CompressingStream):
cdef bint frame_started
cdef size_t stream_pos

cdef void prepopulate(self, const string& initial_data)
cpdef void prepopulate(self, const string& initial_data)
cdef void _free_ctx(self) nogil


Expand Down
Loading

0 comments on commit eb569b7

Please sign in to comment.