diff --git a/BUILDING.markdown b/BUILDING.markdown index 41eb6b4..9a86a2f 100644 --- a/BUILDING.markdown +++ b/BUILDING.markdown @@ -4,5 +4,5 @@ Getting started building py-yajl 1. clone this repository 2. `git submodule update --init` 3. `python setup.py build_ext --inplace` -4. `python tests.py` +4. `runtests.sh` diff --git a/decoder.c b/decoder.c index 6f78bd7..8a6e90d 100644 --- a/decoder.c +++ b/decoder.c @@ -76,7 +76,7 @@ int PlaceObject(_YajlDecoder *self, PyObject *object) * we should only be handling "primitive types" i.e. strings and * numbers, not dict/list. */ - self->root = object; + PyList_Append(self->decoded_objects,object); return success; } return _PlaceObject(self, py_yajl_ps_current(self->elements), object); @@ -94,7 +94,7 @@ static int handle_bool(void *ctx, int value) return PlaceObject(ctx, PyBool_FromLong((long)(value))); } -static int handle_number(void *ctx, const char *value, unsigned int length) +static int handle_number(void *ctx, const char *value, size_t length) { _YajlDecoder *self = (_YajlDecoder *)(ctx); PyObject *object; @@ -134,7 +134,7 @@ static int handle_number(void *ctx, const char *value, unsigned int length) return PlaceObject(self, object); } -static int handle_string(void *ctx, const unsigned char *value, unsigned int length) +static int handle_string(void *ctx, const unsigned char *value, size_t length) { return PlaceObject(ctx, PyUnicode_FromStringAndSize((char *)value, length)); } @@ -149,7 +149,7 @@ static int handle_start_dict(void *ctx) return success; } -static int handle_dict_key(void *ctx, const unsigned char *value, unsigned int length) +static int handle_dict_key(void *ctx, const unsigned char *value, size_t length) { PyObject *object = PyUnicode_FromStringAndSize((const char *) value, length); @@ -172,7 +172,7 @@ static int handle_end_dict(void *ctx) * If this is the last element in the stack * then it's "root" and we should finish up */ - self->root = py_yajl_ps_current(self->elements); + PyList_Append(self->decoded_objects,py_yajl_ps_current(self->elements)); py_yajl_ps_pop(self->elements); return success; } else if (length < 2) { @@ -209,7 +209,7 @@ static int handle_end_list(void *ctx) length = py_yajl_ps_length(self->elements); if (length == 1) { - self->root = py_yajl_ps_current(self->elements); + PyList_Append(self->decoded_objects,py_yajl_ps_current(self->elements)); py_yajl_ps_pop(self->elements); return success; } else if (length < 2) { @@ -236,54 +236,112 @@ static yajl_callbacks decode_callbacks = { handle_start_list, handle_end_list }; - +/* +Parse the contents of *buffer using the yajl json parser +*/ PyObject *_internal_decode(_YajlDecoder *self, char *buffer, unsigned int buflen) { - yajl_handle parser = NULL; yajl_status yrc; - yajl_parser_config config = { 1, 1 }; - - if (self->elements.used > 0) { - py_yajl_ps_free(self->elements); - py_yajl_ps_init(self->elements); - } - if (self->keys.used > 0) { - py_yajl_ps_free(self->keys); - py_yajl_ps_init(self->keys); - } - - /* callbacks, config, allocfuncs */ - parser = yajl_alloc(&decode_callbacks, &config, NULL, (void *)(self)); - yrc = yajl_parse(parser, (const unsigned char *)(buffer), buflen); - yajl_parse_complete(parser); - yajl_free(parser); + yrc = yajl_parse(self->parser, (const unsigned char *)(buffer), buflen); if (yrc != yajl_status_ok) { PyErr_SetObject(PyExc_ValueError, PyUnicode_FromString(yajl_status_to_string(yrc))); return NULL; } + Py_RETURN_NONE; +} + +/* +Return an object from the decoded_objects list +*/ +PyObject *_fetchObject(_YajlDecoder *self) +{ + PyObject *result = NULL; + int len; - if (self->root == NULL) { + len = PySequence_Size(self->decoded_objects); + if (len == 0) { PyErr_SetObject(PyExc_ValueError, - PyUnicode_FromString("The root object is NULL")); + PyUnicode_FromString("No Objects Decoded")); + return NULL; + } + + result = PySequence_GetItem(self->decoded_objects,0); + PySequence_DelItem(self->decoded_objects,0); + Py_DECREF(result); + return result; +} + +/* + * Decode a chunk of input at a time + */ +PyObject *py_yajldecoder_iterdecode(_YajlDecoder *self, PyObject *args) +{ + char *buffer = NULL; + PyObject *pybuffer = NULL; + PyObject *result = NULL; + Py_ssize_t buflen = 0; + + if (!PyArg_ParseTuple(args, "O", &pybuffer)) + return NULL; + + Py_INCREF(pybuffer); + + if (PyUnicode_Check(pybuffer)) { + if (!(result = PyUnicode_AsUTF8String(pybuffer))) { + Py_DECREF(pybuffer); + return NULL; + } + Py_DECREF(pybuffer); + pybuffer = result; + result = NULL; + } + + if (PyString_Check(pybuffer)) { + if (PyString_AsStringAndSize(pybuffer, &buffer, &buflen)) { + Py_DECREF(pybuffer); + return NULL; + } + } else { + /* really seems like this should be a TypeError, but + tests/unit.py:ErrorCasesTests.test_None disagrees */ + Py_DECREF(pybuffer); + PyErr_SetString(PyExc_ValueError, "string or unicode expected"); + return NULL; + } + + if (!buflen) { + Py_DECREF(pybuffer); + Py_RETURN_NONE; + } + + result = _internal_decode(self, buffer, (unsigned int)buflen); + if (!result) { + Py_DECREF(pybuffer); return NULL; } - // Callee now owns memory, we'll leave refcnt at one and - // null out our pointer. - PyObject *root = self->root; - self->root = NULL; - return root; + Py_DECREF(pybuffer); + Py_INCREF(self->decoded_objects); + result = self->decoded_objects; + if (PySequence_Size(self->decoded_objects) > 0) { + Py_DECREF(self->decoded_objects); + self->decoded_objects = PyList_New(0); + } + return result; } -PyObject *py_yajldecoder_decode(PYARGS) +/* +External interface "decode" +*/ +PyObject *py_yajldecoder_decode(_YajlDecoder *self, PyObject *args) { - _YajlDecoder *decoder = (_YajlDecoder *)(self); char *buffer = NULL; PyObject *pybuffer = NULL; PyObject *result = NULL; Py_ssize_t buflen = 0; + yajl_status yrc; if (!PyArg_ParseTuple(args, "O", &pybuffer)) return NULL; @@ -314,34 +372,256 @@ PyObject *py_yajldecoder_decode(PYARGS) } if (!buflen) { + Py_DECREF(pybuffer); PyErr_SetObject(PyExc_ValueError, PyUnicode_FromString("Cannot parse an empty buffer")); return NULL; } - result = _internal_decode(decoder, buffer, (unsigned int)buflen); - Py_DECREF(pybuffer); + result = _internal_decode(self, buffer, (unsigned int)buflen); + if (!result) { + Py_DECREF(pybuffer); + return NULL; + } + + yrc = yajl_complete_parse(self->parser); + if (yrc != yajl_status_ok) { + PyErr_SetObject(PyExc_ValueError, + PyUnicode_FromString(yajl_status_to_string(yrc))); + return NULL; + } + result = _fetchObject(self); return result; } -int yajldecoder_init(PYARGS) +PyObject *py_yajldecoder_iter(PyObject *self) +{ + Py_INCREF(self); + return self; +} + +PyObject *py_yajldecoder_iternext(PyObject *self) { + _YajlDecoder *d = (_YajlDecoder *)self; + + PyObject *buffer = NULL; + PyObject *result = NULL; +#ifdef IS_PYTHON3 + PyObject *bufferstring = NULL; +#endif + + // return an object from the list if there is one available. + //len = PySequence_Size(d->decoded_objects); + if (PySequence_Size(d->decoded_objects) != 0) { + return _fetchObject(d); + } + + // return NULL if stream is not set + if (!d->stream) { + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("no stream to iterate over")); + return NULL; + } + + // while there are no complete objects keep trying to read from the stream + while (PySequence_Size(d->decoded_objects) < 1) { + buffer = PyObject_CallMethod(d->stream,d->read_fn,"O",d->bufsize); + if (!buffer) { + Py_XDECREF(buffer); + return NULL; + } + +#ifdef IS_PYTHON3 + bufferstring = PyUnicode_AsUTF8String(buffer); + if (!bufferstring) + return NULL; +#endif + +#ifdef IS_PYTHON3 + if (PyBytes_Size(bufferstring) == 0) { + Py_XDECREF(buffer); + return NULL; + } +#else + if (PyString_Size(buffer) == 0) { + Py_XDECREF(buffer); + return NULL; + } +#endif + +#ifdef IS_PYTHON3 + result = _internal_decode((_YajlDecoder *)d, PyBytes_AsString(bufferstring), + PyBytes_Size(bufferstring)); + Py_XDECREF(bufferstring); +#else + result = _internal_decode((_YajlDecoder *)d, PyString_AsString(buffer), + PyString_Size(buffer)); +#endif + Py_XDECREF(buffer); + if (result == NULL) { + // error set by _internal_decode + return NULL; + } + } + return _fetchObject(d); +} + +/* +Reset the parser to its starting state +This will also setup the parser for the first time +*/ +PyObject *py_yajldecoder_reset(_YajlDecoder *self,PyObject *args) +{ + // reset pointer structures + if (self->elements.used > 0) { + py_yajl_ps_free(self->elements); + py_yajl_ps_init(self->elements); + } + + if (self->keys.used > 0) { + py_yajl_ps_free(self->keys); + py_yajl_ps_init(self->keys); + } + + // reset decoded object list + Py_XDECREF(self->decoded_objects); + self->decoded_objects = PyList_New(0); + + // reset the parser + if (self->parser) { + yajl_free(self->parser); + } + + self->parser = yajl_alloc(&decode_callbacks,NULL, (void *)(self)); + yajl_config(self->parser,yajl_allow_comments,1); + yajl_config(self->parser,yajl_allow_partial_values,1); + yajl_config(self->parser,yajl_dont_validate_strings,0); + if (PyLong_AsUnsignedLongMask(self->allow_multiple_values) == 1) { + yajl_config(self->parser,yajl_allow_multiple_values,1); + } + + Py_RETURN_NONE; +} + +/* +Return the number of decoded objects in the internal buffer +Useful if allow_multiple_values is enabled +*/ +Py_ssize_t decoder_len(_YajlDecoder *self) +{ + return PySequence_Size(self->decoded_objects); +} + +/* +Init the decoder python object +*/ +int yajldecoder_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *allow_multiple_values = NULL; + PyObject *result = NULL; + PyObject *stream = NULL; + PyObject *bufsize = NULL; + + static char *kwlist[] = {"allow_multiple_values","stream","bufsize",NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOO", kwlist, \ + &allow_multiple_values, &stream,\ + &bufsize)) { + return -1; + } + + if (allow_multiple_values) { + if (!PyBool_Check(allow_multiple_values)) { + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("allow_multiple_values must be a bool")); + return -1; + } + } else { + // default to false + Py_INCREF(Py_False); + allow_multiple_values = Py_False; + } + + + // basic setup _YajlDecoder *me = (_YajlDecoder *)(self); + me->allow_multiple_values = allow_multiple_values; py_yajl_ps_init(me->elements); - py_yajl_ps_init(me->keys); - me->root = NULL; + py_yajl_ps_init(me->keys); + + // stream setup + if (stream) { + me->read_fn = "read"; + if (!PyObject_HasAttrString(stream,"read")) { + if (!PyObject_HasAttrString(stream,"recv")) { + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("stream object must have a read or recv attribute")); + return -1; + } + me->read_fn = "recv"; + } + Py_INCREF(stream); + } + me->stream = stream; + if (bufsize) { +#ifdef IS_PYTHON3 + if (!PyLong_Check(bufsize)) { +#else + if (!PyInt_Check(bufsize)) { +#endif + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("bufsize must be a int")); + return -1; + } +#ifdef IS_PYTHON3 + if (!(PyLong_AsLong(bufsize) >= 1)) { +#else + if (!(PyInt_AsLong(bufsize) >= 1)) { +#endif + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("bufsize must be >= 1")); + return -1; + } + Py_INCREF(bufsize); + me->bufsize = bufsize; + } else { +#ifdef IS_PYTHON3 + me->bufsize = PyLong_FromLong(512); +#else + me->bufsize = PyInt_FromLong(512); +#endif + } + + // reset method also initialises data structures + result = PyObject_CallMethod(self,"reset",NULL); + if (!result) { + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("failed to call self.reset()")); + return -1; + } + Py_XDECREF(result); return 0; } void yajldecoder_dealloc(_YajlDecoder *self) { + // free pointer stack py_yajl_ps_free(self->elements); - py_yajl_ps_init(self->elements); py_yajl_ps_free(self->keys); - py_yajl_ps_init(self->keys); - if (self->root) { - Py_XDECREF(self->root); + + // if we have decoded objects free those + if (self->decoded_objects) { + Py_XDECREF(self->decoded_objects); + } + + // free the parser + if (self->parser) { + yajl_free(self->parser); + } + + // free bufsize + if (self->bufsize) { + Py_XDECREF(self->bufsize); + } + + // free stream + if (self->stream) { + Py_XDECREF(self->stream); } #ifdef IS_PYTHON3 Py_TYPE(self)->tp_free((PyObject*)self); diff --git a/encoder.c b/encoder.c index 082dcfe..b1d3e18 100644 --- a/encoder.c +++ b/encoder.c @@ -42,7 +42,7 @@ static const char *hexdigit = "0123456789abcdef"; /* Located in yajl_hacks.c */ extern yajl_gen_status yajl_gen_raw_string(yajl_gen g, - const unsigned char * str, unsigned int len); + const char * str, unsigned int len); static yajl_gen_status ProcessObject(_YajlEncoder *self, PyObject *object) { @@ -142,7 +142,7 @@ static yajl_gen_status ProcessObject(_YajlEncoder *self, PyObject *object) } } buffer[offset] = '\0'; - status = yajl_gen_raw_string(handle, (const unsigned char *)(buffer), (unsigned int)(offset)); + status = yajl_gen_raw_string(handle, (const char *)(buffer), (unsigned int)(offset)); free(buffer); return status; } @@ -313,12 +313,12 @@ static PyObject * lowLevelStringAlloc(Py_ssize_t size) #ifdef IS_PYTHON3 PyBytesObject * op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size); if (op) { - PyObject_INIT_VAR(op, &PyBytes_Type, size); + (void) PyObject_INIT_VAR(op, &PyBytes_Type, size); } #else PyStringObject * op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size); if (op) { - PyObject_INIT_VAR(op, &PyString_Type, size); + (void) PyObject_INIT_VAR(op, &PyString_Type, size); op->ob_shash = -1; op->ob_sstate = SSTATE_NOT_INTERNED; } @@ -326,9 +326,8 @@ static PyObject * lowLevelStringAlloc(Py_ssize_t size) return (PyObject *) op; } -PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj, yajl_gen_config genconfig) +PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj, yajl_gen gen) { - yajl_gen generator = NULL; yajl_gen_status status; struct StringAndUsedCount sauc; #ifdef IS_PYTHON3 @@ -341,13 +340,13 @@ PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj, yajl_gen_config ge sauc.used = 0; sauc.str = lowLevelStringAlloc(PY_YAJL_CHUNK_SZ); - generator = yajl_gen_alloc2(py_yajl_printer, &genconfig, NULL, (void *) &sauc); + yajl_gen_config(gen,yajl_gen_print_callback,py_yajl_printer,(void *) &sauc); - self->_generator = generator; + self->_generator = gen; status = ProcessObject(self, obj); - yajl_gen_free(generator); + yajl_gen_free(gen); self->_generator = NULL; /* if resize failed inside our printer function we'll have a null sauc.str */ @@ -392,12 +391,13 @@ PyObject *py_yajlencoder_default(PYARGS) PyObject *py_yajlencoder_encode(PYARGS) { _YajlEncoder *encoder = (_YajlEncoder *)(self); - yajl_gen_config config = {0, NULL}; + yajl_gen gen; + gen = yajl_gen_alloc(NULL); PyObject *value; if (!PyArg_ParseTuple(args, "O", &value)) return NULL; - return _internal_encode(encoder, value, config); + return _internal_encode(encoder, value, gen); } int yajlencoder_init(PYARGS) diff --git a/py_yajl.h b/py_yajl.h index af1ef96..2a89a76 100644 --- a/py_yajl.h +++ b/py_yajl.h @@ -35,11 +35,12 @@ #include #include +#include #include "ptrstack.h" #if PY_MAJOR_VERSION >= 3 #define IS_PYTHON3 -#define PyString_AsStringAndSize PyBytes_AsStringAndSize +#define PyString_AsStringAndSize PyBytes_AsStringAndSize #define PyString_Check PyBytes_Check #endif @@ -49,6 +50,12 @@ typedef struct { py_yajl_bytestack elements; py_yajl_bytestack keys; PyObject *root; + PyObject *decoded_objects; + PyObject *allow_multiple_values; + PyObject *stream; + PyObject *bufsize; + yajl_handle parser; + char *read_fn; } _YajlDecoder; @@ -88,10 +95,16 @@ enum { failure, success }; /* * Methods defined for the YajlDecoder type in decoder.c */ -extern PyObject *py_yajldecoder_decode(PYARGS); -extern int yajldecoder_init(PYARGS); +extern PyObject *py_yajldecoder_decode(_YajlDecoder *self, PyObject *args); +extern int yajldecoder_init(PyObject *self, PyObject *args, PyObject *kwargs); extern void yajldecoder_dealloc(_YajlDecoder *self); extern PyObject *_internal_decode(_YajlDecoder *self, char *buffer, unsigned int buflen); +extern PyObject *py_yajldecoder_reset(_YajlDecoder *self,PyObject *args); +extern Py_ssize_t decoder_len(_YajlDecoder *self); +extern PyObject *_fetchObject(_YajlDecoder *self); +extern PyObject *py_yajldecoder_iter(PyObject *self); +extern PyObject *py_yajldecoder_iternext(PyObject *self); +extern PyObject *py_yajldecoder_iterdecode(_YajlDecoder *self, PyObject *args); /* @@ -101,7 +114,7 @@ extern PyObject *py_yajlencoder_encode(PYARGS); extern PyObject* py_yajlencoder_default(PYARGS); extern int yajlencoder_init(PYARGS); extern void yajlencoder_dealloc(_YajlEncoder *self); -extern PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj, yajl_gen_config config); +extern PyObject *_internal_encode(_YajlEncoder *self, PyObject *obj, yajl_gen gen); #endif diff --git a/runtests.sh b/runtests.sh index b467e8d..39f66c2 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,3 +1,9 @@ #!/bin/sh -python setup.py build && PYTHONPATH=.:build/lib.linux-x86_64-2.6 python tests/unit.py && zcat test_data/issue_11.gz| PYTHONPATH=build/lib.linux-x86_64-2.6 ./tests/issue_11.py && python3 setup.py build && PYTHONPATH=build/lib.linux-x86_64-3.1 python3 tests/unit.py +python setup.py build && +PYTHONPATH=.:build/lib.linux-x86_64-2.6 python tests/unit.py && +zcat test_data/issue_11.gz| PYTHONPATH=build/lib.linux-x86_64-2.6 ./tests/issue_11.py && + +python3 setup.py build && +PYTHONPATH=build/lib.linux-x86_64-3.1 python3 tests/unit.py && +zcat test_data/issue_11.gz| PYTHONPATH=build/lib.linux-x86_64-3.1 python3 ./tests/issue_11.py diff --git a/tests/issue_11.py b/tests/issue_11.py index 835dc8a..f8f94ff 100755 --- a/tests/issue_11.py +++ b/tests/issue_11.py @@ -6,4 +6,3 @@ for i, l in enumerate(sys.stdin): l = l.rstrip('\n').split('\t') d = yajl.dumps(tuple(l)) - print i, diff --git a/tests/python2.py b/tests/python2.py index aafae11..c955cd6 100644 --- a/tests/python2.py +++ b/tests/python2.py @@ -11,3 +11,5 @@ IssueSevenTest_chinese_char = u'\u65e9\u5b89, \u7238\u7238' IssueTwelveTest_dict = {u'a' : u'b', u'c' : u'd'} + +IssueTwentySevenTest_dict = u'[{"data":"Podstawow\u0105 opiek\u0119 zdrowotn\u0105"}]' diff --git a/tests/unit.py b/tests/unit.py index 978374c..1b406b5 100644 --- a/tests/unit.py +++ b/tests/unit.py @@ -335,10 +335,10 @@ def runTest(self): class IssueTwentySevenTest(unittest.TestCase): "https://github.com/rtyler/py-yajl/issues/27" def runTest(self): - u = u'[{"data":"Podstawow\u0105 opiek\u0119 zdrowotn\u0105"}]' - self.assertEqual( - yajl.dumps(yajl.loads(u)), - '[{"data":"Podstawow\\u0105 opiek\\u0119 zdrowotn\\u0105"}]') + if not is_python3(): + from tests import python2 + self.assertEqual(yajl.dumps(yajl.loads(python2.IssueTwentySevenTest_dict)), + '[{"data":"Podstawow\\u0105 opiek\\u0119 zdrowotn\\u0105"}]') if __name__ == '__main__': diff --git a/yajl b/yajl index d1e7708..5b0e7df 160000 --- a/yajl +++ b/yajl @@ -1 +1 @@ -Subproject commit d1e770838efaa76811f1cd8930d19d13b14fbc2b +Subproject commit 5b0e7df9f60820dbd9d3faf8bae620f87cbb37bd diff --git a/yajl.c b/yajl.c index 3bb7c24..396c5bc 100644 --- a/yajl.c +++ b/yajl.c @@ -33,8 +33,13 @@ #include "py_yajl.h" +/* As the name says, an empty tuple. */ +static PyObject *empty_tuple; + static PyMethodDef yajldecoder_methods[] = { {"decode", (PyCFunction)(py_yajldecoder_decode), METH_VARARGS, NULL}, + {"reset", (PyCFunction)(py_yajldecoder_reset), METH_VARARGS, NULL}, + {"iterdecode", (PyCFunction)(py_yajldecoder_iterdecode), METH_VARARGS, NULL}, {NULL} }; @@ -44,6 +49,18 @@ static PyMethodDef yajlencoder_methods[] = { {NULL} }; +static PySequenceMethods decoder_as_sequence = { + (lenfunc)decoder_len, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + 0, /* sq_contains */ +}; + + static PyTypeObject YajlDecoderType = { #ifdef IS_PYTHON3 PyVarObject_HEAD_INIT(NULL, 0) @@ -61,7 +78,7 @@ static PyTypeObject YajlDecoderType = { 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ + &decoder_as_sequence, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ @@ -75,8 +92,8 @@ static PyTypeObject YajlDecoderType = { 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ + py_yajldecoder_iter, /* tp_iter */ + py_yajldecoder_iternext, /* tp_iternext */ yajldecoder_methods, /* tp_methods */ NULL, /* tp_members */ 0, /* tp_getset */ @@ -134,55 +151,30 @@ static PyTypeObject YajlEncoderType = { 0, /* tp_alloc */ }; -static PyObject *py_loads(PYARGS) +static PyObject *py_loads(PyObject *self, PyObject *args) { PyObject *decoder = NULL; - PyObject *result = NULL; PyObject *pybuffer = NULL; - char *buffer = NULL; - Py_ssize_t buflen = 0; - - if (!PyArg_ParseTuple(args, "O", &pybuffer)) - return NULL; - - Py_INCREF(pybuffer); - - if (PyUnicode_Check(pybuffer)) { - if (!(result = PyUnicode_AsUTF8String(pybuffer))) { - Py_DECREF(pybuffer); - return NULL; - } - Py_DECREF(pybuffer); - pybuffer = result; - result = NULL; - } - - if (PyString_Check(pybuffer)) { - if (PyString_AsStringAndSize(pybuffer, &buffer, &buflen)) { - Py_DECREF(pybuffer); - return NULL; - } - } else { - /* really seems like this should be a TypeError, but - tests/unit.py:ErrorCasesTests.test_None disagrees */ - Py_DECREF(pybuffer); - PyErr_SetString(PyExc_ValueError, "string or unicode expected"); + PyObject *result = NULL; + + if (!PyArg_ParseTuple(args, "O", &pybuffer)) { return NULL; } - decoder = PyObject_Call((PyObject *)(&YajlDecoderType), NULL, NULL); + decoder = PyObject_CallObject((PyObject *)(&YajlDecoderType), empty_tuple); if (decoder == NULL) { + PyErr_SetString(PyExc_ValueError, "unable to init decoder"); return NULL; } - result = _internal_decode( - (_YajlDecoder *)decoder, buffer, (unsigned int)buflen); - Py_DECREF(pybuffer); + Py_INCREF(pybuffer); + result = PyObject_CallMethod(decoder,"decode","O",pybuffer); + Py_XDECREF(pybuffer); Py_XDECREF(decoder); return result; } -static char *__config_gen_config(PyObject *indent, yajl_gen_config *config) +static char *__config_gen_config(PyObject *indent, yajl_gen *config) { long indentLevel = -1; char *spaces = NULL; @@ -204,15 +196,15 @@ static char *__config_gen_config(PyObject *indent, yajl_gen_config *config) indentLevel = PyLong_AsLong(indent); if (indentLevel >= 0) { - config->beautify = 1; + yajl_gen_config(*config,yajl_gen_beautify, 1); if (indentLevel == 0) { - config->indentString = ""; + yajl_gen_config(*config, yajl_gen_indent_string, ""); } else { spaces = (char *)(malloc(sizeof(char) * (indentLevel + 1))); memset((void *)(spaces), (int)' ', indentLevel); spaces[indentLevel] = '\0'; - config->indentString = spaces; + yajl_gen_config(*config, yajl_gen_indent_string,spaces); } } } @@ -225,7 +217,7 @@ static PyObject *py_dumps(PYARGS) PyObject *obj = NULL; PyObject *result = NULL; PyObject *indent = NULL; - yajl_gen_config config = { 0, NULL }; + yajl_gen gen; static char *kwlist[] = {"object", "indent", NULL}; char *spaces = NULL; @@ -233,7 +225,8 @@ static PyObject *py_dumps(PYARGS) return NULL; } - spaces = __config_gen_config(indent, &config); + gen = yajl_gen_alloc(NULL); + spaces = __config_gen_config(indent, &gen); if (PyErr_Occurred()) { return NULL; } @@ -243,7 +236,7 @@ static PyObject *py_dumps(PYARGS) return NULL; } - result = _internal_encode((_YajlEncoder *)encoder, obj, config); + result = _internal_encode((_YajlEncoder *)encoder, obj, gen); Py_XDECREF(encoder); if (spaces) { free(spaces); @@ -251,7 +244,6 @@ static PyObject *py_dumps(PYARGS) return result; } -static PyObject *__read = NULL; static PyObject *_internal_stream_load(PyObject *args, unsigned int blocking) { PyObject *decoder = NULL; @@ -263,19 +255,16 @@ static PyObject *_internal_stream_load(PyObject *args, unsigned int blocking) #endif if (!PyArg_ParseTuple(args, "O", &stream)) { - goto bad_type; - } - - if (__read == NULL) { - __read = PyUnicode_FromString("read"); + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a single stream object")); + return NULL; } - if (!PyObject_HasAttr(stream, __read)) { - goto bad_type; + if (!PyObject_HasAttrString(stream,"read")) { + PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a single stream object")); + return NULL; } - buffer = PyObject_CallMethodObjArgs(stream, __read, NULL); - + buffer = PyObject_CallMethod(stream,"read",NULL); if (!buffer) return NULL; @@ -285,40 +274,30 @@ static PyObject *_internal_stream_load(PyObject *args, unsigned int blocking) return NULL; #endif - decoder = PyObject_Call((PyObject *)(&YajlDecoderType), NULL, NULL); + decoder = PyObject_CallObject((PyObject *)(&YajlDecoderType), empty_tuple); if (decoder == NULL) { return NULL; } #ifdef IS_PYTHON3 - result = _internal_decode((_YajlDecoder *)decoder, PyBytes_AsString(bufferstring), - PyBytes_Size(bufferstring)); + result = PyObject_CallMethod(decoder,"decode","O",bufferstring); Py_XDECREF(bufferstring); #else - result = _internal_decode((_YajlDecoder *)decoder, PyString_AsString(buffer), - PyString_Size(buffer)); + result = PyObject_CallMethod(decoder,"decode","O",buffer); #endif Py_XDECREF(decoder); Py_XDECREF(buffer); return result; - -bad_type: - PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a single stream object")); - return NULL; } static PyObject *py_load(PYARGS) { return _internal_stream_load(args, 1); } -static PyObject *py_iterload(PYARGS) -{ - return _internal_stream_load(args, 0); -} static PyObject *__write = NULL; static PyObject *_internal_stream_dump(PyObject *object, PyObject *stream, unsigned int blocking, - yajl_gen_config config) + yajl_gen gen) { PyObject *encoder = NULL; PyObject *buffer = NULL; @@ -336,11 +315,11 @@ static PyObject *_internal_stream_dump(PyObject *object, PyObject *stream, unsig return NULL; } - buffer = _internal_encode((_YajlEncoder *)encoder, object, config); + buffer = _internal_encode((_YajlEncoder *)encoder, object, gen); PyObject_CallMethodObjArgs(stream, __write, buffer, NULL); Py_XDECREF(encoder); Py_XDECREF(buffer); - return Py_True; + Py_RETURN_TRUE; bad_type: PyErr_SetObject(PyExc_TypeError, PyUnicode_FromString("Must pass a stream object")); @@ -353,7 +332,7 @@ static PyObject *py_dump(PYARGS) PyObject *indent = NULL; PyObject *stream = NULL; PyObject *result = NULL; - yajl_gen_config config = { 0, NULL }; + yajl_gen gen; static char *kwlist[] = {"object", "stream", "indent", NULL}; char *spaces = NULL; @@ -361,11 +340,12 @@ static PyObject *py_dump(PYARGS) return NULL; } - spaces = __config_gen_config(indent, &config); + gen = yajl_gen_alloc(NULL); + spaces = __config_gen_config(indent, &gen); if (PyErr_Occurred()) { return NULL; } - result = _internal_stream_dump(object, stream, 0, config); + result = _internal_stream_dump(object, stream, 0, gen); if (spaces) { free(spaces); } @@ -379,7 +359,7 @@ static PyObject *py_monkeypatch(PYARGS) PyObject *yajl = PyDict_GetItemString(modules, "yajl"); if (!yajl) { - return Py_False; + Py_RETURN_FALSE; } PyDict_SetItemString(modules, "json_old", PyDict_GetItemString(modules, "json")); @@ -387,11 +367,11 @@ static PyObject *py_monkeypatch(PYARGS) Py_XDECREF(sys); Py_XDECREF(modules); - return Py_True; + Py_RETURN_TRUE; } static struct PyMethodDef yajl_methods[] = { - {"dumps", (PyCFunctionWithKeywords)(py_dumps), METH_VARARGS | METH_KEYWORDS, + {"dumps", (PyCFunction)(py_dumps), METH_VARARGS | METH_KEYWORDS, "yajl.dumps(obj [, indent=None])\n\n\ Returns an encoded JSON string of the specified `obj`\n\ \n\ @@ -407,7 +387,7 @@ Returns a decoded object based on the given JSON `string`"}, "yajl.load(fp)\n\n\ Returns a decoded object based on the JSON read from the `fp` stream-like\n\ object; *Note:* It is expected that `fp` supports the `read()` method"}, - {"dump", (PyCFunctionWithKeywords)(py_dump), METH_VARARGS | METH_KEYWORDS, + {"dump", (PyCFunction)(py_dump), METH_VARARGS | METH_KEYWORDS, "yajl.dump(obj, fp [, indent=None])\n\n\ Encodes the given `obj` and writes it to the `fp` stream-like object. \n\ *Note*: It is expected that `fp` supports the `write()` method\n\ @@ -417,9 +397,6 @@ and object members will be pretty-printed with that indent level. \n\ An indent level of 0 will only insert newlines. None (the default) \n\ selects the most compact representation.\n\ "}, - /* - {"iterload", (PyCFunction)(py_iterload), METH_VARARGS, NULL}, - */ {"monkeypatch", (PyCFunction)(py_monkeypatch), METH_NOARGS, "yajl.monkeypatch()\n\n\ Monkey-patches the yajl module into sys.modules as \"json\"\n\ diff --git a/yajl_hacks.c b/yajl_hacks.c index 5ec9e4e..2209fc1 100644 --- a/yajl_hacks.c +++ b/yajl_hacks.c @@ -30,7 +30,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include - +#include /* * This code was yanked largely from yajl_gen.c @@ -50,8 +50,8 @@ typedef enum { struct yajl_gen_t { + unsigned int flags; unsigned int depth; - unsigned int pretty; const char * indentString; yajl_gen_state state[YAJL_MAX_DEPTH]; yajl_print_t print; @@ -64,21 +64,29 @@ struct yajl_gen_t if (g->state[g->depth] == yajl_gen_map_key || \ g->state[g->depth] == yajl_gen_in_array) { \ g->print(g->ctx, ",", 1); \ - if (g->pretty) g->print(g->ctx, "\n", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); \ } else if (g->state[g->depth] == yajl_gen_map_val) { \ g->print(g->ctx, ":", 1); \ - if (g->pretty) g->print(g->ctx, " ", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1); \ } #define INSERT_WHITESPACE \ - if (g->pretty) { \ + if ((g->flags & yajl_gen_beautify)) { \ if (g->state[g->depth] != yajl_gen_map_val) { \ unsigned int _i; \ for (_i=0;_idepth;_i++) \ - g->print(g->ctx, g->indentString, \ - strlen(g->indentString)); \ + g->print(g->ctx, \ + g->indentString, \ + (unsigned int)strlen(g->indentString)); \ } \ } + +#define ENSURE_NOT_KEY \ + if (g->state[g->depth] == yajl_gen_map_key || \ + g->state[g->depth] == yajl_gen_map_start) { \ + return yajl_gen_keys_must_be_strings; \ + } \ + /* check that we're not complete, or in error state. in a valid state * to be generating */ #define ENSURE_VALID_STATE \ @@ -88,6 +96,12 @@ struct yajl_gen_t return yajl_gen_generation_complete; \ } +#define INCREMENT_DEPTH \ + if (++(g->depth) >= YAJL_MAX_DEPTH) return yajl_max_depth_exceeded; + +#define DECREMENT_DEPTH \ + if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_gen_error; + #define APPENDED_ATOM \ switch (g->state[g->depth]) { \ case yajl_gen_start: \ @@ -108,10 +122,10 @@ struct yajl_gen_t } \ #define FINAL_NEWLINE \ - if (g->pretty && g->state[g->depth] == yajl_gen_complete) \ - g->print(g->ctx, "\n", 1); + if ((g->flags & yajl_gen_beautify) && g->state[g->depth] == yajl_gen_complete) \ + g->print(g->ctx, "\n", 1); -yajl_gen_status yajl_gen_raw_string(yajl_gen g, const unsigned char * str, unsigned int len) +yajl_gen_status yajl_gen_raw_string(yajl_gen g, const char * str, size_t len) { ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE; g->print(g->ctx, "\"", 1);