From c556f77d2abe34f7908877aa48ce2fbf294d8597 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 5 Apr 2020 02:35:11 +0900 Subject: [PATCH 1/5] bpo-1635741: Port _lzma module to multiphase initialization --- ...2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst | 1 + Modules/_lzmamodule.c | 551 +++++++++++------- Modules/clinic/_lzmamodule.c.h | 69 +-- 3 files changed, 365 insertions(+), 256 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst new file mode 100644 index 00000000000000..6937c0f2e48c7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst @@ -0,0 +1 @@ +Port _lzma module to multiphase initialization. diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 2a62a683568505..d112e0fadf53bc 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -23,6 +23,20 @@ } } while (0) #define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) +typedef struct { + PyTypeObject *lzma_compressor_type; + PyTypeObject *lzma_decompressor_type; + PyObject *error_type; + PyObject *empty_tuple; +} _lzma_state; + +static inline _lzma_state* +get_lzma_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (_lzma_state *)state; +} /* Container formats: */ enum { @@ -56,17 +70,10 @@ typedef struct { PyThread_type_lock lock; } Decompressor; -/* LZMAError class object. */ -static PyObject *Error; - -/* An empty tuple, used by the filter specifier parsing code. */ -static PyObject *empty_tuple; - - /* Helper functions. */ static int -catch_lzma_error(lzma_ret lzret) +catch_lzma_error(_lzma_state *state, lzma_ret lzret) { switch (lzret) { case LZMA_OK: @@ -75,31 +82,31 @@ catch_lzma_error(lzma_ret lzret) case LZMA_STREAM_END: return 0; case LZMA_UNSUPPORTED_CHECK: - PyErr_SetString(Error, "Unsupported integrity check"); + PyErr_SetString(state->error_type, "Unsupported integrity check"); return 1; case LZMA_MEM_ERROR: PyErr_NoMemory(); return 1; case LZMA_MEMLIMIT_ERROR: - PyErr_SetString(Error, "Memory usage limit exceeded"); + PyErr_SetString(state->error_type, "Memory usage limit exceeded"); return 1; case LZMA_FORMAT_ERROR: - PyErr_SetString(Error, "Input format not supported by decoder"); + PyErr_SetString(state->error_type, "Input format not supported by decoder"); return 1; case LZMA_OPTIONS_ERROR: - PyErr_SetString(Error, "Invalid or unsupported options"); + PyErr_SetString(state->error_type, "Invalid or unsupported options"); return 1; case LZMA_DATA_ERROR: - PyErr_SetString(Error, "Corrupt input data"); + PyErr_SetString(state->error_type, "Corrupt input data"); return 1; case LZMA_BUF_ERROR: - PyErr_SetString(Error, "Insufficient buffer space"); + PyErr_SetString(state->error_type, "Insufficient buffer space"); return 1; case LZMA_PROG_ERROR: - PyErr_SetString(Error, "Internal error"); + PyErr_SetString(state->error_type, "Internal error"); return 1; default: - PyErr_Format(Error, "Unrecognized error from liblzma: %d", lzret); + PyErr_Format(state->error_type, "Unrecognized error from liblzma: %d", lzret); return 1; } } @@ -186,7 +193,7 @@ INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) the C lzma_filter structs expected by liblzma. */ static void * -parse_filter_spec_lzma(PyObject *spec) +parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "preset", "dict_size", "lc", "lp", "pb", "mode", "nice_len", "mf", "depth", NULL}; @@ -217,11 +224,11 @@ parse_filter_spec_lzma(PyObject *spec) if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); - PyErr_Format(Error, "Invalid compression preset: %u", preset); + PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); return NULL; } - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, uint32_converter, &options->dict_size, @@ -235,20 +242,21 @@ parse_filter_spec_lzma(PyObject *spec) PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); - options = NULL; + return NULL; } + return options; } static void * -parse_filter_spec_delta(PyObject *spec) +parse_filter_spec_delta(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "dist", NULL}; PyObject *id; uint32_t dist = 1; lzma_options_delta *options; - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, &id, uint32_converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); @@ -264,14 +272,14 @@ parse_filter_spec_delta(PyObject *spec) } static void * -parse_filter_spec_bcj(PyObject *spec) +parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "start_offset", NULL}; PyObject *id; uint32_t start_offset = 0; lzma_options_bcj *options; - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, &id, uint32_converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); @@ -286,7 +294,7 @@ parse_filter_spec_bcj(PyObject *spec) } static int -lzma_filter_converter(PyObject *spec, void *ptr) +lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) { lzma_filter *f = (lzma_filter *)ptr; PyObject *id_obj; @@ -311,10 +319,10 @@ lzma_filter_converter(PyObject *spec, void *ptr) switch (f->id) { case LZMA_FILTER_LZMA1: case LZMA_FILTER_LZMA2: - f->options = parse_filter_spec_lzma(spec); + f->options = parse_filter_spec_lzma(state, spec); return f->options != NULL; case LZMA_FILTER_DELTA: - f->options = parse_filter_spec_delta(spec); + f->options = parse_filter_spec_delta(state, spec); return f->options != NULL; case LZMA_FILTER_X86: case LZMA_FILTER_POWERPC: @@ -322,7 +330,7 @@ lzma_filter_converter(PyObject *spec, void *ptr) case LZMA_FILTER_ARM: case LZMA_FILTER_ARMTHUMB: case LZMA_FILTER_SPARC: - f->options = parse_filter_spec_bcj(spec); + f->options = parse_filter_spec_bcj(state, spec); return f->options != NULL; default: PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id); @@ -340,7 +348,7 @@ free_filter_chain(lzma_filter filters[]) } static int -parse_filter_chain_spec(lzma_filter filters[], PyObject *filterspecs) +parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *filterspecs) { Py_ssize_t i, num_filters; @@ -357,7 +365,7 @@ parse_filter_chain_spec(lzma_filter filters[], PyObject *filterspecs) for (i = 0; i < num_filters; i++) { int ok = 1; PyObject *spec = PySequence_GetItem(filterspecs, i); - if (spec == NULL || !lzma_filter_converter(spec, &filters[i])) + if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) ok = 0; Py_XDECREF(spec); if (!ok) { @@ -492,6 +500,8 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) { Py_ssize_t data_size = 0; PyObject *result; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(c)); + assert(state != NULL); result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); if (result == NULL) @@ -509,7 +519,7 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) lzret = LZMA_OK; /* That wasn't a real error */ Py_END_ALLOW_THREADS - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; if ((action == LZMA_RUN && c->lzs.avail_in == 0) || (action == LZMA_FINISH && lzret == LZMA_STREAM_END)) { @@ -588,8 +598,8 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) } static int -Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset, - PyObject *filterspecs) +Compressor_init_xz(_lzma_state *state, lzma_stream *lzs, + int check, uint32_t preset, PyObject *filterspecs) { lzma_ret lzret; @@ -598,19 +608,19 @@ Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset, } else { lzma_filter filters[LZMA_FILTERS_MAX + 1]; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_stream_encoder(lzs, filters, check); free_filter_chain(filters); } - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; } static int -Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) +Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) { lzma_ret lzret; @@ -618,14 +628,14 @@ Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) lzma_options_lzma options; if (lzma_lzma_preset(&options, preset)) { - PyErr_Format(Error, "Invalid compression preset: %u", preset); + PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); return -1; } lzret = lzma_alone_encoder(lzs, &options); } else { lzma_filter filters[LZMA_FILTERS_MAX + 1]; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; if (filters[0].id == LZMA_FILTER_LZMA1 && filters[1].id == LZMA_VLI_UNKNOWN) { @@ -638,14 +648,14 @@ Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) } free_filter_chain(filters); } - if (PyErr_Occurred() || catch_lzma_error(lzret)) + if (PyErr_Occurred() || catch_lzma_error(state, lzret)) return -1; else return 0; } static int -Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) +Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) { lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; @@ -655,11 +665,11 @@ Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) "Must specify filters for FORMAT_RAW"); return -1; } - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_raw_encoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; @@ -706,7 +716,8 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) uint32_t preset = LZMA_PRESET_DEFAULT; PyObject *preset_obj = Py_None; PyObject *filterspecs = Py_None; - + _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + assert(state != NULL); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiOO:LZMACompressor", arg_names, &format, &check, &preset_obj, @@ -745,17 +756,17 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) case FORMAT_XZ: if (check == -1) check = LZMA_CHECK_CRC64; - if (Compressor_init_xz(&self->lzs, check, preset, filterspecs) != 0) + if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) break; return 0; case FORMAT_ALONE: - if (Compressor_init_alone(&self->lzs, preset, filterspecs) != 0) + if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) break; return 0; case FORMAT_RAW: - if (Compressor_init_raw(&self->lzs, filterspecs) != 0) + if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) break; return 0; @@ -774,17 +785,42 @@ static void Compressor_dealloc(Compressor *self) { lzma_end(&self->lzs); - if (self->lock != NULL) + if (self->lock != NULL) { PyThread_free_lock(self->lock); - Py_TYPE(self)->tp_free((PyObject *)self); + } + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +/*[clinic input] +_lzma.LZMACompressor.__reduce__ +[clinic start generated code]*/ + +static PyObject * +_lzma_LZMACompressor___reduce___impl(Compressor *self) +/*[clinic end generated code: output=b49a0538d1cad752 input=6be52aba16b513c1]*/ +{ + PyErr_Format(PyExc_TypeError, + "cannot pickle %s object", + Py_TYPE(self)->tp_name); + return NULL; } static PyMethodDef Compressor_methods[] = { _LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF _LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF + _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF {NULL} }; +static int +Compressor_traverse(Compressor *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + PyDoc_STRVAR(Compressor_doc, "LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n" "\n" @@ -813,47 +849,26 @@ PyDoc_STRVAR(Compressor_doc, "\n" "For one-shot compression, use the compress() function instead.\n"); -static PyTypeObject Compressor_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_lzma.LZMACompressor", /* tp_name */ - sizeof(Compressor), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Compressor_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - Compressor_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Compressor_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Compressor_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot lzma_compressor_type_slots[] = { + {Py_tp_dealloc, Compressor_dealloc}, + {Py_tp_methods, Compressor_methods}, + {Py_tp_init, Compressor_init}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_doc, (char *)Compressor_doc}, + {Py_tp_traverse, Compressor_traverse}, + {0, 0} }; +static PyType_Spec lzma_compressor_type_spec = { + .name = "_lzma.LZMACompressor", + .basicsize = sizeof(Compressor), + // Calling PyType_GetModuleState() on a subclass is not safe. + // lzma_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag + // which prevents to create a subclass. + // So calling PyType_GetModuleState() in this file is always safe. + .flags = Py_TPFLAGS_DEFAULT, + .slots = lzma_compressor_type_slots, +}; /* LZMADecompressor class. */ @@ -867,6 +882,8 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Py_ssize_t data_size = 0; PyObject *result; lzma_stream *lzs = &d->lzs; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(d)); + assert(state != NULL); if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); @@ -888,7 +905,7 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) lzret = LZMA_OK; /* That wasn't a real error */ Py_END_ALLOW_THREADS - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) d->check = lzma_get_check(&d->lzs); @@ -1082,16 +1099,16 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, } static int -Decompressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) +Decompressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) { lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_raw_decoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; @@ -1130,6 +1147,8 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK; uint64_t memlimit_ = UINT64_MAX; lzma_ret lzret; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + assert(state != NULL); if (memlimit != Py_None) { if (format == FORMAT_RAW) { @@ -1179,26 +1198,26 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, switch (format) { case FORMAT_AUTO: lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_XZ: lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_ALONE: self->check = LZMA_CHECK_NONE; lzret = lzma_alone_decoder(&self->lzs, memlimit_); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_RAW: self->check = LZMA_CHECK_NONE; - if (Decompressor_init_raw(&self->lzs, filters) == -1) + if (Decompressor_init_raw(state, &self->lzs, filters) == -1) break; return 0; @@ -1223,13 +1242,38 @@ Decompressor_dealloc(Decompressor *self) lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) + if (self->lock != NULL) { PyThread_free_lock(self->lock); - Py_TYPE(self)->tp_free((PyObject *)self); + } + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Decompressor_traverse(Decompressor *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +/*[clinic input] +_lzma.LZMADecompressor.__reduce__ +[clinic start generated code]*/ + +static PyObject * +_lzma_LZMADecompressor___reduce___impl(Decompressor *self) +/*[clinic end generated code: output=2611fff0104a9c30 input=b9882e030aecd9a5]*/ +{ + PyErr_Format(PyExc_TypeError, + "cannot pickle %s object", + Py_TYPE(self)->tp_name); + return NULL; } static PyMethodDef Decompressor_methods[] = { _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF + _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF {NULL} }; @@ -1257,45 +1301,26 @@ static PyMemberDef Decompressor_members[] = { {NULL} }; -static PyTypeObject Decompressor_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_lzma.LZMADecompressor", /* tp_name */ - sizeof(Decompressor), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Decompressor_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - _lzma_LZMADecompressor___init____doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Decompressor_methods, /* tp_methods */ - Decompressor_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - _lzma_LZMADecompressor___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot lzma_decompressor_type_slots[] = { + {Py_tp_dealloc, Decompressor_dealloc}, + {Py_tp_methods, Decompressor_methods}, + {Py_tp_init, _lzma_LZMADecompressor___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_doc, (char *)_lzma_LZMADecompressor___init____doc__}, + {Py_tp_traverse, Decompressor_traverse}, + {Py_tp_members, Decompressor_members}, + {0, 0} +}; + +static PyType_Spec lzma_decompressor_type_spec = { + .name = "_lzma.LZMADecompressor", + .basicsize = sizeof(Decompressor), + // Calling PyType_GetModuleState() on a subclass is not safe. + // lzma_decompressor_type_spec does not have Py_TPFLAGS_BASETYPE flag + // which prevents to create a subclass. + // So calling PyType_GetModuleState() in this file is always safe. + .flags = Py_TPFLAGS_DEFAULT, + .slots = lzma_decompressor_type_slots, }; @@ -1318,27 +1343,51 @@ _lzma_is_check_supported_impl(PyObject *module, int check_id) return PyBool_FromLong(lzma_check_is_supported(check_id)); } +PyDoc_STRVAR(_lzma__encode_filter_properties__doc__, +"_encode_filter_properties($module, filter, /)\n" +"--\n" +"\n" +"Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n" +"\n" +"The result does not include the filter ID itself, only the options."); -/*[clinic input] -_lzma._encode_filter_properties - filter: lzma_filter(c_default="{LZMA_VLI_UNKNOWN, NULL}") - / +#define _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF \ + {"_encode_filter_properties", (PyCFunction)_lzma__encode_filter_properties, METH_O, _lzma__encode_filter_properties__doc__}, -Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict). +static PyObject * +_lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter); -The result does not include the filter ID itself, only the options. -[clinic start generated code]*/ +static PyObject * +_lzma__encode_filter_properties(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + lzma_filter filter = {LZMA_VLI_UNKNOWN, NULL}; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); + if (!lzma_filter_converter(state, arg, &filter)) { + goto exit; + } + return_value = _lzma__encode_filter_properties_impl(module, filter); + +exit: + /* Cleanup for filter */ + if (filter.id != LZMA_VLI_UNKNOWN) + PyMem_Free(filter.options); + + return return_value; +} static PyObject * _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) -/*[clinic end generated code: output=5c93c8e14e7be5a8 input=d4c64f1b557c77d4]*/ { lzma_ret lzret; uint32_t encoded_size; PyObject *result = NULL; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); lzret = lzma_properties_size(&encoded_size, &filter); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; result = PyBytes_FromStringAndSize(NULL, encoded_size); @@ -1347,7 +1396,7 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) lzret = lzma_properties_encode( &filter, (uint8_t *)PyBytes_AS_STRING(result)); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; return result; @@ -1378,10 +1427,12 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, lzma_ret lzret; PyObject *result = NULL; filter.id = filter_id; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); lzret = lzma_properties_decode( &filter, NULL, encoded_props->buf, encoded_props->len); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return NULL; result = build_filter_spec(&filter); @@ -1392,28 +1443,6 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, return result; } - -/* Module initialization. */ - -static PyMethodDef module_methods[] = { - _LZMA_IS_CHECK_SUPPORTED_METHODDEF - _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF - _LZMA__DECODE_FILTER_PROPERTIES_METHODDEF - {NULL} -}; - -static PyModuleDef _lzmamodule = { - PyModuleDef_HEAD_INIT, - "_lzma", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL, -}; - /* Some of our constants are more than 32 bits wide, so PyModule_AddIntConstant would not work correctly on platforms with 32-bit longs. */ static int @@ -1428,67 +1457,145 @@ module_add_int_constant(PyObject *m, const char *name, long long value) return -1; } -#define ADD_INT_PREFIX_MACRO(m, macro) \ - module_add_int_constant(m, #macro, LZMA_ ## macro) - -PyMODINIT_FUNC -PyInit__lzma(void) +static int +lzma_exec(PyObject *module) { - PyObject *m; +#define ADD_INT_PREFIX_MACRO(module, macro) \ + do { \ + if (module_add_int_constant(module, #macro, LZMA_ ## macro) < 0) { \ + return -1; \ + } \ + } while(0) + +#define ADD_INT_MACRO(module, macro) \ + do { \ + if (PyModule_AddIntMacro(module, macro) < 0) { \ + return -1; \ + } \ + } while (0) - empty_tuple = PyTuple_New(0); - if (empty_tuple == NULL) - return NULL; - m = PyModule_Create(&_lzmamodule); - if (m == NULL) - return NULL; + _lzma_state *state = get_lzma_state(module); - if (PyModule_AddIntMacro(m, FORMAT_AUTO) == -1 || - PyModule_AddIntMacro(m, FORMAT_XZ) == -1 || - PyModule_AddIntMacro(m, FORMAT_ALONE) == -1 || - PyModule_AddIntMacro(m, FORMAT_RAW) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_NONE) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_CRC32) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_CRC64) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_SHA256) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_ID_MAX) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_UNKNOWN) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_LZMA1) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_LZMA2) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_DELTA) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_X86) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_IA64) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_ARM) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_ARMTHUMB) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_SPARC) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_POWERPC) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_HC3) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_HC4) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT2) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT3) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT4) == -1 || - ADD_INT_PREFIX_MACRO(m, MODE_FAST) == -1 || - ADD_INT_PREFIX_MACRO(m, MODE_NORMAL) == -1 || - ADD_INT_PREFIX_MACRO(m, PRESET_DEFAULT) == -1 || - ADD_INT_PREFIX_MACRO(m, PRESET_EXTREME) == -1) - return NULL; + state->empty_tuple = PyTuple_New(0); + if (state->empty_tuple == NULL) { + return -1; + } - Error = PyErr_NewExceptionWithDoc( - "_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); - if (Error == NULL) - return NULL; - Py_INCREF(Error); - if (PyModule_AddObject(m, "LZMAError", Error) == -1) - return NULL; + ADD_INT_MACRO(module, FORMAT_AUTO); + ADD_INT_MACRO(module, FORMAT_XZ); + ADD_INT_MACRO(module, FORMAT_ALONE); + ADD_INT_MACRO(module, FORMAT_RAW); + ADD_INT_PREFIX_MACRO(module, CHECK_NONE); + ADD_INT_PREFIX_MACRO(module, CHECK_CRC32); + ADD_INT_PREFIX_MACRO(module, CHECK_CRC64); + ADD_INT_PREFIX_MACRO(module, CHECK_SHA256); + ADD_INT_PREFIX_MACRO(module, CHECK_ID_MAX); + ADD_INT_PREFIX_MACRO(module, CHECK_UNKNOWN); + ADD_INT_PREFIX_MACRO(module, FILTER_LZMA1); + ADD_INT_PREFIX_MACRO(module, FILTER_LZMA2); + ADD_INT_PREFIX_MACRO(module, FILTER_DELTA); + ADD_INT_PREFIX_MACRO(module, FILTER_X86); + ADD_INT_PREFIX_MACRO(module, FILTER_IA64); + ADD_INT_PREFIX_MACRO(module, FILTER_ARM); + ADD_INT_PREFIX_MACRO(module, FILTER_ARMTHUMB); + ADD_INT_PREFIX_MACRO(module, FILTER_SPARC); + ADD_INT_PREFIX_MACRO(module, FILTER_POWERPC); + ADD_INT_PREFIX_MACRO(module, MF_HC3); + ADD_INT_PREFIX_MACRO(module, MF_HC4); + ADD_INT_PREFIX_MACRO(module, MF_BT2); + ADD_INT_PREFIX_MACRO(module, MF_BT3); + ADD_INT_PREFIX_MACRO(module, MF_BT4); + ADD_INT_PREFIX_MACRO(module, MODE_FAST); + ADD_INT_PREFIX_MACRO(module, MODE_NORMAL); + ADD_INT_PREFIX_MACRO(module, PRESET_DEFAULT); + ADD_INT_PREFIX_MACRO(module, PRESET_EXTREME); + + + state->error_type = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); + if (state->error_type == NULL) { + return -1; + } - if (PyModule_AddType(m, &Compressor_type) < 0) { - return NULL; + if (PyModule_AddType(module, (PyTypeObject *)state->error_type) < 0) { + return -1; } - if (PyModule_AddType(m, &Decompressor_type) < 0) { - return NULL; + + state->lzma_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &lzma_compressor_type_spec, NULL); + if (state->lzma_compressor_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->lzma_compressor_type) < 0) { + return -1; + } + + state->lzma_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &lzma_decompressor_type_spec, NULL); + if (state->lzma_decompressor_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->lzma_decompressor_type) < 0) { + return -1; } - return m; + return 0; +} + +static PyMethodDef lzma_methods[] = { + _LZMA_IS_CHECK_SUPPORTED_METHODDEF + _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF + _LZMA__DECODE_FILTER_PROPERTIES_METHODDEF + {NULL} +}; + +static PyModuleDef_Slot lzma_slots[] = { + {Py_mod_exec, lzma_exec}, + {0, NULL} +}; + +static int +lzma_traverse(PyObject *module, visitproc visit, void *arg) +{ + _lzma_state *state = get_lzma_state(module); + Py_VISIT(state->lzma_compressor_type); + Py_VISIT(state->lzma_decompressor_type); + Py_VISIT(state->empty_tuple); + return 0; +} + +static int +lzma_clear(PyObject *module) +{ + _lzma_state *state = get_lzma_state(module); + Py_CLEAR(state->lzma_compressor_type); + Py_CLEAR(state->lzma_decompressor_type); + Py_CLEAR(state->empty_tuple); + return 0; +} + +static void +lzma_free(void *module) +{ + lzma_clear((PyObject *)module); +} + +static PyModuleDef _lzmamodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_lzma", + .m_size = sizeof(_lzma_state), + .m_methods = lzma_methods, + .m_slots = lzma_slots, + .m_traverse = lzma_traverse, + .m_clear = lzma_clear, + .m_free = lzma_free, +}; + +PyMODINIT_FUNC +PyInit__lzma(void) +{ + return PyModuleDef_Init(&_lzmamodule); } diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index e4e0a7945a8fba..e15cc0c7e743b6 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -65,6 +65,23 @@ _lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored)) return _lzma_LZMACompressor_flush_impl(self); } +PyDoc_STRVAR(_lzma_LZMACompressor___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)_lzma_LZMACompressor___reduce__, METH_NOARGS, _lzma_LZMACompressor___reduce____doc__}, + +static PyObject * +_lzma_LZMACompressor___reduce___impl(Compressor *self); + +static PyObject * +_lzma_LZMACompressor___reduce__(Compressor *self, PyObject *Py_UNUSED(ignored)) +{ + return _lzma_LZMACompressor___reduce___impl(self); +} + PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" @@ -211,6 +228,23 @@ _lzma_LZMADecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs return return_value; } +PyDoc_STRVAR(_lzma_LZMADecompressor___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)_lzma_LZMADecompressor___reduce__, METH_NOARGS, _lzma_LZMADecompressor___reduce____doc__}, + +static PyObject * +_lzma_LZMADecompressor___reduce___impl(Decompressor *self); + +static PyObject * +_lzma_LZMADecompressor___reduce__(Decompressor *self, PyObject *Py_UNUSED(ignored)) +{ + return _lzma_LZMADecompressor___reduce___impl(self); +} + PyDoc_STRVAR(_lzma_is_check_supported__doc__, "is_check_supported($module, check_id, /)\n" "--\n" @@ -241,39 +275,6 @@ _lzma_is_check_supported(PyObject *module, PyObject *arg) return return_value; } -PyDoc_STRVAR(_lzma__encode_filter_properties__doc__, -"_encode_filter_properties($module, filter, /)\n" -"--\n" -"\n" -"Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n" -"\n" -"The result does not include the filter ID itself, only the options."); - -#define _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF \ - {"_encode_filter_properties", (PyCFunction)_lzma__encode_filter_properties, METH_O, _lzma__encode_filter_properties__doc__}, - -static PyObject * -_lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter); - -static PyObject * -_lzma__encode_filter_properties(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - lzma_filter filter = {LZMA_VLI_UNKNOWN, NULL}; - - if (!lzma_filter_converter(arg, &filter)) { - goto exit; - } - return_value = _lzma__encode_filter_properties_impl(module, filter); - -exit: - /* Cleanup for filter */ - if (filter.id != LZMA_VLI_UNKNOWN) - PyMem_Free(filter.options); - - return return_value; -} - PyDoc_STRVAR(_lzma__decode_filter_properties__doc__, "_decode_filter_properties($module, filter_id, encoded_props, /)\n" "--\n" @@ -319,4 +320,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=d6e997ebc269f78f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d89b6159e98544be input=a9049054013a1b77]*/ From 14ef8fa7510a2b413b070526d1f9cab5a60dac84 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:45:43 +0900 Subject: [PATCH 2/5] bpo-1635741: Fix memory leak --- Modules/_lzmamodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index d112e0fadf53bc..40eabb89ab48e3 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1563,6 +1563,7 @@ lzma_traverse(PyObject *module, visitproc visit, void *arg) _lzma_state *state = get_lzma_state(module); Py_VISIT(state->lzma_compressor_type); Py_VISIT(state->lzma_decompressor_type); + Py_VISIT(state->error_type); Py_VISIT(state->empty_tuple); return 0; } @@ -1573,6 +1574,7 @@ lzma_clear(PyObject *module) _lzma_state *state = get_lzma_state(module); Py_CLEAR(state->lzma_compressor_type); Py_CLEAR(state->lzma_decompressor_type); + Py_CLEAR(state->error_type); Py_CLEAR(state->empty_tuple); return 0; } From 304199581b9102ec16343b5cc6ab2986442e3da2 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:47:00 +0900 Subject: [PATCH 3/5] bpo-1635741: Update NEWS.d --- .../2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst index 6937c0f2e48c7c..956d0b68a8dfb1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst @@ -1 +1 @@ -Port _lzma module to multiphase initialization. +Port :mod:`_lzma` to multiphase initialization. From 60d315c31a92666f777a4ff92b184e676917dcc6 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:55:57 +0900 Subject: [PATCH 4/5] bpo-1635741: Apply PEP 7 --- Modules/_lzmamodule.c | 179 ++++++++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 40eabb89ab48e3..89936e698b76b7 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -114,8 +114,9 @@ catch_lzma_error(_lzma_state *state, lzma_ret lzret) static void* PyLzma_Malloc(void *opaque, size_t items, size_t size) { - if (size != 0 && items > (size_t)PY_SSIZE_T_MAX / size) + if (size != 0 && items > (size_t)PY_SSIZE_T_MAX / size) { return NULL; + } /* PyMem_Malloc() cannot be used: the GIL is not held when lzma_code() is called */ return PyMem_RawMalloc(items * size); @@ -139,8 +140,9 @@ grow_buffer(PyObject **buf, Py_ssize_t max_length) Py_ssize_t size = PyBytes_GET_SIZE(*buf); Py_ssize_t newsize = size + (size >> 3) + 6; - if (max_length > 0 && newsize > max_length) + if (max_length > 0 && newsize > max_length) { newsize = max_length; + } return _PyBytes_Resize(buf, newsize); } @@ -207,20 +209,24 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) preset_obj = PyMapping_GetItemString(spec, "preset"); if (preset_obj == NULL) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) + if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); - else + } + else { return NULL; + } } else { int ok = uint32_converter(preset_obj, &preset); Py_DECREF(preset_obj); - if (!ok) + if (!ok) { return NULL; + } } options = (lzma_options_lzma *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); @@ -264,8 +270,9 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) } options = (lzma_options_delta *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } options->type = LZMA_DELTA_TYPE_BYTE; options->dist = dist; return options; @@ -287,8 +294,9 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) } options = (lzma_options_bcj *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } options->start_offset = start_offset; return options; } @@ -313,8 +321,9 @@ lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) } f->id = PyLong_AsUnsignedLongLong(id_obj); Py_DECREF(id_obj); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { return 0; + } switch (f->id) { case LZMA_FILTER_LZMA1: @@ -341,10 +350,9 @@ lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) static void free_filter_chain(lzma_filter filters[]) { - int i; - - for (i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++) + for (int i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++) { PyMem_Free(filters[i].options); + } } static int @@ -353,8 +361,9 @@ parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *fil Py_ssize_t i, num_filters; num_filters = PySequence_Length(filterspecs); - if (num_filters == -1) + if (num_filters == -1) { return -1; + } if (num_filters > LZMA_FILTERS_MAX) { PyErr_Format(PyExc_ValueError, "Too many filters - liblzma supports a maximum of %d", @@ -365,8 +374,9 @@ parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *fil for (i = 0; i < num_filters; i++) { int ok = 1; PyObject *spec = PySequence_GetItem(filterspecs, i); - if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) + if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) { ok = 0; + } Py_XDECREF(spec); if (!ok) { filters[i].id = LZMA_VLI_UNKNOWN; @@ -391,8 +401,9 @@ spec_add_field(PyObject *spec, _Py_Identifier *key, unsigned long long value) PyObject *value_object; value_object = PyLong_FromUnsignedLongLong(value); - if (value_object == NULL) + if (value_object == NULL) { return -1; + } status = _PyDict_SetItemId(spec, key, value_object); Py_DECREF(value_object); @@ -405,8 +416,9 @@ build_filter_spec(const lzma_filter *f) PyObject *spec; spec = PyDict_New(); - if (spec == NULL) + if (spec == NULL) { return NULL; + } #define ADD_FIELD(SOURCE, FIELD) \ do { \ @@ -504,8 +516,9 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) assert(state != NULL); result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); - if (result == NULL) + if (result == NULL) { return NULL; + } c->lzs.next_in = data; c->lzs.avail_in = len; c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result); @@ -516,11 +529,13 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) Py_BEGIN_ALLOW_THREADS lzret = lzma_code(&c->lzs, action); data_size = (char *)c->lzs.next_out - PyBytes_AS_STRING(result); - if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) + if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) { lzret = LZMA_OK; /* That wasn't a real error */ + } Py_END_ALLOW_THREADS - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; + } if ((action == LZMA_RUN && c->lzs.avail_in == 0) || (action == LZMA_FINISH && lzret == LZMA_STREAM_END)) { break; @@ -532,8 +547,9 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) } } if (data_size != PyBytes_GET_SIZE(result)) - if (_PyBytes_Resize(&result, data_size) == -1) + if (_PyBytes_Resize(&result, data_size) == -1) { goto error; + } return result; error: @@ -562,10 +578,12 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) PyObject *result = NULL; ACQUIRE_LOCK(self); - if (self->flushed) + if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); - else + } + else { result = compress(self, data->buf, data->len, LZMA_RUN); + } RELEASE_LOCK(self); return result; } @@ -613,10 +631,12 @@ Compressor_init_xz(_lzma_state *state, lzma_stream *lzs, lzret = lzma_stream_encoder(lzs, filters, check); free_filter_chain(filters); } - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } static int @@ -648,10 +668,12 @@ Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyO } free_filter_chain(filters); } - if (PyErr_Occurred() || catch_lzma_error(state, lzret)) + if (PyErr_Occurred() || catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } static int @@ -665,14 +687,17 @@ Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) "Must specify filters for FORMAT_RAW"); return -1; } - if (parse_filter_chain_spec(state, filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) { return -1; + } lzret = lzma_raw_encoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } /*[-clinic input] @@ -721,8 +746,9 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiOO:LZMACompressor", arg_names, &format, &check, &preset_obj, - &filterspecs)) + &filterspecs)) { return -1; + } if (format != FORMAT_XZ && check != -1 && check != LZMA_CHECK_NONE) { PyErr_SetString(PyExc_ValueError, @@ -736,9 +762,11 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) return -1; } - if (preset_obj != Py_None) - if (!uint32_converter(preset_obj, &preset)) + if (preset_obj != Py_None) { + if (!uint32_converter(preset_obj, &preset)) { return -1; + } + } self->alloc.opaque = NULL; self->alloc.alloc = PyLzma_Malloc; @@ -754,20 +782,24 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) self->flushed = 0; switch (format) { case FORMAT_XZ: - if (check == -1) + if (check == -1) { check = LZMA_CHECK_CRC64; - if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) + } + if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) { break; + } return 0; case FORMAT_ALONE: - if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) + if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) { break; + } return 0; case FORMAT_RAW: - if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) + if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) { break; + } return 0; default: @@ -885,12 +917,15 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) _lzma_state *state = PyType_GetModuleState(Py_TYPE(d)); assert(state != NULL); - if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) + if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) { result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); - else + } + else { result = PyBytes_FromStringAndSize(NULL, max_length); - if (result == NULL) + } + if (result == NULL) { return NULL; + } lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result); lzs->avail_out = PyBytes_GET_SIZE(result); @@ -901,14 +936,17 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Py_BEGIN_ALLOW_THREADS lzret = lzma_code(lzs, LZMA_RUN); data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result); - if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0) + if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0) { lzret = LZMA_OK; /* That wasn't a real error */ + } Py_END_ALLOW_THREADS - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; - if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) + } + if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) { d->check = lzma_get_check(&d->lzs); + } if (lzret == LZMA_STREAM_END) { d->eof = 1; break; @@ -917,19 +955,23 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Maybe lzs's internal state still have a few bytes can be output, grow the output buffer and continue if max_lengh < 0. */ - if (data_size == max_length) + if (data_size == max_length) { break; - if (grow_buffer(&result, max_length) == -1) + } + if (grow_buffer(&result, max_length) == -1) { goto error; + } lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size; lzs->avail_out = PyBytes_GET_SIZE(result) - data_size; } else if (lzs->avail_in == 0) { break; } } - if (data_size != PyBytes_GET_SIZE(result)) - if (_PyBytes_Resize(&result, data_size) == -1) + if (data_size != PyBytes_GET_SIZE(result)) { + if (_PyBytes_Resize(&result, data_size) == -1) { goto error; + } + } return result; @@ -1001,8 +1043,9 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) if (lzs->avail_in > 0) { Py_XSETREF(d->unused_data, PyBytes_FromStringAndSize((char *)lzs->next_in, lzs->avail_in)); - if (d->unused_data == NULL) + if (d->unused_data == NULL) { goto error; + } } } else if (lzs->avail_in == 0) { @@ -1104,14 +1147,17 @@ Decompressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspec lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; - if (parse_filter_chain_spec(state, filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) { return -1; + } lzret = lzma_raw_decoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } /*[clinic input] @@ -1157,8 +1203,9 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, return -1; } memlimit_ = PyLong_AsUnsignedLongLong(memlimit); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { return -1; + } } if (format == FORMAT_RAW && filters == Py_None) { @@ -1192,33 +1239,38 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, self->input_buffer = NULL; self->input_buffer_size = 0; Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0)); - if (self->unused_data == NULL) + if (self->unused_data == NULL) { goto error; + } switch (format) { case FORMAT_AUTO: lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_XZ: lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_ALONE: self->check = LZMA_CHECK_NONE; lzret = lzma_alone_decoder(&self->lzs, memlimit_); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_RAW: self->check = LZMA_CHECK_NONE; - if (Decompressor_init_raw(state, &self->lzs, filters) == -1) + if (Decompressor_init_raw(state, &self->lzs, filters) == -1) { break; + } return 0; default: @@ -1371,8 +1423,9 @@ _lzma__encode_filter_properties(PyObject *module, PyObject *arg) exit: /* Cleanup for filter */ - if (filter.id != LZMA_VLI_UNKNOWN) + if (filter.id != LZMA_VLI_UNKNOWN) { PyMem_Free(filter.options); + } return return_value; } @@ -1396,8 +1449,9 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) lzret = lzma_properties_encode( &filter, (uint8_t *)PyBytes_AS_STRING(result)); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; + } return result; @@ -1432,8 +1486,9 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, lzret = lzma_properties_decode( &filter, NULL, encoded_props->buf, encoded_props->len); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return NULL; + } result = build_filter_spec(&filter); @@ -1449,10 +1504,12 @@ static int module_add_int_constant(PyObject *m, const char *name, long long value) { PyObject *o = PyLong_FromLongLong(value); - if (o == NULL) + if (o == NULL) { return -1; - if (PyModule_AddObject(m, name, o) == 0) + } + if (PyModule_AddObject(m, name, o) == 0) { return 0; + } Py_DECREF(o); return -1; } From f60b5912f610401af5f1fa2185104ead376c8ff7 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 23 Jun 2020 00:30:07 +0900 Subject: [PATCH 5/5] bpo-1635741: Apply Victor's review --- Modules/_lzmamodule.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 89936e698b76b7..24e1d6c2884ce5 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -26,7 +26,7 @@ typedef struct { PyTypeObject *lzma_compressor_type; PyTypeObject *lzma_decompressor_type; - PyObject *error_type; + PyObject *error; PyObject *empty_tuple; } _lzma_state; @@ -82,31 +82,31 @@ catch_lzma_error(_lzma_state *state, lzma_ret lzret) case LZMA_STREAM_END: return 0; case LZMA_UNSUPPORTED_CHECK: - PyErr_SetString(state->error_type, "Unsupported integrity check"); + PyErr_SetString(state->error, "Unsupported integrity check"); return 1; case LZMA_MEM_ERROR: PyErr_NoMemory(); return 1; case LZMA_MEMLIMIT_ERROR: - PyErr_SetString(state->error_type, "Memory usage limit exceeded"); + PyErr_SetString(state->error, "Memory usage limit exceeded"); return 1; case LZMA_FORMAT_ERROR: - PyErr_SetString(state->error_type, "Input format not supported by decoder"); + PyErr_SetString(state->error, "Input format not supported by decoder"); return 1; case LZMA_OPTIONS_ERROR: - PyErr_SetString(state->error_type, "Invalid or unsupported options"); + PyErr_SetString(state->error, "Invalid or unsupported options"); return 1; case LZMA_DATA_ERROR: - PyErr_SetString(state->error_type, "Corrupt input data"); + PyErr_SetString(state->error, "Corrupt input data"); return 1; case LZMA_BUF_ERROR: - PyErr_SetString(state->error_type, "Insufficient buffer space"); + PyErr_SetString(state->error, "Insufficient buffer space"); return 1; case LZMA_PROG_ERROR: - PyErr_SetString(state->error_type, "Internal error"); + PyErr_SetString(state->error, "Internal error"); return 1; default: - PyErr_Format(state->error_type, "Unrecognized error from liblzma: %d", lzret); + PyErr_Format(state->error, "Unrecognized error from liblzma: %d", lzret); return 1; } } @@ -230,7 +230,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); - PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); + PyErr_Format(state->error, "Invalid compression preset: %u", preset); return NULL; } @@ -648,7 +648,7 @@ Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyO lzma_options_lzma options; if (lzma_lzma_preset(&options, preset)) { - PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); + PyErr_Format(state->error, "Invalid compression preset: %u", preset); return -1; } lzret = lzma_alone_encoder(lzs, &options); @@ -1568,13 +1568,12 @@ lzma_exec(PyObject *module) ADD_INT_PREFIX_MACRO(module, PRESET_DEFAULT); ADD_INT_PREFIX_MACRO(module, PRESET_EXTREME); - - state->error_type = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); - if (state->error_type == NULL) { + state->error = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); + if (state->error == NULL) { return -1; } - if (PyModule_AddType(module, (PyTypeObject *)state->error_type) < 0) { + if (PyModule_AddType(module, (PyTypeObject *)state->error) < 0) { return -1; } @@ -1620,7 +1619,7 @@ lzma_traverse(PyObject *module, visitproc visit, void *arg) _lzma_state *state = get_lzma_state(module); Py_VISIT(state->lzma_compressor_type); Py_VISIT(state->lzma_decompressor_type); - Py_VISIT(state->error_type); + Py_VISIT(state->error); Py_VISIT(state->empty_tuple); return 0; } @@ -1631,7 +1630,7 @@ lzma_clear(PyObject *module) _lzma_state *state = get_lzma_state(module); Py_CLEAR(state->lzma_compressor_type); Py_CLEAR(state->lzma_decompressor_type); - Py_CLEAR(state->error_type); + Py_CLEAR(state->error); Py_CLEAR(state->empty_tuple); return 0; }