Skip to content

Commit

Permalink
reimplemented argument frame caching (previous approach was not threa…
Browse files Browse the repository at this point in the history 
…d-safe when GIL is used)
  • Loading branch information
@florianlink
florianlink committed Sep 17, 2018
1 parent 28c1c4b commit 170d4a4
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 197 deletions.
4 changes: 1 addition & 3 deletions src/PythonQt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,11 +391,9 @@ PythonQtPrivate::~PythonQtPrivate() {
delete i.next().value();
}
}
PythonQtConv::global_valueStorage.clear();
PythonQtConv::global_ptrStorage.clear();
PythonQtConv::global_variantStorage.clear();

PythonQtMethodInfo::cleanupCachedMethodInfos();
PythonQtArgumentFrame::cleanupFreeList();
}

void PythonQt::setRedirectStdInCallback(PythonQtInputChangedCB* callback, void * callbackData)
Expand Down
104 changes: 50 additions & 54 deletions src/PythonQtConversion.cpp
View file

Large diffs are not rendered by default.

12 changes: 3 additions & 9 deletions src/PythonQtConversion.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,10 @@ class PYTHONQT_EXPORT PythonQtConv {
static PyObject* ConvertQtValueToPython(const PythonQtMethodInfo::ParameterInfo& info, const void* data);

//! convert python object to Qt (according to the given parameter) and if the conversion should be strict (classInfo is currently not used anymore)
static void* ConvertPythonToQt(const PythonQtMethodInfo::ParameterInfo& info, PyObject* obj, bool strict, PythonQtClassInfo* classInfo, void* alreadyAllocatedCPPObject = NULL);
static void* ConvertPythonToQt(const PythonQtMethodInfo::ParameterInfo& info, PyObject* obj, bool strict, PythonQtClassInfo* classInfo, void* alreadyAllocatedCPPObject, PythonQtArgumentFrame* frame = NULL);

//! creates a data storage for the passed parameter type and returns a void pointer to be set as arg[0] of qt_metacall
static void* CreateQtReturnValue(const PythonQtMethodInfo::ParameterInfo& info);
static void* CreateQtReturnValue(const PythonQtMethodInfo::ParameterInfo& info, PythonQtArgumentFrame* frame);

//! converts QString to Python string (unicode!)
static PyObject* QStringToPyObject(const QString& str);
Expand Down Expand Up @@ -193,19 +193,13 @@ class PYTHONQT_EXPORT PythonQtConv {
//! Returns if the given object is a string (or unicode string)
static bool isStringType(PyTypeObject* type);

public:

static PythonQtValueStorage<qint64, 128> global_valueStorage;
static PythonQtValueStorage<void*, 128> global_ptrStorage;
static PythonQtValueStorageWithCleanup<QVariant, 128> global_variantStorage;

protected:
static QHash<int, PythonQtConvertMetaTypeToPythonCB*> _metaTypeToPythonConverters;
static QHash<int, PythonQtConvertPythonToMetaTypeCB*> _pythonToMetaTypeConverters;
static PythonQtConvertPythonSequenceToQVariantListCB* _pythonSequenceToQVariantListCB;

//! handle automatic conversion of some special types (QColor, QBrush, ...)
static void* handlePythonToQtAutoConversion(int typeId, PyObject* obj, void* alreadyAllocatedCPPObject);
static void* handlePythonToQtAutoConversion(int typeId, PyObject* obj, void* alreadyAllocatedCPPObject, PythonQtArgumentFrame* frame);

//! converts the list of pointers of given type to Python
static PyObject* ConvertQListOfPointerTypeToPythonList(QList<void*>* list, const PythonQtMethodInfo::ParameterInfo& info);
Expand Down
74 changes: 74 additions & 0 deletions src/PythonQtMisc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,78 @@
//----------------------------------------------------------------------------------

#include "PythonQtMisc.h"
#include <iostream>

#define PYTHONQT_MAX_ARGUMENT_FRAME_SIZE (PYTHONQT_MAX_ARGS * 2)

PythonQtArgumentFrame* PythonQtArgumentFrame::_freeListHead = NULL;

PythonQtArgumentFrame::PythonQtArgumentFrame()
{
_freeListNext = NULL;

// it is important to reserve the memory immediately,
// otherwise pointers would change while pushing back new arguments.
_variantArgs.reserve(PYTHONQT_MAX_ARGUMENT_FRAME_SIZE);
_podArgs.reserve(PYTHONQT_MAX_ARGUMENT_FRAME_SIZE);
}

PythonQtArgumentFrame::~PythonQtArgumentFrame()
{
}

PythonQtArgumentFrame* PythonQtArgumentFrame::newFrame()
{
PythonQtArgumentFrame* frame = NULL;
if (_freeListHead) {
frame = _freeListHead;
_freeListHead = _freeListHead->_freeListNext;
frame->_freeListNext = NULL;
} else {
frame = new PythonQtArgumentFrame();
}
return frame;
}

void PythonQtArgumentFrame::deleteFrame(PythonQtArgumentFrame* frame)
{
frame->reset();
frame->_freeListNext = _freeListHead;
_freeListHead = frame;
}

void PythonQtArgumentFrame::cleanupFreeList()
{
PythonQtArgumentFrame* head = _freeListHead;
while (head) {
PythonQtArgumentFrame* tmp = head;
head = head->_freeListNext;
delete tmp;
}
_freeListHead = NULL;
}

void PythonQtArgumentFrame::reset()
{
// Note: clear still keeps the capacity of the vectors, which is what we want!
_variantArgs.clear();
_podArgs.clear();
}

QVariant* PythonQtArgumentFrame::nextVariantPtr()
{
if (_variantArgs.size() >= PYTHONQT_MAX_ARGUMENT_FRAME_SIZE) {
std::cerr << "PYTHONQT_MAX_ARGUMENT_FRAME_SIZE QVariants exceeded, use less complex slots or increase size!" << std::endl;
}
_variantArgs.push_back(QVariant());
return &_variantArgs[_variantArgs.size() - 1];
}

quint64* PythonQtArgumentFrame::nextPODPtr()
{
if (_podArgs.size() >= PYTHONQT_MAX_ARGUMENT_FRAME_SIZE) {
std::cerr << "PYTHONQT_MAX_ARGUMENT_FRAME_SIZE PODs exceeded, use less complex slots or increase size!" << std::endl;
}
_podArgs.push_back(0);
return &_podArgs[_podArgs.size() - 1];
}
152 changes: 43 additions & 109 deletions src/PythonQtMisc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,133 +43,67 @@
//----------------------------------------------------------------------------------

#include "PythonQtPythonInclude.h"
#include <QList>
#include <vector>
#include <QVariant>

#define PythonQtValueStorage_ADD_VALUE(store, type, value, ptr) \
{ type* item = (type*)store.nextValuePtr(); \
#define PYTHONQT_MAX_ARGS 32

#define PythonQtArgumentFrame_ADD_VALUE(store, type, value, ptr) \
{ type* item = (type*)store->nextPODPtr(); \
*item = value; \
ptr = (void*)item; \
}

#define PythonQtValueStorage_ADD_VALUE_IF_NEEDED(alreadyAllocatedPtr,store, type, value, ptr) \
#define PythonQtArgumentFrame_ADD_VALUE_IF_NEEDED(alreadyAllocatedPtr,store, type, value, ptr) \
{ \
type* item = (type*)(alreadyAllocatedPtr?alreadyAllocatedPtr:store.nextValuePtr()); \
type* item = (type*)(alreadyAllocatedPtr?alreadyAllocatedPtr:store->nextPODPtr()); \
*item = value; \
ptr = (void*)item; \
}

//! stores a position in the PythonQtValueStorage
class PythonQtValueStoragePosition {

public:
PythonQtValueStoragePosition() { chunkIdx = 0; chunkOffset = 0; }
#define PythonQtArgumentFrame_ADD_VARIANT_VALUE(store, value, ptr) \
{ QVariant* item = store->nextVariantPtr(); \
*item = value; \
ptr = (void*)item; \
}

int chunkIdx;
int chunkOffset;
#define PythonQtArgumentFrame_ADD_VARIANT_VALUE_IF_NEEDED(alreadyAllocatedPtr,store, value, ptr) \
{ \
QVariant* item = (QVariant*)(alreadyAllocatedPtr?alreadyAllocatedPtr:store->nextVariantPtr()); \
*item = value; \
ptr = (void*)item; \
}

};
//! Stores C++ arguments for a qt_metacall (which are created when converting data from Python to C++)
class PythonQtArgumentFrame {

//! a helper class that stores basic C++ value types in chunks
template <typename T, int chunkEntries> class PythonQtValueStorage
{
public:
PythonQtValueStorage() {
_chunkIdx = 0;
_chunkOffset = 0;
_currentChunk = new T[chunkEntries];
_chunks.append(_currentChunk);
};

//! clear all memory
void clear() {
T* chunk;
Q_FOREACH(chunk, _chunks) {
delete[]chunk;
}
_chunks.clear();
}

//! get the current position to be restored with setPos
void getPos(PythonQtValueStoragePosition & pos) {
pos.chunkIdx = _chunkIdx;
pos.chunkOffset = _chunkOffset;
}

//! set the current position (without freeing memory, thus caching old entries for reuse)
void setPos(const PythonQtValueStoragePosition& pos) {
_chunkOffset = pos.chunkOffset;
if (_chunkIdx != pos.chunkIdx) {
_chunkIdx = pos.chunkIdx;
_currentChunk = _chunks.at(_chunkIdx);
}
}

//! add one default constructed value and return the pointer to it
T* nextValuePtr() {
if (_chunkOffset>=chunkEntries) {
_chunkIdx++;
if (_chunkIdx >= _chunks.size()) {
T* newChunk = new T[chunkEntries];
_chunks.append(newChunk);
_currentChunk = newChunk;
} else {
_currentChunk = _chunks.at(_chunkIdx);
}
_chunkOffset = 0;
}
T* newEntry = _currentChunk + _chunkOffset;
_chunkOffset++;
return newEntry;
};

protected:
QList<T*> _chunks;

int _chunkIdx;
int _chunkOffset;
T* _currentChunk;
//! Create a new (empty) frame (which is typically reused from a freelist)
static PythonQtArgumentFrame* newFrame();
//! Frees the frame (resetting it and putting it back to the freelist)
static void deleteFrame(PythonQtArgumentFrame* frame);

};
//! Frees all PythonQtArgumentFrame frames that are stored.
static void cleanupFreeList();

//! a helper class that stores basic C++ value types in chunks and clears the unused values on setPos() usage.
template <typename T, int chunkEntries> class PythonQtValueStorageWithCleanup : public PythonQtValueStorage<T, chunkEntries>
{
public:
void setPos(const PythonQtValueStoragePosition& pos) {
if (_chunkIdx > pos.chunkIdx) {
T* firstChunk = _chunks.at(pos.chunkIdx);
// clear region in first chunk
for (int i = pos.chunkOffset; i < chunkEntries; i++) {
firstChunk[i] = T();
}
for (int chunk = pos.chunkIdx + 1; chunk < _chunkIdx; chunk++) {
// clear the full chunks between the first and last chunk
T* fullChunk = _chunks.at(chunk);
for (int i = 0; i < chunkEntries; i++) {
fullChunk[i] = T();
}
}
// clear region in last chunk
T* lastChunk = _chunks.at(_chunkIdx);
for (int i = 0; i < _chunkOffset; i++) {
lastChunk[i] = T();
}
} else if (_chunkIdx == pos.chunkIdx) {
// clear the region in the last chunk only
T* lastChunk = _chunks.at(_chunkIdx);
for (int i = pos.chunkOffset; i<_chunkOffset; i++) {
lastChunk[i] = T();
}
}

PythonQtValueStorage<T, chunkEntries>::setPos(pos);
}
//! Resets the pod and variant argument lists to empty lists.
void reset();

//! Get next pointer to a variant
QVariant* nextVariantPtr();
//! Get next pointer to a POD.
quint64* nextPODPtr();

private:
using PythonQtValueStorage<T, chunkEntries>::_chunks;
using PythonQtValueStorage<T, chunkEntries>::_chunkIdx;
using PythonQtValueStorage<T, chunkEntries>::_chunkOffset;
using PythonQtValueStorage<T, chunkEntries>::_currentChunk;
PythonQtArgumentFrame();
~PythonQtArgumentFrame();

std::vector<quint64> _podArgs;
std::vector<QVariant> _variantArgs;

PythonQtArgumentFrame* _freeListNext;

static PythonQtArgumentFrame* _freeListHead;
};

#endif
26 changes: 4 additions & 22 deletions src/PythonQtSlot.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,26 +56,12 @@
#include <cxxabi.h>
#endif

#define PYTHONQT_MAX_ARGS 32


bool PythonQtCallSlot(PythonQtClassInfo* classInfo, QObject* objectToCall, PyObject* args, bool strict, PythonQtSlotInfo* info, void* firstArgument, PyObject** pythonReturnValue, void** directReturnValuePointer, PythonQtPassThisOwnershipType* passThisOwnershipToCPP)
{
static unsigned int recursiveEntry = 0;

if (directReturnValuePointer) {
*directReturnValuePointer = NULL;
}
// store the current storage position, so that we can get back to this state after a slot is called
// (do this locally, so that we have all positions on the stack
PythonQtValueStoragePosition globalValueStoragePos;
PythonQtValueStoragePosition globalPtrStoragePos;
PythonQtValueStoragePosition globalVariantStoragePos;
PythonQtConv::global_valueStorage.getPos(globalValueStoragePos);
PythonQtConv::global_ptrStorage.getPos(globalPtrStoragePos);
PythonQtConv::global_variantStorage.getPos(globalVariantStoragePos);

recursiveEntry++;
PythonQtArgumentFrame* frame = PythonQtArgumentFrame::newFrame();

// the arguments that are passed to qt_metacall
void* argList[PYTHONQT_MAX_ARGS];
Expand Down Expand Up @@ -115,7 +101,7 @@ bool PythonQtCallSlot(PythonQtClassInfo* classInfo, QObject* objectToCall, PyObj
}
for (int i = 1 + instanceDecoOffset; i<argc && ok; i++) {
const PythonQtSlotInfo::ParameterInfo& param = params.at(i);
argList[i] = PythonQtConv::ConvertPythonToQt(param, PyTuple_GET_ITEM(args, i - 1 - instanceDecoOffset), strict, classInfo);
argList[i] = PythonQtConv::ConvertPythonToQt(param, PyTuple_GET_ITEM(args, i - 1 - instanceDecoOffset), strict, classInfo, NULL, frame);
if (argList[i]==NULL) {
ok = false;
break;
Expand Down Expand Up @@ -143,7 +129,7 @@ bool PythonQtCallSlot(PythonQtClassInfo* classInfo, QObject* objectToCall, PyObj
// create empty default value for the return value
if (!directReturnValuePointer) {
// create empty default value for the return value
argList[0] = PythonQtConv::CreateQtReturnValue(returnValueParam);
argList[0] = PythonQtConv::CreateQtReturnValue(returnValueParam, frame);
if (argList[0]==NULL) {
// return value could not be created, maybe we have a registered class with a default constructor, so that we can construct the pythonqt wrapper object and
// pass its internal pointer
Expand Down Expand Up @@ -249,12 +235,8 @@ bool PythonQtCallSlot(PythonQtClassInfo* classInfo, QObject* objectToCall, PyObj
ok = false;
}
}
recursiveEntry--;

// reset the parameter storage position to the stored pos to "pop" the parameter stack
PythonQtConv::global_valueStorage.setPos(globalValueStoragePos);
PythonQtConv::global_ptrStorage.setPos(globalPtrStoragePos);
PythonQtConv::global_variantStorage.setPos(globalVariantStoragePos);
PythonQtArgumentFrame::deleteFrame(frame);

*pythonReturnValue = result;

Expand Down

0 comments on commit 170d4a4

Please sign in to comment.