From 65178100b3e95bbe1be7eb3f5691ac8aa97eacb9 Mon Sep 17 00:00:00 2001 From: Dmitriy Korneev Date: Mon, 13 Nov 2017 16:10:27 +0200 Subject: [PATCH 1/5] add python2 support for pyhermes --- pyhermes/MANIFEST.in | 2 ++ pyhermes/py_midhermes.c | 26 ++++++++++---------------- pyhermes/py_transport_wrapper.c | 2 +- pyhermes/setup.py | 8 +++----- pyhermes/test.py | 20 +++++++++++--------- pyhermes/tox.ini | 5 +++++ pyhermes/transport.c | 25 ++++++++++++++++++++----- pyhermes/transport.h | 2 ++ 8 files changed, 54 insertions(+), 36 deletions(-) create mode 100644 pyhermes/MANIFEST.in create mode 100644 pyhermes/tox.ini diff --git a/pyhermes/MANIFEST.in b/pyhermes/MANIFEST.in new file mode 100644 index 0000000..aa112ff --- /dev/null +++ b/pyhermes/MANIFEST.in @@ -0,0 +1,2 @@ +include *.c +include *.h \ No newline at end of file diff --git a/pyhermes/py_midhermes.c b/pyhermes/py_midhermes.c index 266c200..f293b4d 100644 --- a/pyhermes/py_midhermes.c +++ b/pyhermes/py_midhermes.c @@ -49,30 +49,24 @@ static int MidHermes_init(pyhermes_MidHermesObject *self, PyObject *args, PyObje PyObject *transport = PyObject_CallMethod(credential_store_transport, "get_hermes_transport", NULL); if (!HermesTransportWrapper_Check(transport)) { - //Py_DECREF(transport); PyErr_SetString(PyExc_TypeError, "credential_store return transport with incorrect type"); return -1; } hm_rpc_transport_t *credential_hermes_transport = ((pyhermes_HermesTransportWrapperObject_t *) transport)->hermes_transport; - //Py_DECREF(transport); transport = PyObject_CallMethod(key_store_transport, "get_hermes_transport", NULL); if (HermesTransportWrapper_Check(transport) != 1) { - //Py_DECREF(transport); PyErr_SetString(PyExc_TypeError, "credential_store return transport with incorrect type"); return -1; } hm_rpc_transport_t *key_hermes_transport = ((pyhermes_HermesTransportWrapperObject_t *) transport)->hermes_transport; - //Py_DECREF(transport); transport = PyObject_CallMethod(data_store_transport, "get_hermes_transport", NULL); if (HermesTransportWrapper_Check(transport) != 1) { - //Py_DECREF(transport); PyErr_SetString(PyExc_TypeError, "credential_store return transport with incorrect type"); return -1; } hm_rpc_transport_t *data_hermes_transport = ((pyhermes_HermesTransportWrapperObject_t *) transport)->hermes_transport; - //Py_DECREF(transport); if (!(credential_store_transport) || !(data_store_transport) @@ -113,7 +107,7 @@ static PyObject *MidHermes_addBlock(pyhermes_MidHermesObject *self, PyObject *ar size_t block_id_length = 0, block_length = 0, meta_length = 0; static char *kwlist[] = {"id", "data", "meta", NULL}; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "y#y#y#", kwlist, &block_id, &block_id_length, &block, &block_length, &meta, &meta_length)) { + args, kwds, "s#s#s#", kwlist, &block_id, &block_id_length, &block, &block_length, &meta, &meta_length)) { PyErr_SetString(HermesError, "MidHermes.addBlock invalid parameters"); return NULL; } @@ -131,7 +125,7 @@ static PyObject *MidHermes_updBlock(pyhermes_MidHermesObject *self, PyObject *ar size_t block_id_length = 0, block_length = 0, meta_length = 0; static char *kwlist[] = {"id", "data", "meta", NULL}; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "y#y#y#", kwlist, &block_id, &block_id_length, &block, &block_length, &meta, &meta_length)) { + args, kwds, "s#s#s#", kwlist, &block_id, &block_id_length, &block, &block_length, &meta, &meta_length)) { PyErr_SetString(HermesError, "MidHermes.updBlock invalid parameters"); return NULL; } @@ -148,7 +142,7 @@ static PyObject *MidHermes_delBlock(pyhermes_MidHermesObject *self, PyObject *ar const char *block_id = NULL; size_t block_id_length = 0; static char *kwlist[] = {"id", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#", kwlist, &block_id, &block_id_length)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#", kwlist, &block_id, &block_id_length)) { PyErr_SetString(HermesError, "MidHermes.delBlock invalid parameters"); return NULL; } @@ -163,7 +157,7 @@ static PyObject *MidHermes_getBlock(pyhermes_MidHermesObject *self, PyObject *ar const char *block_id = NULL, *block = NULL, *meta = NULL; size_t block_id_length = 0, block_length = 0, meta_length = 0; static char *kwlist[] = {"id", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#", kwlist, &block_id, &block_id_length)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#", kwlist, &block_id, &block_id_length)) { PyErr_SetString(HermesError, "MidHermes.getBlock invalid parameters"); return NULL; } @@ -173,14 +167,14 @@ static PyObject *MidHermes_getBlock(pyhermes_MidHermesObject *self, PyObject *ar PyErr_SetString(HermesError, "MidHermes.getBlock error"); return NULL; } - return Py_BuildValue("y#y#", block, block_length, meta, meta_length); + return Py_BuildValue("OO", PyBytes_FromStringAndSize(block, block_length), PyBytes_FromStringAndSize(meta, meta_length)); } static PyObject *MidHermes_rotateBlock(pyhermes_MidHermesObject *self, PyObject *args, PyObject *kwds) { const char *block_id = NULL; size_t block_id_length = 0; static char *kwlist[] = {"id", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#", kwlist, &block_id, &block_id_length)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#", kwlist, &block_id, &block_id_length)) { PyErr_SetString(HermesError, "MidHermes.rotateBlock invalid parameters"); return NULL; } @@ -196,7 +190,7 @@ static PyObject *MidHermes_grantReadAccess(pyhermes_MidHermesObject *self, PyObj const char *block_id = NULL, *user_id = NULL; size_t block_id_length = 0, user_id_length = 0; static char *kwlist[] = {"id", "user", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &block_id, &block_id_length, &user_id, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &block_id, &block_id_length, &user_id, &user_id_length)) { PyErr_SetString(HermesError, "MidHermes.grantReadAccess invalid parameters"); return NULL; @@ -213,7 +207,7 @@ static PyObject *MidHermes_grantUpdateAccess(pyhermes_MidHermesObject *self, PyO const char *block_id = NULL, *user_id = NULL; size_t block_id_length = 0, user_id_length = 0; static char *kwlist[] = {"id", "user", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &block_id, &block_id_length, &user_id, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &block_id, &block_id_length, &user_id, &user_id_length)) { PyErr_SetString(HermesError, "MidHermes.grantUpdateAccess invalid parameters"); return NULL; @@ -230,7 +224,7 @@ static PyObject *MidHermes_denyReadAccess(pyhermes_MidHermesObject *self, PyObje const char *block_id = NULL, *user_id = NULL; size_t block_id_length = 0, user_id_length = 0; static char *kwlist[] = {"id", "user", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &block_id, &block_id_length, &user_id, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &block_id, &block_id_length, &user_id, &user_id_length)) { PyErr_SetString(HermesError, "MidHermes.denyReadAccess invalid parameters"); return NULL; @@ -247,7 +241,7 @@ static PyObject *MidHermes_denyUpdateAccess(pyhermes_MidHermesObject *self, PyOb const char *block_id = NULL, *user_id = NULL; size_t block_id_length = 0, user_id_length = 0; static char *kwlist[] = {"id", "user", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "y#y#", kwlist, &block_id, &block_id_length, &user_id, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#s#", kwlist, &block_id, &block_id_length, &user_id, &user_id_length)) { PyErr_SetString(HermesError, "MidHermes.denyUpdateAccess invalid parameters"); return NULL; diff --git a/pyhermes/py_transport_wrapper.c b/pyhermes/py_transport_wrapper.c index bf2f0f7..dd792e2 100644 --- a/pyhermes/py_transport_wrapper.c +++ b/pyhermes/py_transport_wrapper.c @@ -89,4 +89,4 @@ PyTypeObject pyhermes_HermesTransportWrapperType = { NULL,//(initproc)MidHermes_init, /* tp_init */ 0, /* tp_alloc */ HermesTransportWrapper_new, /* tp_new */ -}; \ No newline at end of file +}; diff --git a/pyhermes/setup.py b/pyhermes/setup.py index c5d0671..b7ca804 100644 --- a/pyhermes/setup.py +++ b/pyhermes/setup.py @@ -25,8 +25,6 @@ sources=['pyhermes.c', 'transport.c', 'py_secure_transport.c', 'py_midhermes.c', 'py_transport_wrapper.c', 'py_transport.c' ], - include_dirs=['../include'], - library_dirs=['../build'], libraries=['hermes_mid_hermes', 'hermes_mid_hermes_ll', 'hermes_credential_store', 'hermes_data_store', 'hermes_key_store', 'hermes_rpc', 'hermes_common', 'themis', 'soter', 'hermes_secure_transport']) @@ -42,14 +40,14 @@ classifiers=[ "Intended Audience :: Developers", "Natural Language :: English", - #"Operating System :: MacOS :: MacOS X", + "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX", #"Operating System :: POSIX :: BSD", "Operating System :: POSIX :: Linux", "Programming Language :: Python", - #"Programming Language :: Python :: 2", + "Programming Language :: Python :: 2", #"Programming Language :: Python :: 2.6", - #"Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", diff --git a/pyhermes/test.py b/pyhermes/test.py index ad7bad2..3a54214 100644 --- a/pyhermes/test.py +++ b/pyhermes/test.py @@ -20,13 +20,15 @@ # coding: utf-8 import socket import base64 +import threading try: + # Python3 from queue import Queue except ImportError: + # Python2 from Queue import Queue -import threading from unittest import TestCase, main import pyhermes @@ -34,7 +36,7 @@ QUEUE_TIMEOUT = THREAD_TIMEOUT = 2 -class TCPTransport: +class TCPTransport(object): def __init__(self, host, port): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((host, port)) @@ -63,7 +65,7 @@ def receive(self, needed_length): return b''.join(output) -class BufferTransport: +class BufferTransport(object): def __init__(self, buffer_in, buffer_out): self.buffer_in = buffer_in self.buffer_out = buffer_out @@ -92,9 +94,9 @@ def create_transport(self, user_id, private, public_id, public, transport, is_se def f(): hermes_transport = pyhermes.SecureHermesTransport(user_id, private, public_id, public, transport, is_server) if is_server: - result.put(hermes_transport, True) + result.put((hermes_transport, True), timeout=QUEUE_TIMEOUT) else: - result.put(hermes_transport, False) + result.put((hermes_transport, False), timeout=QUEUE_TIMEOUT) return f @@ -122,8 +124,8 @@ def test_secure_transport(self): transportA, transportB = [result.get(timeout=QUEUE_TIMEOUT) for _ in range(2)] def test_transport(self): - buffer_in = Queue(5) - buffer_out = Queue(5) + buffer_in = Queue() + buffer_out = Queue() transportClient = BufferTransport(buffer_out, buffer_in) transportServer = BufferTransport(buffer_in, buffer_out) hermes_transport_server = pyhermes.HermesTransport(transportServer) @@ -178,7 +180,7 @@ def test_secure_midhermes(self): data_store_transport2 = pyhermes.SecureHermesTransport( self.USER_ID1, self.PRIVATE_KEY1, self.DATA_STORE_ID, self.DATA_STORE_PUBLIC, TCPTransport("127.0.0.1", 8890), False) - except (ConnectionRefusedError, pyhermes.HermesTransportError): + except (IOError, pyhermes.HermesTransportError): self.skipTest("credential|key|data store service not started on 8888|8889|8890 port respectively") return @@ -195,7 +197,7 @@ def test_simple_transport_mid_hermes(self): credential_store_transport2 = pyhermes.HermesTransport(TCPTransport("127.0.0.1", 8888)) key_store_transport2 = pyhermes.HermesTransport(TCPTransport("127.0.0.1", 8889)) data_store_transport2 = pyhermes.HermesTransport(TCPTransport("127.0.0.1", 8890)) - except ConnectionRefusedError: + except IOError: self.skipTest("credential|key|data store service not started on 8888|8889|8890 port respectively") return self._test_mid_hermes(credential_store_transport1, key_store_transport1, data_store_transport1, diff --git a/pyhermes/tox.ini b/pyhermes/tox.ini new file mode 100644 index 0000000..5fa22d8 --- /dev/null +++ b/pyhermes/tox.ini @@ -0,0 +1,5 @@ +[tox] +envlist = py27,py34,py36 +[testenv] +deps=nose +commands={posargs:python -m unittest} diff --git a/pyhermes/transport.c b/pyhermes/transport.c index f23342a..8b456de 100644 --- a/pyhermes/transport.c +++ b/pyhermes/transport.c @@ -20,14 +20,29 @@ #include "transport.h" -#include - uint32_t transport_send(void *transport, const uint8_t *buf, const size_t buf_length) { if (!transport || !buf || !buf_length) { return HM_FAIL; } - PyObject_CallMethod((PyObject *) transport, "send", "y#", (const char *) buf, buf_length); + PyObject* buf_object = PyBytes_FromStringAndSize((char*)buf, buf_length); + if (!buf_object){ + return HM_FAIL; + } + PyObject* method_name = PyUnicode_FromString("send"); + if(!method_name){ + Py_DECREF(buf_object); + return HM_FAIL; + } + + PyObject* object = (PyObject*) transport; + Py_INCREF(object); + + PyObject_CallMethodObjArgs(object, method_name, buf_object, NULL); + + Py_DECREF(object); + Py_DECREF(method_name); + Py_DECREF(buf_object); return HM_SUCCESS; } @@ -40,7 +55,7 @@ uint32_t transport_recv(void *transport, uint8_t *buf, size_t buf_length) { return HM_FAIL; } if (!(PyBytes_Check(result) || ((size_t) PyBytes_Size(result) != buf_length))) { - Py_XDECREF(result); + Py_DECREF(result); return HM_FAIL; } memcpy(buf, PyBytes_AsString(result), buf_length); @@ -64,7 +79,7 @@ uint32_t transport_destroy(hm_rpc_transport_t **transport) { if (!transport || !(*transport) || !((*transport)->user_data)) { return HM_FAIL; } - Py_XDECREF((PyObject *) (*transport)->user_data); + Py_DECREF((PyObject *) (*transport)->user_data); free(*transport); *transport = NULL; return HM_SUCCESS; diff --git a/pyhermes/transport.h b/pyhermes/transport.h index b0857a2..1e11275 100644 --- a/pyhermes/transport.h +++ b/pyhermes/transport.h @@ -24,7 +24,9 @@ #define TRANSPORT_H #include +#include "bytesobject.h" #include +#include extern PyObject *HermesTransportError; From c463660ce12e51a0126d071985964b2a8ae2adfe Mon Sep 17 00:00:00 2001 From: Dmitriy Korneev Date: Mon, 13 Nov 2017 16:21:32 +0200 Subject: [PATCH 2/5] run pyhermes tests in circleci --- circle.yml | 7 +++++-- pyhermes/tox.ini | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 1bbc217..18bbbd1 100644 --- a/circle.yml +++ b/circle.yml @@ -20,13 +20,16 @@ dependencies: pre: - - sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev build-essential + - sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev build-essential python-dev python3-dev python-setuptool python3-setuptools - git clone https://github.com/cossacklabs/themis && cd themis && sudo make install override: - make - sudo make install + - sudo pip install tox test: override: - - make test + - make test + # run only TestHermesTransport until add support of running stores + - cd pyhermes && tox -- python -m unittest test.TestHermesTransport diff --git a/pyhermes/tox.ini b/pyhermes/tox.ini index 5fa22d8..ff75635 100644 --- a/pyhermes/tox.ini +++ b/pyhermes/tox.ini @@ -1,5 +1,4 @@ [tox] -envlist = py27,py34,py36 +envlist = py2,py3 [testenv] -deps=nose commands={posargs:python -m unittest} From 613586276d76e04c71c929c9921d6eee2de1cbf4 Mon Sep 17 00:00:00 2001 From: Dmitriy Korneev Date: Mon, 13 Nov 2017 16:24:19 +0200 Subject: [PATCH 3/5] fix package name --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 18bbbd1..5568595 100644 --- a/circle.yml +++ b/circle.yml @@ -20,7 +20,7 @@ dependencies: pre: - - sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev build-essential python-dev python3-dev python-setuptool python3-setuptools + - sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libssl-dev build-essential python-dev python3-dev python-setuptools python3-setuptools - git clone https://github.com/cossacklabs/themis && cd themis && sudo make install override: From c6e2ddc2e9e631b461033fcad1cfee891173c578 Mon Sep 17 00:00:00 2001 From: Dmitriy Korneev Date: Mon, 13 Nov 2017 16:45:41 +0200 Subject: [PATCH 4/5] set explicitly python versions --- circle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 5568595..4a33797 100644 --- a/circle.yml +++ b/circle.yml @@ -27,9 +27,10 @@ dependencies: - make - sudo make install - sudo pip install tox + - pyenv global 2.7.12 3.1.5 3.3.6 3.4.4 3.5.3 3.6.2 test: override: - make test # run only TestHermesTransport until add support of running stores - - cd pyhermes && tox -- python -m unittest test.TestHermesTransport + - cd pyhermes && tox -e py31,py33,py34,py35,py36,py27 -- python -m unittest test.TestHermesTransport From 638cdc2308cd9bef32eb75a0ed0040858d96d6dd Mon Sep 17 00:00:00 2001 From: Dmitriy Korneev Date: Mon, 13 Nov 2017 16:58:11 +0200 Subject: [PATCH 5/5] drop unsupported version --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 4a33797..2019315 100644 --- a/circle.yml +++ b/circle.yml @@ -27,10 +27,10 @@ dependencies: - make - sudo make install - sudo pip install tox - - pyenv global 2.7.12 3.1.5 3.3.6 3.4.4 3.5.3 3.6.2 + - pyenv global 2.7.12 3.3.6 3.4.4 3.5.3 3.6.2 test: override: - make test # run only TestHermesTransport until add support of running stores - - cd pyhermes && tox -e py31,py33,py34,py35,py36,py27 -- python -m unittest test.TestHermesTransport + - cd pyhermes && tox -e py33,py34,py35,py36,py27 -- python -m unittest test.TestHermesTransport