From a7f95d55cd660d1a212fa76a527063b3b7dbe8bb Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 13:58:25 +0100 Subject: [PATCH 1/7] Added changes to enable 3.11 builds --- .appveyor.yml | 4 ++++ .github/workflows/macosx.yml | 2 +- .github/workflows/manylinux.yml | 6 +++--- .github/workflows/opensuse-tumbleweed.yml | 2 +- .github/workflows/sdist.yml | 4 ++-- .travis.yml | 3 +++ setup.py | 1 + 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..b580525f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.10.6 install: - ps: | diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..24fa6ddd 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Setup Python diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..520e5ba6 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -5,16 +5,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - manylinux2010_x86_64 - manylinux_2_24_x86_64 - musllinux_1_1_x86_64 exclude: - image: manylinux2010_x86_64 - python-abi: cp310-cp310 + python-abi: cp311-cp311 - image: manylinux2010_i686 - python-abi: cp310-cp310 + python-abi: cp311-cp311 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..273f7f7f 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,7 +6,7 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..d7959208 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/setup.py b/setup.py index 9a3c9277..5c7e0da5 100644 --- a/setup.py +++ b/setup.py @@ -533,6 +533,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], From b7683774f747c7aed6a0b30e6045da679bc68760 Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:00:32 +0100 Subject: [PATCH 2/7] Added changes to enable 3.11 builds --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b580525f..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ environment: - python: 311 python_version: 3.11.2 - python: 311-x64 - python_version: 3.10.6 + python_version: 3.11.2 install: - ps: | From bddf28e68a2509a287f9889aaeadc3adab80ccbc Mon Sep 17 00:00:00 2001 From: Dan Vella Date: Fri, 17 Mar 2023 14:52:40 +0100 Subject: [PATCH 3/7] bumped isort to 5.11.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0545b12f..6b8fdf67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 2c58d43eedf72590e3e201252f5fc5ddae36f8c6 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 22 Mar 2023 20:36:45 +0100 Subject: [PATCH 4/7] Fix #244 - Fix failing test with libxmlsec-1.2.36, also make libxmlsec version available from Python. --- src/main.c | 13 ++++ tests/data/sign5-out-xmlsec_1_2_36_to_37.xml | 67 ++++++++++++++++++++ tests/test_ds.py | 6 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/data/sign5-out-xmlsec_1_2_36_to_37.xml diff --git a/src/main.c b/src/main.c index ffcae14f..5773db3b 100644 --- a/src/main.c +++ b/src/main.c @@ -119,6 +119,13 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { Py_RETURN_NONE; } +static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ + "get_libxmlsec_version() -> tuple\n" + "Returns Version tuple of wrapped libxml library."; +static PyObject* PyXmlSec_GetLibXmlSecVersion() { + return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -386,6 +393,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_PyShutdown__doc__ }, + { + "get_libxmlsec_version", + (PyCFunction)PyXmlSec_GetLibXmlSecVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlSecVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml new file mode 100644 index 00000000..f359b138 --- /dev/null +++ b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml @@ -0,0 +1,67 @@ + + + + + Hello, World! + + + + + + + + + + +HjY8ilZAIEM2tBbPn5mYO1ieIX4= + + +SIaj/6KY3C1SmDXU2++Gm31U1xTadFp04WhBgfsJFbxrL+q7GKSKN9kfQ+UpN9+i +D5fWmuavXEHe4Gw6RMaMEkq2URQo7F68+d5J/ajq8/l4n+xE6/reGScVwT6L4dEP +XXVJcAi2ZnQ3O7GTNvNGCPibL9mUcyCWBFZ92Uemtc/vJFCQ7ZyKMdMfACgxOwyN +T/9971oog241/2doudhonc0I/3mgPYWkZdX6yvr62mEjnG+oUZkhWYJ4ewZJ4hM4 +JjbFqZO+OEzDRSbw3DkmuBA/mtlx+3t13SESfEub5hqoMdVmtth/eTb64dsPdl9r +3k1ACVX9f8aHfQQdJOmLFQ== + + + + + + +Test Issuer +1 + +MIIE3zCCBEigAwIBAgIBBTANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tMB4X +DTAzMDMzMTA0MDIyMloXDTEzMDMyODA0MDIyMlowgb8xCzAJBgNVBAYTAlVTMRMw +EQYDVQQIEwpDYWxpZm9ybmlhMT0wOwYDVQQKEzRYTUwgU2VjdXJpdHkgTGlicmFy +eSAoaHR0cDovL3d3dy5hbGVrc2V5LmNvbS94bWxzZWMpMSEwHwYDVQQLExhFeGFt +cGxlcyBSU0EgQ2VydGlmaWNhdGUxFjAUBgNVBAMTDUFsZWtzZXkgU2FuaW4xITAf +BgkqhkiG9w0BCQEWEnhtbHNlY0BhbGVrc2V5LmNvbTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAJe4/rQ/gzV4FokE7CthjL/EXwCBSkXm2c3p4jyXO0Wt +quaNC3dxBwFPfPl94hmq3ZFZ9PHPPbp4RpYRnLZbRjlzVSOq954AXOXpSew7nD+E +mTqQrd9+ZIbGJnLOMQh5fhMVuOW/1lYCjWAhTCcYZPv7VXD2M70vVXDVXn6ZrqTg +qkVHE6gw1aCKncwg7OSOUclUxX8+Zi10v6N6+PPslFc5tKwAdWJhVLTQ4FKG+F53 +7FBDnNK6p4xiWryy/vPMYn4jYGvHUUk3eH4lFTCr+rSuJY8i/KNIf/IKim7g/o3w +Ae3GM8xrof2mgO8GjK/2QDqOQhQgYRIf4/wFsQXVZcMCAwEAAaOCAVcwggFTMAkG +A1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRp +ZmljYXRlMB0GA1UdDgQWBBQkhCzy1FkgYosuXIaQo6owuicanDCB+AYDVR0jBIHw +MIHtgBS0ue+a5pcOaGUemM76VQ2JBttMfKGB0aSBzjCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tggEA +MA0GCSqGSIb3DQEBBAUAA4GBALU/mzIxSv8vhDuomxFcplzwdlLZbvSQrfoNkMGY +1UoS3YJrN+jZLWKSyWE3mIaPpElqXiXQGGkwD5iPQ1iJMbI7BeLvx6ZxX/f+c8Wn +ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL +NJ2D + + + + + diff --git a/tests/test_ds.py b/tests/test_ds.py index 694ad431..38f0b25c 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -182,7 +182,11 @@ def test_sign_case5(self): self.assertEqual("rsakey.pem", ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign5-out.xml"), root) + if (1, 2, 36) <= xmlsec.get_libxmlsec_version() <= (1, 2, 37): + expected_xml_file = 'sign5-out-xmlsec_1_2_36_to_37.xml' + else: + expected_xml_file = 'sign5-out.xml' + self.assertEqual(self.load_xml(expected_xml_file), root) def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() From 1cf6785b3e06c2b8f6cac1266cf8f1934650bc4b Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Wed, 21 Dec 2022 14:01:14 +0100 Subject: [PATCH 5/7] Fix #164 - Add support for loading keys from engine (e.g. pkcs11). --- README.rst | 2 +- src/keys.c | 47 +++++++++++++++++++++++++++++++++++++++++ src/xmlsec/__init__.pyi | 2 ++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 34bdd377..e1924652 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Check the `examples `_ se Requirements ************ - ``libxml2 >= 2.9.1`` -- ``libxmlsec1 >= 1.2.18`` +- ``libxmlsec1 >= 1.2.33`` Install ******* diff --git a/src/keys.c b/src/keys.c index 1362b128..1440331c 100644 --- a/src/keys.c +++ b/src/keys.c @@ -185,6 +185,47 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* return NULL; } +static const char PyXmlSec_KeyFromEngine__doc__[] = \ + "from_engine(engine_and_key_id) -> xmlsec.Key\n" + "Loads PKI key from an engine.\n\n" + ":param engine_and_key_id: engine and key id, i.e. 'pkcs11;pkcs11:token=XmlsecToken;object=XmlsecKey;pin-value=password'\n" + ":type engine_and_key_id: :class:`str`, " + ":return: pointer to newly created key\n" + ":rtype: :class:`~xmlsec.Key`"; +static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = {"engine_and_key_id", NULL}; + + const char* engine_and_key_id = NULL; + PyXmlSec_Key* key = NULL; + + PYXMLSEC_DEBUG("load key from engine - start"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s:from_engine", kwlist, &engine_and_key_id)) { + goto ON_FAIL; + } + + if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; + + Py_BEGIN_ALLOW_THREADS; + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), + (void*)engine_and_key_id); + Py_END_ALLOW_THREADS; + + if (key->handle == NULL) { + PyXmlSec_SetLastError("cannot read key"); + goto ON_FAIL; + } + + key->is_own = 1; + + PYXMLSEC_DEBUG("load key from engine - ok"); + return (PyObject*)key; + +ON_FAIL: + PYXMLSEC_DEBUG("load key from engine - fail"); + Py_XDECREF(key); + return NULL; +} + static const char PyXmlSec_KeyGenerate__doc__[] = \ "generate(klass, size, type) -> xmlsec.Key\n" "Generates key of kind ``klass`` with ``size`` and ``type``.\n\n" @@ -494,6 +535,12 @@ static PyMethodDef PyXmlSec_KeyMethods[] = { METH_CLASS|METH_VARARGS|METH_KEYWORDS, PyXmlSec_KeyFromFile__doc__ }, + { + "from_engine", + (PyCFunction)PyXmlSec_KeyFromEngine, + METH_CLASS|METH_VARARGS|METH_KEYWORDS, + PyXmlSec_KeyFromEngine__doc__ + }, { "generate", (PyCFunction)PyXmlSec_KeyGenerate, diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index 56356e55..6c326f56 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -49,6 +49,8 @@ class Key: @classmethod def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod + def from_engine(cls: type[Self], engine_and_key_id: AnyStr) -> Self: ... + @classmethod def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... From 0b5939fc65e98cebb669d311b4fb58844bf887e5 Mon Sep 17 00:00:00 2001 From: Tomas Divis Date: Thu, 13 Apr 2023 14:28:02 +0200 Subject: [PATCH 6/7] Fix #164 - Add tests for pkcs11 (softhsm) key. --- .github/workflows/sdist.yml | 3 +- tests/softhsm_setup.py | 265 ++++++++++++++++++++++++++++++++++++ tests/test_pkcs11.py | 57 ++++++++ 3 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 tests/softhsm_setup.py create mode 100644 tests/test_pkcs11.py diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index d7959208..6ca7a657 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -16,9 +16,8 @@ jobs: run: | python setup.py sdist - name: Install test dependencies - env: - PYXMLSEC_STATIC_DEPS: true run: | + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl pip install --upgrade -r requirements-test.txt pip install black # for stub generation tests pip install dist/xmlsec-$(python setup.py --version).tar.gz diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py new file mode 100644 index 00000000..0a7c37de --- /dev/null +++ b/tests/softhsm_setup.py @@ -0,0 +1,265 @@ +""" +Testing the PKCS#11 shim layer. +Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj +under licence "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import traceback +import unittest +from typing import Dict, List, Optional, Tuple + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") + + +def paths_for_component(component: str, default_paths: List[str]): + env_path = os.environ.get(component) + return [env_path] if env_path else default_paths + + +def find_alts(component_name, alts: List[str]) -> str: + for a in alts: + if os.path.exists(a): + return a + raise unittest.SkipTest("Required component is missing: {}".format(component_name)) + + +def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: + env = {} + if softhsm_conf is not None: + env['SOFTHSM_CONF'] = softhsm_conf + env['SOFTHSM2_CONF'] = softhsm_conf + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + out, err = proc.communicate() + if err is not None and len(err) > 0: + logging.error(err) + if out is not None and len(out) > 0: + logging.debug(out) + rv = proc.wait() + if rv: + with open(softhsm_conf) as f: + conf = f.read() + msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' + msg = msg.format( + cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + ) + raise RuntimeError(msg) + return out, err + + +component_default_paths: Dict[str, List[str]] = { + 'P11_MODULE': [ + '/usr/lib/softhsm/libsofthsm2.so', + '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', + '/usr/lib/softhsm/libsofthsm.so', + '/usr/lib64/softhsm/libsofthsm2.so', + ], + 'P11_ENGINE': [ + '/usr/lib/ssl/engines/libpkcs11.so', + '/usr/lib/engines/engine_pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/libpkcs11.so', + '/usr/lib64/engines-3/pkcs11.so', + '/usr/lib64/engines-3/libpkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/libpkcs11.so', + ], + 'PKCS11_TOOL': [ + '/usr/bin/pkcs11-tool', + ], + 'SOFTHSM': [ + '/usr/bin/softhsm2-util', + '/usr/bin/softhsm', + ], + 'OPENSSL': [ + '/usr/bin/openssl', + ], +} + +component_path: Dict[str, str] = { + component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) + for component_name, default_paths in component_default_paths.items() +} + +softhsm_version = 1 +if component_path['SOFTHSM'].endswith('softhsm2-util'): + softhsm_version = 2 + +openssl_version = subprocess.check_output([component_path['OPENSSL'], + 'version'] + )[8:11].decode() + +p11_test_files: List[str] = [] +softhsm_conf: Optional[str] = None +softhsm_db: Optional[str] = None + + +def _temp_file() -> str: + f = tempfile.NamedTemporaryFile(delete=False) + p11_test_files.append(f.name) + return f.name + + +def _temp_dir() -> str: + d = tempfile.mkdtemp() + p11_test_files.append(d) + return d + + +@unittest.skipIf(component_path['P11_MODULE'] is None, "SoftHSM PKCS11 module not installed") +def setup() -> None: + logging.debug("Creating test pkcs11 token using softhsm") + try: + global softhsm_conf + softhsm_conf = _temp_file() + logging.debug("Generating softhsm.conf") + with open(softhsm_conf, "w") as f: + if softhsm_version == 2: + softhsm_db = _temp_dir() + f.write(""" +# Generated by test +directories.tokendir = %s +objectstore.backend = file +log.level = DEBUG +""" % softhsm_db) + else: + softhsm_db = _temp_file() + f.write(""" +# Generated by test +0:%s +""" % softhsm_db) + + logging.debug("Initializing the token") + out, err = run_cmd([component_path['SOFTHSM'], + '--slot', '0', + '--label', 'test', + '--init-token', + '--pin', 'secret1', + '--so-pin', 'secret2'], + softhsm_conf=softhsm_conf) + + # logging.debug("Generating 1024 bit RSA key in token") + # run_cmd([component_path['PKCS11_TOOL'], + # '--module', component_path['P11_MODULE'], + # '-l', + # '-k', + # '--key-type', 'rsa:1024', + # '--id', 'a1b2', + # '--label', 'test', + # '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + hash_priv_key = _temp_file() + logging.debug("Converting test private key to format for softhsm") + run_cmd([component_path['OPENSSL'], 'pkcs8', + '-topk8', + '-inform', 'PEM', + '-outform', 'PEM', + '-nocrypt', + '-in', os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', hash_priv_key], softhsm_conf=softhsm_conf) + + logging.debug("Importing the test key to softhsm") + run_cmd([component_path['SOFTHSM'], + '--import', hash_priv_key, + '--token', 'test', + '--id', 'a1b2', + '--label', 'test', + '--pin', 'secret1'], + softhsm_conf=softhsm_conf) + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + signer_cert_pem = _temp_file() + openssl_conf = _temp_file() + logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) + with open(openssl_conf, "w") as f: + # Might be needed with some versions of openssl, but in more recent versions dynamic_path breaks it. + # dynamic_path = ( + # "dynamic_path = %s" % component_path['P11_ENGINE'] + # if openssl_version.startswith(b'1.') + # else "" + # ) + f.write("\n".join([ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ])) + + with open(openssl_conf, "r") as f: + logging.debug('-------- START DEBUG openssl_conf --------') + logging.debug(f.readlines()) + logging.debug('-------- END DEBUG openssl_conf --------') + logging.debug('-------- START DEBUG paths --------') + logging.debug(run_cmd(['ls', '-ld', component_path['P11_ENGINE']])) + logging.debug(run_cmd(['ls', '-ld', component_path['P11_MODULE']])) + logging.debug('-------- END DEBUG paths --------') + + signer_cert_der = _temp_file() + + logging.debug("Generating self-signed certificate") + run_cmd([component_path['OPENSSL'], 'req', + '-new', + '-x509', + '-subj', "/CN=Test Signer", + '-engine', 'pkcs11', + '-config', openssl_conf, + '-keyform', 'engine', + '-key', 'label_test', + '-passin', 'pass:secret1', + '-out', signer_cert_pem], softhsm_conf=softhsm_conf) + + run_cmd([component_path['OPENSSL'], 'x509', + '-inform', 'PEM', + '-outform', 'DER', + '-in', signer_cert_pem, + '-out', signer_cert_der], softhsm_conf=softhsm_conf) + + logging.debug("Importing certificate into token") + + run_cmd([component_path['PKCS11_TOOL'], + '--module', component_path['P11_MODULE'], + '-l', + '--slot-index', '0', + '--id', 'a1b2', + '--label', 'test', + '-y', 'cert', + '-w', signer_cert_der, + '--pin', 'secret1'], softhsm_conf=softhsm_conf) + + # TODO: Should be teardowned in teardown: + os.environ['SOFTHSM_CONF'] = softhsm_conf + os.environ['SOFTHSM2_CONF'] = softhsm_conf + + except Exception as ex: + print("-" * 64) + traceback.print_exc() + print("-" * 64) + logging.error("PKCS11 tests disabled: unable to initialize test token: %s" % ex) + raise ex + + +def teardown() -> None: + global p11_test_files + for o in p11_test_files: + if os.path.exists(o): + if os.path.isdir(o): + shutil.rmtree(o) + else: + os.unlink(o) + p11_test_files = [] diff --git a/tests/test_pkcs11.py b/tests/test_pkcs11.py new file mode 100644 index 00000000..accd29ae --- /dev/null +++ b/tests/test_pkcs11.py @@ -0,0 +1,57 @@ +import xmlsec +from tests import base +from xmlsec import constants as consts + +KEY_URL = "pkcs11;pkcs11:token=test;object=test;pin-value=secret1" + + +def setUpModule(): + from tests import softhsm_setup + + softhsm_setup.setup() + + +def tearDownModule(): + from tests import softhsm_setup + + softhsm_setup.teardown() + + +class TestKeys(base.TestMemoryLeaks): + def test_del_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + + def test_sign_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaises(TypeError): + ctx.sign('') + + def test_sign_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + ctx.sign(self.load_xml('sign1-in.xml')) + + def test_sign_case1(self): + """Should sign a pre-constructed template file using a key from a pkcs11 engine.""" + root = self.load_xml("sign1-in.xml") + sign = xmlsec.tree.find_node(root, consts.NodeSignature) + self.assertIsNotNone(sign) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + ctx.key.name = 'rsakey.pem' + self.assertEqual("rsakey.pem", ctx.key.name) + + ctx.sign(sign) + self.assertEqual(self.load_xml("sign1-out.xml"), root) From 4f5daea286df89c64fbfc615f422be62b2cdf114 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:55:28 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/softhsm_setup.py | 220 +++++++++++++++++++++++++++-------------- 1 file changed, 147 insertions(+), 73 deletions(-) diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py index 0a7c37de..432d4b1b 100644 --- a/tests/softhsm_setup.py +++ b/tests/softhsm_setup.py @@ -45,7 +45,11 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: conf = f.read() msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' msg = msg.format( - cmd=" ".join(args), code=rv, out=out.strip(), err=err.strip(), conf=conf, + cmd=" ".join(args), + code=rv, + out=out.strip(), + err=err.strip(), + conf=conf, ) raise RuntimeError(msg) return out, err @@ -90,9 +94,7 @@ def run_cmd(args, softhsm_conf=None) -> Tuple[bytes, bytes]: if component_path['SOFTHSM'].endswith('softhsm2-util'): softhsm_version = 2 -openssl_version = subprocess.check_output([component_path['OPENSSL'], - 'version'] - )[8:11].decode() +openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() p11_test_files: List[str] = [] softhsm_conf: Optional[str] = None @@ -121,27 +123,41 @@ def setup() -> None: with open(softhsm_conf, "w") as f: if softhsm_version == 2: softhsm_db = _temp_dir() - f.write(""" + f.write( + """ # Generated by test directories.tokendir = %s objectstore.backend = file log.level = DEBUG -""" % softhsm_db) +""" + % softhsm_db + ) else: softhsm_db = _temp_file() - f.write(""" + f.write( + """ # Generated by test 0:%s -""" % softhsm_db) +""" + % softhsm_db + ) logging.debug("Initializing the token") - out, err = run_cmd([component_path['SOFTHSM'], - '--slot', '0', - '--label', 'test', - '--init-token', - '--pin', 'secret1', - '--so-pin', 'secret2'], - softhsm_conf=softhsm_conf) + out, err = run_cmd( + [ + component_path['SOFTHSM'], + '--slot', + '0', + '--label', + 'test', + '--init-token', + '--pin', + 'secret1', + '--so-pin', + 'secret2', + ], + softhsm_conf=softhsm_conf, + ) # logging.debug("Generating 1024 bit RSA key in token") # run_cmd([component_path['PKCS11_TOOL'], @@ -155,26 +171,45 @@ def setup() -> None: hash_priv_key = _temp_file() logging.debug("Converting test private key to format for softhsm") - run_cmd([component_path['OPENSSL'], 'pkcs8', - '-topk8', - '-inform', 'PEM', - '-outform', 'PEM', - '-nocrypt', - '-in', os.path.join(DATA_DIR, 'rsakey.pem'), - '-out', hash_priv_key], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'pkcs8', + '-topk8', + '-inform', + 'PEM', + '-outform', + 'PEM', + '-nocrypt', + '-in', + os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', + hash_priv_key, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing the test key to softhsm") - run_cmd([component_path['SOFTHSM'], - '--import', hash_priv_key, - '--token', 'test', - '--id', 'a1b2', - '--label', 'test', - '--pin', 'secret1'], - softhsm_conf=softhsm_conf) - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--pin', 'secret1', '-O'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['SOFTHSM'], + '--import', + hash_priv_key, + '--token', + 'test', + '--id', + 'a1b2', + '--label', + 'test', + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) + run_cmd( + [component_path['PKCS11_TOOL'], '--module', component_path['P11_MODULE'], '-l', '--pin', 'secret1', '-O'], + softhsm_conf=softhsm_conf, + ) signer_cert_pem = _temp_file() openssl_conf = _temp_file() logging.debug("Generating OpenSSL config for version {}".format(openssl_version)) @@ -185,21 +220,25 @@ def setup() -> None: # if openssl_version.startswith(b'1.') # else "" # ) - f.write("\n".join([ - "openssl_conf = openssl_def", - "[openssl_def]", - "engines = engine_section", - "[engine_section]", - "pkcs11 = pkcs11_section", - "[req]", - "distinguished_name = req_distinguished_name", - "[req_distinguished_name]", - "[pkcs11_section]", - "engine_id = pkcs11", - # dynamic_path, - "MODULE_PATH = %s" % component_path['P11_MODULE'], - "init = 0", - ])) + f.write( + "\n".join( + [ + "openssl_conf = openssl_def", + "[openssl_def]", + "engines = engine_section", + "[engine_section]", + "pkcs11 = pkcs11_section", + "[req]", + "distinguished_name = req_distinguished_name", + "[req_distinguished_name]", + "[pkcs11_section]", + "engine_id = pkcs11", + # dynamic_path, + "MODULE_PATH = %s" % component_path['P11_MODULE'], + "init = 0", + ] + ) + ) with open(openssl_conf, "r") as f: logging.debug('-------- START DEBUG openssl_conf --------') @@ -213,34 +252,69 @@ def setup() -> None: signer_cert_der = _temp_file() logging.debug("Generating self-signed certificate") - run_cmd([component_path['OPENSSL'], 'req', - '-new', - '-x509', - '-subj', "/CN=Test Signer", - '-engine', 'pkcs11', - '-config', openssl_conf, - '-keyform', 'engine', - '-key', 'label_test', - '-passin', 'pass:secret1', - '-out', signer_cert_pem], softhsm_conf=softhsm_conf) - - run_cmd([component_path['OPENSSL'], 'x509', - '-inform', 'PEM', - '-outform', 'DER', - '-in', signer_cert_pem, - '-out', signer_cert_der], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['OPENSSL'], + 'req', + '-new', + '-x509', + '-subj', + "/CN=Test Signer", + '-engine', + 'pkcs11', + '-config', + openssl_conf, + '-keyform', + 'engine', + '-key', + 'label_test', + '-passin', + 'pass:secret1', + '-out', + signer_cert_pem, + ], + softhsm_conf=softhsm_conf, + ) + + run_cmd( + [ + component_path['OPENSSL'], + 'x509', + '-inform', + 'PEM', + '-outform', + 'DER', + '-in', + signer_cert_pem, + '-out', + signer_cert_der, + ], + softhsm_conf=softhsm_conf, + ) logging.debug("Importing certificate into token") - run_cmd([component_path['PKCS11_TOOL'], - '--module', component_path['P11_MODULE'], - '-l', - '--slot-index', '0', - '--id', 'a1b2', - '--label', 'test', - '-y', 'cert', - '-w', signer_cert_der, - '--pin', 'secret1'], softhsm_conf=softhsm_conf) + run_cmd( + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--slot-index', + '0', + '--id', + 'a1b2', + '--label', + 'test', + '-y', + 'cert', + '-w', + signer_cert_der, + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) # TODO: Should be teardowned in teardown: os.environ['SOFTHSM_CONF'] = softhsm_conf