From befdc142d91cceb16ecc2864aa4beaddc2f9e738 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 1 Feb 2023 20:43:03 +0100 Subject: [PATCH] Initialize python bindings from pythonix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copies part of the tree from Pythonix at https://github.com/Mic92/pythonix/commit/fbc84900b5dcdde558c68d98ab746d76e7884ded into the ./python subdirectory. These Python bindings don't build yet with the current Nix version, which is why they're not built yet in this commit. Co-Authored-By: Jörg Thalheim --- python/LICENSE.md | 7 ++ python/meson.build | 13 +++ python/src/eval.cc | 60 ++++++++++ python/src/internal/errors.hh | 8 ++ python/src/internal/eval.hh | 8 ++ python/src/internal/nix-to-python.hh | 13 +++ python/src/internal/ptr.hh | 13 +++ python/src/internal/python-to-nix.hh | 15 +++ python/src/meson.build | 12 ++ python/src/nix-to-python.cc | 93 ++++++++++++++++ python/src/python-module.cc | 53 +++++++++ python/src/python-to-nix.cc | 159 +++++++++++++++++++++++++++ python/tests.py | 20 ++++ 13 files changed, 474 insertions(+) create mode 100644 python/LICENSE.md create mode 100644 python/meson.build create mode 100644 python/src/eval.cc create mode 100644 python/src/internal/errors.hh create mode 100644 python/src/internal/eval.hh create mode 100644 python/src/internal/nix-to-python.hh create mode 100644 python/src/internal/ptr.hh create mode 100644 python/src/internal/python-to-nix.hh create mode 100644 python/src/meson.build create mode 100644 python/src/nix-to-python.cc create mode 100644 python/src/python-module.cc create mode 100644 python/src/python-to-nix.cc create mode 100644 python/tests.py diff --git a/python/LICENSE.md b/python/LICENSE.md new file mode 100644 index 000000000000..79695402947a --- /dev/null +++ b/python/LICENSE.md @@ -0,0 +1,7 @@ +Copyright 2017 Jörg Thalheim + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/python/meson.build b/python/meson.build new file mode 100644 index 000000000000..f7d86300c474 --- /dev/null +++ b/python/meson.build @@ -0,0 +1,13 @@ +project('python-nix', 'cpp', + version : '0.1.8', + license : 'LGPL-2.0', +) + +python_mod = import('python3') +python_dep = dependency('python3', required : true) +nix_expr_dep = dependency('nix-expr', required: true) + +python = python_mod.find_python() +test('python test', python, args : files('tests.py')) + +subdir('src') diff --git a/python/src/eval.cc b/python/src/eval.cc new file mode 100644 index 000000000000..b3e5af118ee3 --- /dev/null +++ b/python/src/eval.cc @@ -0,0 +1,60 @@ +#include "internal/eval.hh" +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/python-to-nix.hh" +#include + +#include +#include + +namespace pythonnix { + +const char *currentExceptionTypeName() { + int status; + auto res = abi::__cxa_demangle(abi::__cxa_current_exception_type()->name(), 0, + 0, &status); + return res ? res : "(null)"; +} + +static PyObject *_eval(const char *expression, PyObject *vars) { + nix::Strings storePath; + nix::EvalState state(storePath, nix::openStore()); + + nix::Env *env = nullptr; + auto staticEnv = pythonToNixEnv(state, vars, &env); + if (!staticEnv) { + return nullptr; + } + + auto e = state.parseExprFromString(expression, ".", *staticEnv); + nix::Value v; + e->eval(state, *env, v); + + state.forceValueDeep(v); + + nix::PathSet context; + return nixToPythonObject(state, v, context); +} + +PyObject *eval(PyObject *self, PyObject *args, PyObject *keywds) { + const char *expression = nullptr; + PyObject *vars = nullptr; + + const char *kwlist[] = {"expression", "vars", nullptr}; + + if (!PyArg_ParseTupleAndKeywords(args, keywds, "s|O!", + const_cast(kwlist), &expression, + &PyDict_Type, &vars)) { + return nullptr; + } + + try { + return _eval(expression, vars); + } catch (nix::Error &e) { + return PyErr_Format(NixError, "%s", e.what()); + } catch (...) { + return PyErr_Format(NixError, "unexpected C++ exception: '%s'", + currentExceptionTypeName()); + } +} +} // namespace pythonnix diff --git a/python/src/internal/errors.hh b/python/src/internal/errors.hh new file mode 100644 index 000000000000..86d0cf577bc7 --- /dev/null +++ b/python/src/internal/errors.hh @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace pythonnix { + +extern PyObject *NixError; +} diff --git a/python/src/internal/eval.hh b/python/src/internal/eval.hh new file mode 100644 index 000000000000..dc6fe1f75809 --- /dev/null +++ b/python/src/internal/eval.hh @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace pythonnix { + +PyObject *eval(PyObject *self, PyObject *args, PyObject *kwdict); +} diff --git a/python/src/internal/nix-to-python.hh b/python/src/internal/nix-to-python.hh new file mode 100644 index 000000000000..27849fd03290 --- /dev/null +++ b/python/src/internal/nix-to-python.hh @@ -0,0 +1,13 @@ +#pragma once + +#include + +#include + +#include + +namespace pythonnix { + +PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, + nix::PathSet &context); +} // namespace pythonnix diff --git a/python/src/internal/ptr.hh b/python/src/internal/ptr.hh new file mode 100644 index 000000000000..165587728484 --- /dev/null +++ b/python/src/internal/ptr.hh @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +namespace pythonnix { + +struct PyObjectDeleter { + void operator()(PyObject *const obj) { Py_DECREF(obj); } +}; + +typedef std::unique_ptr PyObjPtr; +} // namespace pythonnix diff --git a/python/src/internal/python-to-nix.hh b/python/src/internal/python-to-nix.hh new file mode 100644 index 000000000000..c0ddaf012e17 --- /dev/null +++ b/python/src/internal/python-to-nix.hh @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +#include +#include + +namespace pythonnix { + +nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj); + +std::optional pythonToNixEnv(nix::EvalState &state, + PyObject *vars, nix::Env **env); +} // namespace pythonnix diff --git a/python/src/meson.build b/python/src/meson.build new file mode 100644 index 000000000000..a50f866d84bd --- /dev/null +++ b/python/src/meson.build @@ -0,0 +1,12 @@ +src = [ + 'nix-to-python.cc', + 'python-to-nix.cc', + 'eval.cc', + 'python-module.cc', +] + +python_mod.extension_module('nix', src, + dependencies : [python_dep, nix_expr_dep], + install: true, + install_dir: python_mod.sysconfig_path('platlib'), + cpp_args: ['-std=c++17', '-fvisibility=hidden']) diff --git a/python/src/nix-to-python.cc b/python/src/nix-to-python.cc new file mode 100644 index 000000000000..399f8d6f2ce8 --- /dev/null +++ b/python/src/nix-to-python.cc @@ -0,0 +1,93 @@ +#include + +#include "internal/errors.hh" +#include "internal/nix-to-python.hh" +#include "internal/ptr.hh" + +namespace pythonnix { + +PyObject *nixToPythonObject(nix::EvalState &state, nix::Value &v, + nix::PathSet &context) { + switch (v.type()) { + case nix::nInt: + return PyLong_FromLong(v.integer); + + case nix::nBool: + if (v.boolean) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + case nix::nString: + copyContext(v, context); + return PyUnicode_FromString(v.string.s); + + case nix::nPath: + return PyUnicode_FromString(state.copyPathToStore(context, v.path).c_str()); + + case nix::nNull: + Py_RETURN_NONE; + + case nix::nAttrs: { + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + PyObjPtr dict(PyDict_New()); + if (!dict) { + return (PyObject *)nullptr; + } + + nix::StringSet names; + + for (auto &j : *v.attrs) { + names.insert(j.name); + } + for (auto &j : names) { + nix::Attr &a(*v.attrs->find(state.symbols.create(j))); + + auto value = nixToPythonObject(state, *a.value, context); + if (!value) { + return nullptr; + } + PyDict_SetItemString(dict.get(), j.c_str(), value); + } + return dict.release(); + } else { + return nixToPythonObject(state, *i->value, context); + } + } + + case nix::nList: { + PyObjPtr list(PyList_New(v.listSize())); + if (!list) { + return (PyObject *)nullptr; + } + + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto value = nixToPythonObject(state, *v.listElems()[n], context); + if (!value) { + return nullptr; + } + PyList_SET_ITEM(list.get(), n, value); + } + return list.release(); + } + + case nix::nExternal: + return PyUnicode_FromString(""); + + case nix::nThunk: + return PyUnicode_FromString(""); + + case nix::nFunction: + return PyUnicode_FromString(""); + + case nix::nFloat: + return PyFloat_FromDouble(v.fpoint); + + default: + PyErr_Format(NixError, "cannot convert nix type '%s' to a python object", + showType(v).c_str()); + return nullptr; + } +} +} // namespace pythonnix diff --git a/python/src/python-module.cc b/python/src/python-module.cc new file mode 100644 index 000000000000..4dfd5aa93a8a --- /dev/null +++ b/python/src/python-module.cc @@ -0,0 +1,53 @@ +#include + +#include "internal/eval.hh" +#include "internal/ptr.hh" + +#include + +#include + +namespace pythonnix { + +#define _public_ __attribute__((visibility("default"))) + +PyObject *NixError = nullptr; + +static PyMethodDef NixMethods[] = {{"eval", (PyCFunction)eval, + METH_VARARGS | METH_KEYWORDS, + "Eval nix expression"}, + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef nixmodule = { + PyModuleDef_HEAD_INIT, "nix", "Nix expression bindings", + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + NixMethods}; + +extern "C" _public_ PyObject *PyInit_nix(void) { + nix::initGC(); + + PyObjPtr m(PyModule_Create(&nixmodule)); + + if (!m) { + return nullptr; + } + + NixError = PyErr_NewExceptionWithDoc( + "nix.NixError", /* char *name */ + "Base exception class for the nix module.", /* char *doc */ + NULL, /* PyObject *base */ + NULL /* PyObject *dict */ + ); + + if (!NixError) { + return nullptr; + } + + if (PyModule_AddObject(m.get(), "NixError", NixError) == -1) { + return nullptr; + } + + return m.release(); +} +} // namespace pythonnix diff --git a/python/src/python-to-nix.cc b/python/src/python-to-nix.cc new file mode 100644 index 000000000000..f79134dfa4ac --- /dev/null +++ b/python/src/python-to-nix.cc @@ -0,0 +1,159 @@ +#include + +#include "internal/errors.hh" +#include "internal/ptr.hh" +#include "internal/python-to-nix.hh" + +#include + +namespace pythonnix { + +static const char *checkNullByte(const char *str, const Py_ssize_t size) { + for (Py_ssize_t i = 0; i < size; i++) { + if (str[0] == '\0') { + PyErr_Format(NixError, "invalid character: nix strings are not allowed " + "to contain null bytes"); + return nullptr; + } + } + return str; +} + +static const char *checkAttrKey(PyObject *obj) { + Py_ssize_t size = 0; + + if (!PyUnicode_Check(obj)) { + PyObjPtr typeName(PyObject_Str(PyObject_Type(obj))); + if (!typeName) { + return nullptr; + } + auto utf8 = PyUnicode_AsUTF8AndSize(typeName.get(), &size); + if (!utf8) { + return nullptr; + } + PyErr_Format(NixError, "key of nix attrsets must be strings, got type: %s", + utf8); + return nullptr; + } + + auto utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + if (!utf8) { + return nullptr; + } + + return checkNullByte(utf8, size); +} + +static std::optional dictToAttrSet(PyObject *obj, + nix::EvalState &state) { + PyObject *key = nullptr, *val = nullptr; + Py_ssize_t pos = 0; + + nix::ValueMap attrs; + while (PyDict_Next(obj, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + attrs[state.symbols.create(name)] = attrVal; + } + + return attrs; +} + +nix::Value *pythonToNixValue(nix::EvalState &state, PyObject *obj) { + auto v = state.allocValue(); + + if (obj == Py_True && obj == Py_False) { + nix::mkBool(*v, obj == Py_True); + } else if (obj == Py_None) { + nix::mkNull(*v); + } else if (PyBytes_Check(obj)) { + auto str = checkNullByte(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + if (!str) { + return nullptr; + } + + nix::mkString(*v, str); + } else if (PyUnicode_Check(obj)) { + Py_ssize_t size; + const char *utf8 = PyUnicode_AsUTF8AndSize(obj, &size); + auto str = checkNullByte(utf8, size); + if (!str) { + return nullptr; + } + + nix::mkString(*v, utf8); + } else if (PyFloat_Check(obj)) { + nix::mkFloat(*v, PyFloat_AS_DOUBLE(obj)); + } else if (PyLong_Check(obj)) { + nix::mkInt(*v, PyLong_AsLong(obj)); + } else if (PyList_Check(obj)) { + state.mkList(*v, PyList_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyList_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyTuple_Check(obj)) { + state.mkList(*v, PyTuple_GET_SIZE(obj)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(obj); i++) { + auto val = pythonToNixValue(state, PyTuple_GET_ITEM(obj, i)); + if (!val) { + return nullptr; + } + v->listElems()[i] = val; + } + } else if (PyDict_Check(obj)) { + auto attrs = dictToAttrSet(obj, state); + if (!attrs) { + return nullptr; + } + state.mkAttrs(*v, attrs->size()); + for (auto &attr : *attrs) { + v->attrs->push_back(nix::Attr(attr.first, attr.second)); + } + v->attrs->sort(); + } + return v; +} + +std::optional pythonToNixEnv(nix::EvalState &state, + PyObject *vars, nix::Env **env) { + Py_ssize_t pos = 0; + PyObject *key = nullptr, *val = nullptr; + + *env = &state.allocEnv(vars ? PyDict_Size(vars) : 0); + (*env)->up = &state.baseEnv; + + nix::StaticEnv staticEnv(false, &state.staticBaseEnv); + + if (!vars) { + return staticEnv; + } + + auto displ = 0; + while (PyDict_Next(vars, &pos, &key, &val)) { + auto name = checkAttrKey(key); + if (!name) { + return {}; + } + + auto attrVal = pythonToNixValue(state, val); + if (!attrVal) { + return {}; + } + staticEnv.vars[state.symbols.create(name)] = displ; + (*env)->values[displ++] = attrVal; + } + + return staticEnv; +} +} // namespace pythonnix diff --git a/python/tests.py b/python/tests.py new file mode 100644 index 000000000000..9d909e98a73d --- /dev/null +++ b/python/tests.py @@ -0,0 +1,20 @@ +import nix +import unittest + + +class TestPythonNix(unittest.TestCase): + def test_dict(self): + val = dict(a=1) + self.assertEqual(nix.eval("a", vars=dict(a=val)), val) + + def test_string(self): + self.assertEqual(nix.eval("a", vars=dict(a="foo")), "foo") + + def test_bool(self): + self.assertEqual(nix.eval("a", vars=dict(a=True)), True) + + def test_none(self): + self.assertEqual(nix.eval("a", vars=dict(a=None)), None) + +if __name__ == '__main__': + unittest.main()