pyosys: rewrite using pybind11

- Rewrite all Python features to use the pybind11 library instead of boost::python.
  Unlike boost::python, pybind11 is a header-only library that is just included by Pyosys code, saving a lot of compile time on wheels.
- Factor out as much "translation" code from the generator into proper C++ files
- Fix running the embedded interpreter not supporting "from pyosys import libyosys as ys" like wheels
- Move Python-related elements to `pyosys` directory at the root of the repo
- Slight shift in bridging semantics:
  - Containers are declared as "opaque types" and are passed by reference to Python - many methods have been implemented to make them feel right at home without the overhead/ambiguity of copying to Python and then copying back after mutation
  - Monitor/Pass use "trampoline" pattern to support virual methods overridable in Python: virtual methods no longer require `py_` prefix
- Create really short test set for pyosys that just exercises basic functionality

Author: donn

.github/workflows/wheels.yml

@@ -55,11 +55,6 @@ jobs:
           submodules: true
           persist-credentials: false
       - uses: actions/setup-python@v5
-      - name: Get Boost Source
-        shell: bash
-        run: |
-          mkdir -p boost
-          curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost
       - name: Get FFI
         shell: bash
         run: |
@@ -103,21 +98,16 @@ jobs:
           CIBW_BEFORE_ALL: bash ./.github/workflows/wheels/cibw_before_all.sh
           CIBW_ENVIRONMENT: >
             OPTFLAGS=-O3
-            CXXFLAGS=-I./boost/pfx/include
-            LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
-            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
             PATH="$PWD/bison/src:$PATH"
           CIBW_ENVIRONMENT_MACOS: >
             OPTFLAGS=-O3
-            CXXFLAGS=-I./boost/pfx/include
-            LINKFLAGS=-L./boost/pfx/lib
             PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
             MACOSX_DEPLOYMENT_TARGET=11
-            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+            makeFlags='CONFIG=clang'
             PATH="$PWD/bison/src:$PATH"
           CIBW_BEFORE_BUILD: bash ./.github/workflows/wheels/cibw_before_build.sh
-          CIBW_TEST_COMMAND: python3 {project}/tests/arch/ecp5/add_sub.py
+          CIBW_TEST_COMMAND: python3 {project}/tests/pyosys/run_tests.py python3
       - uses: actions/upload-artifact@v4
         with:
           name: python-wheels-${{ matrix.os.runner }}

.github/workflows/wheels/cibw_before_build.sh

@@ -11,24 +11,3 @@ if [ "$(uname)" != "Linux" ]; then
 fi
 python3 --version
 python3-config --includes
-
-# Build boost
-cd ./boost
-## Delete the artefacts from previous builds (if any)
-rm -rf ./pfx
-## Bootstrap bjam
-./bootstrap.sh --prefix=./pfx
-## Build Boost against current version of Python, only for
-## static linkage (Boost is statically linked because system boost packages
-## wildly vary in versions, including the libboost_python3 version)
-./b2\
-    -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\
-    --prefix=./pfx\
-    --with-filesystem\
-    --with-system\
-    --with-python\
-    cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\
-    cflags="$(python3-config --includes) -fPIC"\
-    link=static\
-    variant=release\
-    install

CODEOWNERS

@@ -38,7 +38,8 @@ techlibs/gowin/                @pepijndevos
 techlibs/gatemate/             @pu-cc
 
 # pyosys
-misc/*.py                      @btut
+pyosys/*                       @donn
+setup.py                       @donn
 
 backends/firrtl                @ucbjrl @azidar
 

Makefile

@@ -128,10 +128,6 @@ LINKFLAGS += -rdynamic
 ifneq ($(shell :; command -v brew),)
 BREW_PREFIX := $(shell brew --prefix)/opt
 $(info $$BREW_PREFIX is [${BREW_PREFIX}])
-ifeq ($(ENABLE_PYOSYS),1)
-CXXFLAGS += -I$(BREW_PREFIX)/boost/include
-LINKFLAGS += -L$(BREW_PREFIX)/boost/lib -L$(BREW_PREFIX)/boost-python3/lib
-endif
 CXXFLAGS += -I$(BREW_PREFIX)/readline/include -I$(BREW_PREFIX)/flex/include
 LINKFLAGS += -L$(BREW_PREFIX)/readline/lib -L$(BREW_PREFIX)/flex/lib
 PKG_CONFIG_PATH := $(BREW_PREFIX)/libffi/lib/pkgconfig:$(PKG_CONFIG_PATH)
@@ -347,26 +343,14 @@ ifeq ($(ENABLE_PYOSYS),1)
 LINKFLAGS += $(filter-out -l%,$(shell $(PYTHON_CONFIG) --ldflags))
 LIBS += $(shell $(PYTHON_CONFIG) --libs)
 EXE_LIBS += $(filter-out $(LIBS),$(shell $(PYTHON_CONFIG_FOR_EXE) --libs))
+PYBIND11_INCLUDE ?= $(shell $(PYTHON_EXECUTABLE) -m pybind11 --includes)
+CXXFLAGS += -I$(PYBIND11_INCLUDE) -DWITH_PYTHON
 CXXFLAGS += $(shell $(PYTHON_CONFIG) --includes) -DWITH_PYTHON
 
-# Detect name of boost_python library. Some distros use boost_python-py<version>, other boost_python<version>, some only use the major version number, some a concatenation of major and minor version numbers
-CHECK_BOOST_PYTHON = (echo "int main(int argc, char ** argv) {return 0;}" | $(CXX) -xc -o /dev/null $(LINKFLAGS) $(EXE_LIBS) $(LIBS) -l$(1) - > /dev/null 2>&1 && echo "-l$(1)")
-BOOST_PYTHON_LIB ?= $(shell \
-	$(call CHECK_BOOST_PYTHON,boost_python-py$(subst .,,$(PYTHON_VERSION))) || \
-	$(call CHECK_BOOST_PYTHON,boost_python-py$(PYTHON_MAJOR_VERSION)) || \
-	$(call CHECK_BOOST_PYTHON,boost_python$(subst .,,$(PYTHON_VERSION))) || \
-	$(call CHECK_BOOST_PYTHON,boost_python$(PYTHON_MAJOR_VERSION)) \
-)
-
-ifeq ($(BOOST_PYTHON_LIB),)
-$(error BOOST_PYTHON_LIB could not be detected. Please define manually)
-endif
-
-LIBS += $(BOOST_PYTHON_LIB) -lboost_system -lboost_filesystem
-PY_WRAPPER_FILE = kernel/python_wrappers
+PY_WRAPPER_FILE = pyosys/wrappers
 OBJS += $(PY_WRAPPER_FILE).o
-PY_GEN_SCRIPT= py_wrap_generator
-PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).print_includes()")
+PY_GEN_SCRIPT = pyosys/generator.py
+PY_WRAP_INCLUDES := $(shell $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) --print-includes)
 endif # ENABLE_PYOSYS
 
 ifeq ($(ENABLE_READLINE),1)
@@ -759,9 +743,9 @@ endif
 	$(P) cat $< | grep -E -v "#[ ]*(include|error)" | $(CXX) $(CXXFLAGS) -x c++ -o $@ -E -P -
 
 ifeq ($(ENABLE_PYOSYS),1)
-$(PY_WRAPPER_FILE).cc: misc/$(PY_GEN_SCRIPT).py $(PY_WRAP_INCLUDES)
+$(PY_WRAPPER_FILE).cc: $(PY_GEN_SCRIPT) pyosys/wrappers_tpl.cc $(PY_WRAP_INCLUDES) pyosys/hashlib.h
 	$(Q) mkdir -p $(dir $@)
-	$(P) $(PYTHON_EXECUTABLE) -c "from misc import $(PY_GEN_SCRIPT); $(PY_GEN_SCRIPT).gen_wrappers(\"$(PY_WRAPPER_FILE).cc\")"
+	$(P) $(PYTHON_EXECUTABLE) $(PY_GEN_SCRIPT) $(PY_WRAPPER_FILE).cc
 endif
 
 %.o: %.cpp

docs/source/yosys_internals/hashing.rst

@@ -36,7 +36,7 @@ The main characteristics are:
    all compilers, standard libraries and architectures.
 
 In addition to ``dict<K, T>`` and ``pool<T>`` there is also an ``idict<K>`` that
-creates a bijective map from ``K`` to the integers. For example:
+creates a bijective map from ``K`` to incrementing integers. For example:
 
 ::
 

examples/python-api/pass.py

@@ -1,22 +1,25 @@
 #!/usr/bin/python3
 
-import libyosys as ys
+from pyosys import libyosys as ys
+
+from pathlib import Path
 
 import matplotlib.pyplot as plt
-import numpy as np
+
+__file_dir__ = Path(__file__).absolute().parent
 
 class CellStatsPass(ys.Pass):
 
     def __init__(self):
         super().__init__("cell_stats", "Shows cell stats as plot")
 
-    def py_help(self):
+    def help(self):
         ys.log("This pass uses the matplotlib library to display cell stats\n")
 
-    def py_execute(self, args, design):
+    def execute(self, args, design):
         ys.log_header(design, "Plotting cell stats\n")
         cell_stats = {}
-        for module in design.selected_whole_modules_warn():
+        for module in design.all_selected_whole_modules():
             for cell in module.selected_cells():
                 if cell.type.str() in cell_stats:
                     cell_stats[cell.type.str()] += 1
@@ -29,4 +32,11 @@ def py_execute(self, args, design):
     def py_clear_flags(self):
         ys.log("Clear Flags - CellStatsPass\n")
 
-p = CellStatsPass()
+p = CellStatsPass() # register
+
+if __name__ == "__main__":
+    design = ys.Design()
+    ys.run_pass(f"read_verilog {__file_dir__.parents[1] / 'tests' / 'simple' / 'fiedler-cooley.v'}", design)
+    ys.run_pass("prep", design)
+    ys.run_pass("opt -full", design)
+    ys.run_pass("cell_stats", design)

kernel/driver.cc

@@ -92,8 +92,9 @@ int main(int argc, char **argv)
 	yosys_banner();
 	yosys_setup();
 #ifdef WITH_PYTHON
-	PyRun_SimpleString(("sys.path.append(\""+proc_self_dirname()+"\")").c_str());
-	PyRun_SimpleString(("sys.path.append(\""+proc_share_dirname()+"plugins\")").c_str());
+	py::object sys = py::module_::import("sys");
+	sys.attr("path").attr("append")(proc_self_dirname());
+	sys.attr("path").attr("append")(proc_share_dirname());
 #endif
 
 	if (argc == 2)
@@ -516,8 +517,9 @@ int main(int argc, char **argv)
 
 	yosys_setup();
 #ifdef WITH_PYTHON
-	PyRun_SimpleString(("sys.path.append(\""+proc_self_dirname()+"\")").c_str());
-	PyRun_SimpleString(("sys.path.append(\""+proc_share_dirname()+"plugins\")").c_str());
+	py::object sys = py::module_::import("sys");
+	sys.attr("path").attr("append")(proc_self_dirname());
+	sys.attr("path").attr("append")(proc_share_dirname());
 #endif
 	log_error_atexit = yosys_atexit;
 
@@ -567,21 +569,18 @@ int main(int argc, char **argv)
 #endif
 		} else if (scriptfile_python) {
 #ifdef WITH_PYTHON
-			PyObject *sys = PyImport_ImportModule("sys");
+			py::list new_argv;
 			int py_argc = special_args.size() + 1;
-			PyObject *new_argv = PyList_New(py_argc);
-			PyList_SetItem(new_argv, 0, PyUnicode_FromString(scriptfile.c_str()));
+			new_argv.append(scriptfile);
 			for (int i = 1; i < py_argc; ++i)
-				PyList_SetItem(new_argv, i, PyUnicode_FromString(special_args[i - 1].c_str()));
+				new_argv.append(special_args[i - 1]);
 
-			PyObject *old_argv = PyObject_GetAttrString(sys, "argv");
-			PyObject_SetAttrString(sys, "argv", new_argv);
-			Py_DECREF(old_argv);
+			py::setattr(sys, "argv", new_argv);
 
-			PyObject *py_path = PyUnicode_FromString(scriptfile.c_str());
-			PyObject_SetAttrString(sys, "_yosys_script_path", py_path);
-			Py_DECREF(py_path);
-			PyRun_SimpleString("import os, sys; sys.path.insert(0, os.path.dirname(os.path.abspath(sys._yosys_script_path)))");
+			py::object Path = py::module_::import("pathlib").attr("Path");
+			py::object scriptfile_python_path = Path(scriptfile).attr("parent");
+
+			sys.attr("path").attr("insert")(0, py::str(scriptfile_python_path));
 
 			FILE *scriptfp = fopen(scriptfile.c_str(), "r");
 			if (scriptfp == nullptr) {

kernel/yosys.cc

@@ -64,13 +64,8 @@
 #endif
 
 #ifdef WITH_PYTHON
-#if PY_MAJOR_VERSION >= 3
-#   define INIT_MODULE PyInit_libyosys
-    extern "C" PyObject* INIT_MODULE();
-#else
-#   define INIT_MODULE initlibyosys
-	extern "C" void INIT_MODULE();
-#endif
+extern "C" PyObject* PyInit_libyosys();
+extern "C" PyObject* PyInit_pyosys();
 #include <signal.h>
 #endif
 
@@ -189,6 +184,17 @@ int run_command(const std::string &command, std::function<void(const std::string
 bool already_setup = false;
 bool already_shutdown = false;
 
+#ifdef WITH_PYTHON
+// Include pyosys as a module so 'from pyosys import libyosys' also works
+// in interpreter mode.
+//
+// This should not affect using wheels as the dylib has to actually be called
+// pyosys.so for this module to be interacted with at all.
+PYBIND11_MODULE(pyosys, m) {
+	m.add_object("libyosys", m.import("libyosys"));
+}
+#endif
+
 void yosys_setup()
 {
 	if(already_setup)
@@ -197,11 +203,12 @@ void yosys_setup()
 	already_shutdown = false;
 
 #ifdef WITH_PYTHON
-	// With Python 3.12, calling PyImport_AppendInittab on an already
+	// Starting Python 3.12, calling PyImport_AppendInittab on an already
 	// initialized platform fails (such as when libyosys is imported
 	// from a Python interpreter)
 	if (!Py_IsInitialized()) {
-		PyImport_AppendInittab((char*)"libyosys", INIT_MODULE);
+		PyImport_AppendInittab((char*)"libyosys", PyInit_libyosys);
+		PyImport_AppendInittab((char*)"pyosys", PyInit_pyosys);
 		Py_Initialize();
 		PyRun_SimpleString("import sys");
 		signal(SIGINT, SIG_DFL);

kernel/yosys_common.h

@@ -55,6 +55,9 @@
 
 #ifdef WITH_PYTHON
 #include <Python.h>
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
 #endif
 
 #ifndef _YOSYS_