python: Integrate incremental and CI build

Makes the python bindings build, both incrementally and for CI.
Documentation is not yet included

flake.nix

@@ -410,6 +410,11 @@
               postUnpack = "sourceRoot=$sourceRoot/perl";
             });
 
+            passthru.python-bindings = final.callPackage ./python {
+              inherit self system;
+              python = final.python3;
+            };
+
             meta.platforms = lib.platforms.unix;
           };
 
@@ -465,6 +470,8 @@
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
+        pythonBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.python-bindings);
+
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
@@ -585,6 +592,7 @@
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
+        pythonBindings = self.hydraJobs.pythonBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
@@ -651,6 +659,7 @@
                 # Make bash completion work.
                 XDG_DATA_DIRS+=:$out/share
               '';
+<<<<<<< HEAD
           };
         in
         forAllSystems (system:
@@ -667,5 +676,21 @@
               default = self.devShells.${system}.native-stdenvPackages;
             }
         );
+||||||| parent of 35191cfa1 (python: Integrate incremental and CI build)
+          }
+        )
+        // { default = self.devShells.${system}.stdenv; }
+      );
+
+=======
+          }
+        )
+        // {
+          default = self.devShells.${system}.stdenv;
+          python = self.packages.${system}.nix.python-bindings.shell;
+        }
+      );
+
+>>>>>>> 35191cfa1 (python: Integrate incremental and CI build)
   };
 }

python/.gitignore

@@ -0,0 +1,2 @@
+.cache
+build

python/Makefile

@@ -0,0 +1,49 @@
+# This Makefile is only used for development of the Python bindings, it is not
+# used in the Nix build.
+# The reason this exists is to make it easier to develop the Python bindings in
+# tandem with the main Nix.
+# The default `make` (defaults to `make build`) calls the main Nix projects
+# `make install` before calling the Python bindings' `meson compile`, therefore
+# ensuring that the needed Nix dynamic libraries are up-to-date
+
+builddir=build
+
+.PHONY: build
+build: nix-install setup-done
+	meson compile -C $(builddir)
+
+.PHONY: test
+test: nix-install setup-done
+	meson test -C $(builddir) -v
+
+.PHONY: clean
+clean:
+	rm -rf $(builddir)
+
+# We include the main Nix projects Makefile.config file to know the $(libdir)
+# variable, which is where Nix is installed in, which we can then use to setup
+# the meson build
+include ../Makefile.config
+
+# We need the file to exist though
+../Makefile.config:
+	@# Throw a good error message in case ./configure hasn't been run yet
+	@[[ -e ../config.status ]] || ( echo "The main Nix project needs to be configured first, see https://nixos.org/manual/nix/stable/contributing/hacking.html" && exit 1 )
+	@# If ./configure is done, we can create the file ourselves
+	$(MAKE) -C .. Makefile.config
+
+.PHONY: setup
+setup: nix-install
+	@# Make meson be able to find the locally-installed Nix
+	PKG_CONFIG_PATH=$(libdir)/pkgconfig:$$PKG_CONFIG_PATH meson setup $(builddir)
+
+.PHONY: setup-done
+setup-done:
+	@# A better error message in case the build directory doesn't exist yet
+	@[[ -e $(builddir) ]] || ( echo "Run 'make setup' once to configure the project build directory" && exit 1 )
+
+.PHONY: nix-install
+nix-install:
+	@# The python bindings don't technically need an _entire_ Nix installation,
+	@# but it seems non-trivial to pick out only exactly the files it actually needs
+	$(MAKE) -C .. install

python/default.nix

@@ -0,0 +1,45 @@
+{ self, system, lib, python, ninja, meson, nix, mkShell }:
+python.pkgs.buildPythonPackage {
+  name = "nix";
+  format = "other";
+
+  src = self;
+
+  strictDeps = true;
+
+  nativeBuildInputs = lib.optionals (nix != null) nix.nativeBuildInputs ++ [
+    ninja
+    (meson.override { python3 = python; })
+    nix
+  ];
+
+  buildInputs = lib.optionals (nix != null) nix.buildInputs ++ [
+    nix
+  ];
+
+  # We need to be able to generate tests/common.sh, which requires running
+  # `make`, which requires having run `autoreconf` and `./configure`.
+  # So we propagate `autoreconfHook` from nix.nativeBuildInputs for that to
+  # work, but after that we also need to cd into the python directory and run the
+  # meson configure phase for the python bindings.
+  # Can't use `postConfigure` for this because that would create a loop since
+  # `mesonConfigurePhase` calls `postConfigure` itself.
+  # A small problem with this is that `{pre,post}Configure` get run twice
+  dontUseMesonConfigure = true;
+  preBuild = ''
+    cd python
+    mesonConfigurePhase
+  '';
+
+  mesonBuildType = "release";
+
+  doInstallCheck = true;
+  installCheckPhase = "meson test -v";
+
+  passthru.shell = mkShell {
+    inputsFrom = [
+      self.devShells.${system}.stdenv
+      (nix.python-bindings.override { nix = null; })
+    ];
+  };
+}

python/meson.build

@@ -6,8 +6,16 @@ project('python-nix', 'cpp',
 python_mod = import('python3')
 python_dep = dependency('python3', required : true)
 nix_expr_dep = dependency('nix-expr', required: true)
-
-python = python_mod.find_python()
-test('python test', python, args : files('tests.py'))
+nix_main_dep = dependency('nix-main', required: true)
 
 subdir('src')
+
+fs = import('fs')
+
+nix_root = fs.parent(meson.project_source_root())
+run_command('make', '-C', nix_root, 'tests/common.sh', check: true)
+
+env = environment()
+env.prepend('PYTHONPATH', fs.parent(pythonix.full_path()))
+bash = find_program('bash')
+test('python test', bash, args : files('test.sh'), env : env)

python/src/internal/nix-to-python.hh

@@ -2,7 +2,7 @@
 
 #include <Python.h>
 
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 

python/src/internal/python-to-nix.hh

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Python.h>
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 #include <optional>

python/src/meson.build

@@ -5,8 +5,14 @@ src = [
   'python-module.cc',
 ]
 
-python_mod.extension_module('nix', src,
-    dependencies : [python_dep, nix_expr_dep],
+pythonix = python_mod.extension_module('nix', src,
+    dependencies : [python_dep, nix_expr_dep, nix_main_dep],
     install: true,
     install_dir: python_mod.sysconfig_path('platlib'),
-    cpp_args: ['-std=c++17', '-fvisibility=hidden'])
+    cpp_args: [
+      '-std=c++17',
+      # -Wnon-virtual-dtor is unnecessarily turned on by Meson
+      # This is fixed in Meson 1.0.0 with https://github.com/mesonbuild/meson/pull/10339
+      '-Wno-non-virtual-dtor',
+      '-fvisibility=hidden'
+    ])

python/src/python-module.cc

@@ -3,7 +3,7 @@
 #include "internal/eval.hh"
 #include "internal/ptr.hh"
 
-#include <nix/config.h>
+#include <config.h>
 
 #include <eval.hh>
 #include <globals.hh>

python/test.sh

@@ -0,0 +1,7 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+cd "$SCRIPT_DIR"/../tests
+
+source init.sh
+
+python "$SCRIPT_DIR"/tests.py