[WIP] poetry2nix based environment

.envrc

@@ -1 +1,2 @@
-use flake .?submodules=1
+export PROJECT_ROOT=$PWD
+use flake --impure

Makefile

@@ -459,7 +459,7 @@ ifeq ($(PYTEST),)
 	@echo "py.test is required to run the protocol tests, please install using 'pip3 install -r requirements.txt', and rerun 'configure'."; false
 else
 ifeq ($(DEVELOPER),1)
-	@(cd external/lnprototest && PYTHONPATH=$(MY_CHECK_PYTHONPATH) LIGHTNING_SRC=../.. $(PYTEST) --runner lnprototest.clightning.Runner $(PYTEST_OPTS))
+	@(cd external/lnprototest && LIGHTNING_SRC=../.. $(PYTEST) --runner lnprototest.clightning.Runner $(PYTEST_OPTS))
 else
 	@echo "lnprototest target requires DEVELOPER=1, skipping"
 endif
@@ -471,7 +471,7 @@ ifeq ($(PYTEST),)
 	exit 1
 else
 # Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
-	PYTHONPATH=$(MY_CHECK_PYTHONPATH) TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
+	TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 check-fuzz: $(ALL_FUZZ_TARGETS)
@@ -532,7 +532,7 @@ PYSRC=$(shell git ls-files "*.py" | grep -v /text.py)
 # allows it to find that
 PYLN_PATH=$(shell pwd)/lightningd:$(PATH)
 check-pyln-%: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
-	@(cd contrib/$(shell echo $@ | cut -b 7-) && PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(MAKE) check)
+	@(cd contrib/$(shell echo $@ | cut -b 7-) && PATH=$(PYLN_PATH) $(MAKE) check)
 
 check-python: check-python-flake8 check-pytest-pyln-proto check-pyln-client check-pyln-testing
 
@@ -544,7 +544,7 @@ check-python-flake8:
 	@flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
 
 check-pytest-pyln-proto:
-	PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/
+	PATH=$(PYLN_PATH) $(PYTEST) contrib/pyln-proto/tests/
 
 check-includes: check-src-includes check-hdr-includes
 	@tools/check-includes.sh

flake.nix

@@ -2,27 +2,100 @@
   description = "A very basic flake";
 
   inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/23.05";
     flake-utils.url = "github:numtide/flake-utils";
+    poetry2nix = {
+      url = "github:nix-community/poetry2nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = {
     self,
     nixpkgs,
     flake-utils,
+    poetry2nix,
   }:
     flake-utils.lib.eachDefaultSystem (system: let
       pkgs = nixpkgs.legacyPackages.${system};
       inherit (pkgs) lib;
+
+      inherit
+        (poetry2nix.legacyPackages.${system})
+        defaultPoetryOverrides
+        mkPoetryEnv
+        mkPoetryEditablePackage
+        ;
     in {
       packages = rec {
         default = cln;
-        cln = pkgs.clightning.overrideAttrs (upstream: rec {
+        cln = (pkgs.callPackage ("${nixpkgs}/pkgs/applications/blockchains/clightning") {
+          python3 = cln-meta-project; # FIXME neither this nor cln-meta-project.python actually works, but this is conceptually the right thing IIUC
+        }).overrideAttrs (upstream: rec {
           name = "cln";
           version = "${self.lastModifiedDate}-flake-${self.dirtyShortRev}"; # TODO emulate git describe using CHANGELOG.md or fetch git? impure runCommand?
-          makeFlags = [ "VERSION=${version}" "MTIME=${self.lastModifiedDate}" ];
+          makeFlags = ["VERSION=${version}" "MTIME=${self.lastModifiedDate}"];
+          nativeBuildInputs = [ cln-meta-project ] ++ upstream.nativeBuildInputs; # FIXME why does python3 in callPackage not work? work around by placing cln-meta-project's python first
           src = ./.;
+          # configureFlags = ["--enable-developer"]; # default flags disable developer and valgrind
+          doCheck = true;
+          # TODO remove PYTHONPATH
+          postPatch = upstream.postPatch +
+            ''
+            patchShebangs tools/check-*.sh
+          '';
         });
+
+        cln-meta-project = mkPoetryEnv {
+          projectDir = ./.;
+
+          # See https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md
+          overrides = defaultPoetryOverrides.extend (self: super:
+            lib.genAttrs [
+              "protobuf3"
+              "pytest-custom-exit-code"
+              "pytest-test-groups"
+            ] (name:
+              super.${name}.overridePythonAttrs (old: {
+                nativeBuildInputs = (old.nativeBuildInputs or []) ++ [self.setuptools];
+              }))
+          // {
+            python-bitcoinlib = super.python-bitcoinlib.overridePythonAttrs (old: {
+              postPatch = pkgs.python3Packages.bitcoinlib.postPatch; # fixes libssl loading
+            });
+            # TODO bitcoin.core.key needs libssl or libeay32
+          });
+
+          # FIXME why does contrib/pyln-grpc-proto have its own poetry.lock?
+          # TODO external/lnprototest/
+          editablePackageSources = let
+            # set PROJECT_ROOT environment variable and use --impure to obtain an
+            # editable project directory, e.g. `PROJECT_ROOT=$PWD nix develop --impure`
+            impureRoot = builtins.getEnv "PROJECT_ROOT";
+            root =
+              if impureRoot == ""
+              then ./.
+              else impureRoot;
+          in
+            lib.genAttrs [
+              "pyln-client"
+              "pyln-proto"
+              "pyln-grpc-proto"
+              "pyln-spec"
+              "pyln-testing"
+            ] (name: root + "/contrib/${name}");
+          preferWheels = builtins.getEnv "PREFER_WHEELS" == "1"; # requires --impure to take effect, quicker to build
+        };
       };
+
+      # devShells = {
+      #   default = pkgs.mkShell {
+      #     buildInputs = [
+      #       pkgs.poetry
+      #     ];
+      #   };
+      # };
+
       formatter = pkgs.alejandra;
     });
 }

pyproject.toml

@@ -19,6 +19,7 @@ cryptography = "^41.0.2"
 
 [tool.poetry.dev-dependencies]
 # Test dependencies and inherited dependencies belong here
+pytest = "^7"
 crc32c = "^2.2.post0"  # Belongs to lnprototest
 pytest-xdist = "^2.5.0"
 pytest-test-groups = "^1.0.3"