Add pre- and post-build script to work around Poetry bug (#868)

The new packaging strategy introduced in #849 doesn't work correctly for
wheels due to a bug in poetry
(python-poetry/poetry#8994), which breaks
installing both from Github and PyPi. This PR introduces a workaround.

Since the bug is related to the `src` directory, this workaround adds a
`pre-build/sh` script which unpacks the `src` directory before building,
and a `post-build.sh` script which moves the package back into the `src`
directory afterwards.

This leads to the following result:
- Installing from Github leads to an installation including all
component files
- Installing locally without running the build script leads to an
installation including all component files
- Installing from PyPi leads to an installation with only the
`fondant_component.yaml` files for each component

In the ideal case, we get the same result for all installation methods,
but this already at least leads to a working result for each method.

.github/workflows/prep-release.yaml

@@ -71,6 +71,7 @@ jobs:
  - name: Build a binary wheel and a source tarball
  run: |
  pip install poetry
+ ./scripts/pre-build.sh
  poetry build
 
  - name: Publish distribution 📦 to Test PyPI

pyproject.toml

@@ -21,8 +21,8 @@ maintainers = [
  "Niels Rogge <niels.rogge@ml6.eu>",
 ]
 repository = "https://github.com/ml6team/fondant"
-include = ["*.txt", "*.rst", "*.md", "src/fondant/components/**/*.yaml"]
-exclude = ["src/fondant/components"] # Exclude everything except for component specs
+include = ["*.txt", "*.rst", "*.md", "fondant/components/**/*.yaml"]
+exclude = ["fondant/components"] # Exclude everything except for component specs
 classifiers = [
  "Development Status :: 2 - Pre-Alpha",
  "Intended Audience :: Developers",
@@ -75,6 +75,9 @@ vertex = ["docker", "kfp", "google-cloud-aiplatform"]
 sagemaker = ["sagemaker", "boto3"]
 docker = ["docker"]
 
+all = ["dask", "dask-cuda", "s3fs", "adlfs", "gcsfs", "docker", "kfp", "google-cloud-aiplatform",
+ "sagemaker", "boto3"]
+
 [tool.poetry.group.test]
 optional = true
 

scripts/post-build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This script reverts the `scripts/pre-build.sh` script by moving the package code back into a
+# `src` directory.
+# It should be run after running scripts/pre-build.sh and building the fondant package
+set -e
+
+scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+root_path=$(dirname "$scripts_path")
+
+pushd "$root_path"
+mkdir -p src/fondant
+mv fondant/* src/fondant
+rm -rf fondant
+popd

scripts/pre-build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# This script unpacks the src folder before building to make the poetry `include` and `exclude`
+# sections work correctly for both sdist and wheel
+# (https://github.com/python-poetry/poetry/issues/8994)
+# Building without running this script also works, but includes the full code for the components.
+# This script makes changes to the local files, which should not be committed to git. Run
+# scripts/post-build.sh to clean them up.
+set -e
+
+scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+root_path=$(dirname "$scripts_path")
+
+pushd "$root_path"
+mkdir fondant
+mv src/fondant/* fondant
+rm -rf src
+popd

tox.ini

@@ -42,8 +42,10 @@ allowlist_externals=
  /bin/bash
  /usr/bin/bash
 commands_pre=
- poetry lock
- poetry install --all-extras
- poetry show
+ bash ./scripts/pre-build.sh
+ poetry build
+ poetry run pip install dist/fondant-0.1.dev0-py3-none-any.whl[all]
+ poetry run pip list
+ bash ./scripts/post-build.sh
 commands=
  poetry run python -m pytest tests -vv --cov fondant --cov-report term-missing --ignore=tests/integration_tests