From 16719c37ae139608db7982615a014f0ebaa7f960 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 13 Aug 2024 12:39:21 +0200 Subject: [PATCH 01/21] Adding typing stubs to python package using pybind11-stubgen. --- cpp/pybind/make_python_package.cmake | 5 +++++ python/requirements_build.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 01e0d5663f5..0c0a3ede2e3 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -136,3 +136,8 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") + +# Generate typing stub files (.pyi) +message(STATUS "Generating typing stubs...") +message(STATUS "pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR}/stubs") +execute_process(COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}") \ No newline at end of file diff --git a/python/requirements_build.txt b/python/requirements_build.txt index b9bb3b1f28a..9d8dc63cb4b 100644 --- a/python/requirements_build.txt +++ b/python/requirements_build.txt @@ -1,3 +1,4 @@ setuptools>=67.3.2 wheel==0.38.4 yapf==0.30.0 +pybind11-stubgen==2.5.1 \ No newline at end of file From 4739963309af974a9163468f26ddb30e84841aab Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Wed, 14 Aug 2024 23:12:33 +0200 Subject: [PATCH 02/21] Added py.typed marker file to python package. --- cpp/pybind/make_python_package.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 0c0a3ede2e3..8ab69f78e2f 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -137,7 +137,8 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") -# Generate typing stub files (.pyi) +# Generate typing stub files (.pyi) and py.typed marker file. message(STATUS "Generating typing stubs...") message(STATUS "pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR}/stubs") -execute_process(COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}") \ No newline at end of file +execute_process(COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}") +file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") From 6c26d61775d21a08c2cdb85d8da687e6cb2170c5 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 11:54:02 +0200 Subject: [PATCH 03/21] Added IGNORE_STUBGEN_ERRORS cmake option. --- CMakeLists.txt | 1 + cpp/pybind/CMakeLists.txt | 1 + cpp/pybind/make_python_package.cmake | 8 +++++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1bd4706288..1908c41f9ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ option(BUILD_EXAMPLES "Build Open3D examples programs" ON option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) option(BUILD_PYTHON_MODULE "Build the python module" ON ) +option(IGNORE_STUBGEN_ERRORS "Ignore all errors during stub generation" ON ) option(BUILD_CUDA_MODULE "Build the CUDA module" OFF) option(BUILD_WITH_CUDA_STATIC "Build with static CUDA libraries" ON ) option(BUILD_COMMON_CUDA_ARCHS "Build for common CUDA GPUs (for release)" OFF) diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 6efae9a17fd..b0be33798f9 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -226,6 +226,7 @@ add_custom_target(python-package -DPYTHON_VERSION=${PYTHON_VERSION} "-DCOMPILED_MODULE_PATH_LIST=${COMPILED_MODULE_PATH_LIST}" "-DPYTHON_EXTRA_LIBRARIES=${PYTHON_EXTRA_LIBRARIES}" + -DIGNORE_STUBGEN_ERRORS=${IGNORE_STUBGEN_ERRORS} -DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION} -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} -DBUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 8ab69f78e2f..e702f75ce9a 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -138,7 +138,13 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") # Generate typing stub files (.pyi) and py.typed marker file. +if(NOT IGNORE_STUBGEN_ERRORS) + list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") +endif() message(STATUS "Generating typing stubs...") message(STATUS "pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR}/stubs") -execute_process(COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}") +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}" ${PYBIND11_STUBGEN_FLAGS} + COMMAND_ERROR_IS_FATAL ANY +) file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") From ee90f58d64c1dd895faa5fed33e3c507cf2a50b5 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 11:55:46 +0200 Subject: [PATCH 04/21] Added changelog entry. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1f4f43b17..8c71e011c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - Fix render to depth image on Apple Retina displays (PR #7001) - Fix infinite loop in segment_plane if num_points < ransac_n (PR #7032) - Add select_by_index method to Feature class (PR #7039) +- Include typing stubs in python package (PR #6917) ## 0.13 From dbfe46ac8b996c8a7252cd2d51284f3ec90ed61a Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 12:20:11 +0200 Subject: [PATCH 05/21] Removed message intended for debugging. --- cpp/pybind/make_python_package.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index e702f75ce9a..8e0b4601860 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -142,7 +142,6 @@ if(NOT IGNORE_STUBGEN_ERRORS) list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") endif() message(STATUS "Generating typing stubs...") -message(STATUS "pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR}/stubs") execute_process( COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}" ${PYBIND11_STUBGEN_FLAGS} COMMAND_ERROR_IS_FATAL ANY From 138f3dde04403ab0383e8a3ff2c97eaa05baede3 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 13:15:26 +0200 Subject: [PATCH 06/21] Added COMMAND_ECHO to execute_process for CI debugging. --- cpp/pybind/make_python_package.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 8e0b4601860..a5f7e8a5ed6 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -144,6 +144,7 @@ endif() message(STATUS "Generating typing stubs...") execute_process( COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}" ${PYBIND11_STUBGEN_FLAGS} + COMMAND_ECHO STDOUT COMMAND_ERROR_IS_FATAL ANY ) file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") From 2620894424ce3a0005912afc603e421f9267ee25 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 14:37:36 +0200 Subject: [PATCH 07/21] Changed CI to use requirements_build.txt. --- .github/workflows/windows.yml | 20 +++++++------------- util/ci_utils.sh | 22 ++++++++-------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 04a408802d0..67852eab2b0 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,8 +21,6 @@ concurrency: env: PIP_VER: "23.2.1" - WHEEL_VER: "0.38.4" - STOOLS_VER: "67.3.2" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 CUDA_VERSION: "12.1.0" @@ -217,11 +215,9 @@ jobs: working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} ` - jedi==${{ env.JEDI_VER }} ` - idna==${{ env.IDNA_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt + python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - name: Import python package # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. @@ -300,9 +296,8 @@ jobs: working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append @@ -366,9 +361,8 @@ jobs: python -m venv open3d_test_venv open3d_test_venv\Scripts\Activate.ps1 - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 1adb4b7d8c5..8c8f4217361 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -30,9 +30,6 @@ TORCH_VER="2.2.2" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python PIP_VER="23.2.1" -WHEEL_VER="0.38.4" -STOOLS_VER="67.3.2" -YAPF_VER="0.30.0" PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install @@ -42,10 +39,10 @@ install_python_dependencies() { echo "Installing Python dependencies" options="$(echo "$@" | tr ' ' '|')" - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install --upgrade pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" fi if [[ "with-cuda" =~ ^($options)$ ]]; then TF_ARCH_NAME=tensorflow @@ -64,7 +61,6 @@ install_python_dependencies() { TORCH_GLNX="torch==${TORCH_VER}+cpu" fi - # TODO: modify other locations to use requirements.txt python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" if [[ "with-jupyter" =~ ^($options)$ ]]; then python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" @@ -88,7 +84,6 @@ install_python_dependencies() { fi fi if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then - python -m pip install -U yapf=="$YAPF_VER" # Fix Protobuf compatibility issue # https://stackoverflow.com/a/72493690/1255535 # https://github.com/protocolbuffers/protobuf/issues/10051 @@ -241,8 +236,8 @@ test_wheel() { python -m venv open3d_test.venv # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install -U pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" echo -n "Using python: $(command -v python)" python --version echo -n "Using pip: " @@ -285,7 +280,7 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then @@ -356,9 +351,8 @@ install_docs_dependencies() { echo Install Python dependencies for building docs command -v python python -V - python -m pip install -U -q "wheel==$WHEEL_VER" \ - "pip==$PIP_VER" - python -m pip install -U -q "yapf==$YAPF_VER" + python -m pip install -U -q "pip==$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1" echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}" From c10dee27506ff73e0c9074cc95c1250cf6734051 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 14:56:28 +0200 Subject: [PATCH 08/21] Updated pyproject.toml build-system requirements. --- python/pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 3d229011876..f65c9888bcb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,12 @@ [build-system] -requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"] +requires = [ + "ipywidgets>=8.0.4", + "pygments>=2.7.4", + "jupyter_packaging~=0.12", + "jupyterlab>=3.0.0,==3.*", + "setuptools>=67.3.2", + "wheel==0.38.4", + "yapf==0.30.0", + "pybind11-stubgen==2.5.1", +] build-backend = "setuptools.build_meta" From babbff0ba6e8f6248d0ae9d4d6460a3c34ead7c0 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 14:57:02 +0200 Subject: [PATCH 09/21] Changed style check workflow to use requirements_style.txt. --- .github/workflows/style.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 043cb0c7b97..9871c13eed3 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -27,7 +27,7 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat + pip install -U -r python/requirements_style.txt - name: Run style check run: | python util/check_style.py From 2674d5b3ba1149d2c22df2ebeeae8bfb9ec46290 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 15:29:36 +0200 Subject: [PATCH 10/21] Added missing copy of requirements_build.txt to docker. --- docker/Dockerfile.wheel | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index eb60286b620..e9f688f4ab9 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -109,6 +109,7 @@ RUN /root/Open3D/util/install_deps_ubuntu.sh assume-yes \ # Open3D Python dependencies COPY ./util/ci_utils.sh /root/Open3D/util/ COPY ./python/requirements.txt /root/Open3D/python/ +COPY ./python/requirements_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ From cabef3c125c804605444b2b6153bb08235504a0e Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 15:50:53 +0200 Subject: [PATCH 11/21] Removed "" from execute_process args to fix build error. --- cpp/pybind/make_python_package.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index a5f7e8a5ed6..85018619396 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -143,7 +143,7 @@ if(NOT IGNORE_STUBGEN_ERRORS) endif() message(STATUS "Generating typing stubs...") execute_process( - COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH="${PYTHON_PACKAGE_DST_DIR}" pybind11-stubgen open3d -o "${PYTHON_PACKAGE_DST_DIR}" ${PYBIND11_STUBGEN_FLAGS} + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS} COMMAND_ECHO STDOUT COMMAND_ERROR_IS_FATAL ANY ) From 1abe058e7161d304dc6a780bff16557741b7975a Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 15 Aug 2024 23:58:40 +0200 Subject: [PATCH 12/21] Install open3d-ml dependencies for docker wheel build for stubgen. --- docker/Dockerfile.wheel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index e9f688f4ab9..edf30a1015c 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -115,6 +115,9 @@ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ && install_python_dependencies with-jupyter +# Open3D-ML Python dependencies +RUN python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt" + # Open3D Jupyter dependencies RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && apt-get install -y nodejs \ From 026aa0fab37d626b91e6ccbfb7f2545ce328765b Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 01:20:32 +0200 Subject: [PATCH 13/21] Fixed wrong working dir in windows workflow. --- .github/workflows/windows.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 67852eab2b0..a45d8b90eec 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -211,13 +211,17 @@ jobs: cmake --build . --config ${{ matrix.CONFIG }} .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test Remove-Item "C:\Program Files\Open3D" -Recurse - - name: Install Python package - working-directory: ${{ env.BUILD_DIR }} + - name: Install Open3D python requirements + working-directory: ${{ env.SOURCE_DIR }} run: | $ErrorActionPreference = 'Stop' python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -U -r python/requirements_build.txt python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} + - name: Install Python package + working-directory: ${{ env.BUILD_DIR }} + run: | + $ErrorActionPreference = 'Stop' cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - name: Import python package # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. @@ -268,7 +272,9 @@ jobs: working-directory: ${{ env.SRC_DIR }} run: | $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -r python/requirements.txt + python -m pip install -r python/requirements_build.txt python -m pip install -r python/requirements_jupyter_build.txt - name: Config @@ -296,8 +302,6 @@ jobs: working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install -U pip==${{ env.PIP_VER }} - python -m pip install -U -r python/requirements_build.txt cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append From acae49a8ee551069b4447c9d49a23956771e2684 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 02:13:40 +0200 Subject: [PATCH 14/21] Added install step in MacOS Build wheel for o3d-ml requirements. --- .github/workflows/macos.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a184df2ec42..fafb45431f4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -269,6 +269,11 @@ jobs: brew install ccache ccache -M 2G # See .github/workflows/readme.md for ccache strategy. + - name: Install Open3D-ML Python dependencies + working-directory: ${{ env.OPEN3D_ML_ROOT }} + run: | + python -m pip install -r requirements.txt + - name: Config and build wheel run: | PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH From 89c482b25059122f059d57555313bbbbff36d2c7 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 02:38:51 +0200 Subject: [PATCH 15/21] Added missing install of requirements.txt in windows workflow. --- .github/workflows/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a45d8b90eec..44e10428903 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -217,6 +217,7 @@ jobs: $ErrorActionPreference = 'Stop' python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -U -r python/requirements_build.txt + python -m pip install -U -r python/requirements.txt python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} - name: Install Python package working-directory: ${{ env.BUILD_DIR }} From 8ed61cf1067b5d001356821a200659228549f0b7 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 15:34:11 +0200 Subject: [PATCH 16/21] Added WITH_STUBGEN cmake flag to toogle typing stub generation. --- CMakeLists.txt | 1 + cpp/pybind/CMakeLists.txt | 1 + cpp/pybind/make_python_package.cmake | 20 +++++++++++--------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1908c41f9ff..2433ecee515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ option(BUILD_EXAMPLES "Build Open3D examples programs" ON option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) option(BUILD_PYTHON_MODULE "Build the python module" ON ) +option(WITH_STUBGEN "Use pybind11-stubgen to generate stubs" ON ) option(IGNORE_STUBGEN_ERRORS "Ignore all errors during stub generation" ON ) option(BUILD_CUDA_MODULE "Build the CUDA module" OFF) option(BUILD_WITH_CUDA_STATIC "Build with static CUDA libraries" ON ) diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index b0be33798f9..b569119835a 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -226,6 +226,7 @@ add_custom_target(python-package -DPYTHON_VERSION=${PYTHON_VERSION} "-DCOMPILED_MODULE_PATH_LIST=${COMPILED_MODULE_PATH_LIST}" "-DPYTHON_EXTRA_LIBRARIES=${PYTHON_EXTRA_LIBRARIES}" + -DWITH_STUBGEN=${WITH_STUBGEN} -DIGNORE_STUBGEN_ERRORS=${IGNORE_STUBGEN_ERRORS} -DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION} -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 85018619396..d1b38659770 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -138,13 +138,15 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") # Generate typing stub files (.pyi) and py.typed marker file. -if(NOT IGNORE_STUBGEN_ERRORS) - list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") +if(WITH_STUBGEN) + if(NOT IGNORE_STUBGEN_ERRORS) + list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") + endif() + message(STATUS "Generating typing stubs...") + execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS} + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) + file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") endif() -message(STATUS "Generating typing stubs...") -execute_process( - COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS} - COMMAND_ECHO STDOUT - COMMAND_ERROR_IS_FATAL ANY -) -file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") From be32c1bbddd9e38d9040508627f2363b552224c4 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 15:34:50 +0200 Subject: [PATCH 17/21] Added WITH_STUBGEN and IGNORE_STUBGEN_ERRORS to cmake config summary. --- cmake/Open3DPrintConfigurationSummary.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index 3e3d42da11e..bd4cfc700bc 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -37,6 +37,10 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Unit Tests" "${BUILD_UNIT_TESTS}") open3d_aligned_print("Build Examples" "${BUILD_EXAMPLES}") open3d_aligned_print("Build Python Module" "${BUILD_PYTHON_MODULE}") + open3d_aligned_print("Generate Typing Stubs" "${WITH_STUBGEN}") + if(WITH_STUBGEN) + open3d_aligned_print("Ignore All Stubgen Errors" "${IGNORE_STUBGEN_ERRORS}") + endif() open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") From d44139f2447a1226010e053cd3feb1a8a5366a4d Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 15:36:15 +0200 Subject: [PATCH 18/21] Disable stubgen for windows shared libs workflows. --- .github/workflows/windows.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 44e10428903..cb5154c148b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -48,6 +48,7 @@ jobs: - BUILD_CUDA_MODULE: ON env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' ) && 'OFF' || 'ON' }} steps: - name: Disk space used @@ -121,6 +122,7 @@ jobs: -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} ` -DBUILD_UNIT_TESTS=ON ` -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} ` + -DWITH_STUBGEN=${{ matrix.WITH_STUBGEN }} ` ${{ env.SRC_DIR }} - name: Build @@ -211,14 +213,19 @@ jobs: cmake --build . --config ${{ matrix.CONFIG }} .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test Remove-Item "C:\Program Files\Open3D" -Recurse - - name: Install Open3D python requirements + - name: Install Open3D python build requirements working-directory: ${{ env.SOURCE_DIR }} run: | $ErrorActionPreference = 'Stop' python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -U -r python/requirements_build.txt - python -m pip install -U -r python/requirements.txt python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} + - name: Install Open3D python requirements for stubgen + working-directory: ${{ env.SOURCE_DIR }} + if: ${{ matrix.WITH_STUBGEN == 'ON' }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U -r python/requirements.txt - name: Install Python package working-directory: ${{ env.BUILD_DIR }} run: | From 6a4e6d23b7a20afb7d8f5c3684e7e7dd09a3805d Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 17:40:57 +0200 Subject: [PATCH 19/21] Disabled stubgen for windows cuda checks and fixed WITH_STUBGEN typo. --- .github/workflows/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cb5154c148b..cb4e76f2960 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -48,7 +48,7 @@ jobs: - BUILD_CUDA_MODULE: ON env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} - WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' ) && 'OFF' || 'ON' }} + WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' || matrix.BUILD_CUDA_MODULE == 'ON' ) && 'OFF' || 'ON' }} steps: - name: Disk space used @@ -122,7 +122,7 @@ jobs: -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} ` -DBUILD_UNIT_TESTS=ON ` -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} ` - -DWITH_STUBGEN=${{ matrix.WITH_STUBGEN }} ` + -DWITH_STUBGEN=${{ env.WITH_STUBGEN }} ` ${{ env.SRC_DIR }} - name: Build From c9fe4f0771ece1c26abfcfba6f5a28c98b7e8c49 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 16 Aug 2024 19:03:51 +0200 Subject: [PATCH 20/21] Fixed typo in windows workflow. --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cb4e76f2960..8fab5818504 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -222,7 +222,7 @@ jobs: python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} - name: Install Open3D python requirements for stubgen working-directory: ${{ env.SOURCE_DIR }} - if: ${{ matrix.WITH_STUBGEN == 'ON' }} + if: ${{ env.WITH_STUBGEN == 'ON' }} run: | $ErrorActionPreference = 'Stop' python -m pip install -U -r python/requirements.txt From 7dc7a3804b406931c121fae0449e43c36f6f241f Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sat, 31 Aug 2024 12:22:47 +0200 Subject: [PATCH 21/21] Delete open3d reference in __init__.py to fix `open3d = ` in stubgen. --- python/open3d/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index d4d5c613b0e..5a8c11556fb 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -212,4 +212,4 @@ def _jupyter_nbextension_paths(): if sys.platform == "win32": _win32_dll_dir.close() -del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names +del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names, open3d