Include generated stubs in python package

.github/workflows/macos.yml

@@ -269,6 +269,11 @@ jobs:
           brew install ccache
           ccache -M 2G  # See .github/workflows/readme.md for ccache strategy.
 
+      - name: Install Open3D-ML Python dependencies
+        working-directory: ${{ env.OPEN3D_ML_ROOT }}
+        run: |
+          python -m pip install -r requirements.txt
+
       - name: Config and build wheel
         run: |
           PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH

.github/workflows/style.yml

@@ -27,7 +27,7 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat
+          pip install -U -r python/requirements_style.txt
       - name: Run style check
         run: |
           python util/check_style.py

.github/workflows/windows.yml

@@ -21,8 +21,6 @@ concurrency:
 
 env:
   PIP_VER: "23.2.1"
-  WHEEL_VER: "0.38.4"
-  STOOLS_VER: "67.3.2"
   JEDI_VER: "0.17.2"  # https://github.com/ipython/ipython/issues/12740
   IDNA_VER: "2.8"  # https://github.com/psf/requests/issues/5710
   CUDA_VERSION: "12.1.0"
@@ -50,6 +48,7 @@ jobs:
           - BUILD_CUDA_MODULE: ON
     env:
       BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }}
+      WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' || matrix.BUILD_CUDA_MODULE == 'ON' ) && 'OFF' || 'ON' }}
 
     steps:
       - name: Disk space used
@@ -123,6 +122,7 @@ jobs:
             -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} `
             -DBUILD_UNIT_TESTS=ON `
             -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} `
+            -DWITH_STUBGEN=${{ env.WITH_STUBGEN }} `
             ${{ env.SRC_DIR }}
 
       - name: Build
@@ -213,15 +213,23 @@ jobs:
           cmake --build . --config ${{ matrix.CONFIG }}
           .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test
           Remove-Item "C:\Program Files\Open3D" -Recurse
+      - name: Install Open3D python build requirements
+        working-directory: ${{ env.SOURCE_DIR }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -r python/requirements_build.txt
+          python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }}
+      - name: Install Open3D python requirements for stubgen
+        working-directory: ${{ env.SOURCE_DIR }}
+        if: ${{ env.WITH_STUBGEN == 'ON' }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U -r python/requirements.txt
       - name: Install Python package
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }} `
-            jedi==${{ env.JEDI_VER }} `
-            idna==${{ env.IDNA_VER }}
           cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package
       - name: Import python package
         # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python.
@@ -272,7 +280,9 @@ jobs:
         working-directory: ${{ env.SRC_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
           python -m pip install -r python/requirements.txt
+          python -m pip install -r python/requirements_build.txt
           python -m pip install -r python/requirements_jupyter_build.txt
 
       - name: Config
@@ -300,9 +310,6 @@ jobs:
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-                                          wheel==${{ env.WHEEL_VER }} `
-                                          setuptools==${{ env.STOOLS_VER }}
           cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package
           $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name
           echo "PIP_PKG_NAME=$PIP_PKG_NAME"  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
@@ -366,9 +373,8 @@ jobs:
           python -m venv open3d_test_venv
           open3d_test_venv\Scripts\Activate.ps1
 
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }}
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -r python/requirements_build.txt
           python -m pip install -U -r python/requirements_test.txt
           $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
           if (Test-Path -Path "pip_package") {

CHANGELOG.md

@@ -55,6 +55,7 @@
 -   Fix render to depth image on Apple Retina displays (PR #7001)
 -   Fix infinite loop in segment_plane if num_points < ransac_n (PR #7032)
 -   Add select_by_index method to Feature class (PR #7039)
+-   Include typing stubs in python package (PR #6917)
 
 
 ## 0.13

CMakeLists.txt

@@ -39,6 +39,8 @@ option(BUILD_EXAMPLES             "Build Open3D examples programs"           ON
 option(BUILD_UNIT_TESTS           "Build Open3D unit tests"                  OFF)
 option(BUILD_BENCHMARKS           "Build the micro benchmarks"               OFF)
 option(BUILD_PYTHON_MODULE        "Build the python module"                  ON )
+option(WITH_STUBGEN               "Use pybind11-stubgen to generate stubs"   ON )
+option(IGNORE_STUBGEN_ERRORS      "Ignore all errors during stub generation" ON )
 option(BUILD_CUDA_MODULE          "Build the CUDA module"                    OFF)
 option(BUILD_WITH_CUDA_STATIC     "Build with static CUDA libraries"         ON )
 option(BUILD_COMMON_CUDA_ARCHS    "Build for common CUDA GPUs (for release)" OFF)

cmake/Open3DPrintConfigurationSummary.cmake

@@ -37,6 +37,10 @@ function(open3d_print_configuration_summary)
     open3d_aligned_print("Build Unit Tests" "${BUILD_UNIT_TESTS}")
     open3d_aligned_print("Build Examples" "${BUILD_EXAMPLES}")
     open3d_aligned_print("Build Python Module" "${BUILD_PYTHON_MODULE}")
+    open3d_aligned_print("Generate Typing Stubs" "${WITH_STUBGEN}")
+    if(WITH_STUBGEN)
+        open3d_aligned_print("Ignore All Stubgen Errors" "${IGNORE_STUBGEN_ERRORS}")
+    endif()
     open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}")
     open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}")
     open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}")

cpp/pybind/CMakeLists.txt

@@ -226,6 +226,8 @@ add_custom_target(python-package
             -DPYTHON_VERSION=${PYTHON_VERSION}
             "-DCOMPILED_MODULE_PATH_LIST=${COMPILED_MODULE_PATH_LIST}"
             "-DPYTHON_EXTRA_LIBRARIES=${PYTHON_EXTRA_LIBRARIES}"
+            -DWITH_STUBGEN=${WITH_STUBGEN}
+            -DIGNORE_STUBGEN_ERRORS=${IGNORE_STUBGEN_ERRORS}
             -DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION}
             -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS}
             -DBUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS}

cpp/pybind/make_python_package.cmake

@@ -136,3 +136,17 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/"
      DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples")
 file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/"
      DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples")
+
+# Generate typing stub files (.pyi) and py.typed marker file.
+if(WITH_STUBGEN)
+    if(NOT IGNORE_STUBGEN_ERRORS)
+        list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code")
+    endif()
+    message(STATUS "Generating typing stubs...")
+    execute_process(
+        COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS}
+        COMMAND_ECHO STDOUT
+        COMMAND_ERROR_IS_FATAL ANY
+    )
+    file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed")
+endif()

docker/Dockerfile.wheel

@@ -109,11 +109,15 @@ RUN /root/Open3D/util/install_deps_ubuntu.sh assume-yes \
 # Open3D Python dependencies
 COPY ./util/ci_utils.sh /root/Open3D/util/
 COPY ./python/requirements.txt /root/Open3D/python/
+COPY ./python/requirements_build.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/
 RUN source /root/Open3D/util/ci_utils.sh \
  && install_python_dependencies with-jupyter
 
+# Open3D-ML Python dependencies
+RUN python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt"
+
 # Open3D Jupyter dependencies
 RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
  && apt-get install -y nodejs \

python/open3d/__init__.py

@@ -212,4 +212,4 @@ def _jupyter_nbextension_paths():
 
 if sys.platform == "win32":
     _win32_dll_dir.close()
-del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names
+del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names, open3d

python/pyproject.toml

@@ -1,3 +1,12 @@
 [build-system]
-requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"]
+requires = [
+    "ipywidgets>=8.0.4",
+    "pygments>=2.7.4",
+    "jupyter_packaging~=0.12",
+    "jupyterlab>=3.0.0,==3.*",
+    "setuptools>=67.3.2",
+    "wheel==0.38.4",
+    "yapf==0.30.0",
+    "pybind11-stubgen==2.5.1",
+]
 build-backend = "setuptools.build_meta"

python/requirements_build.txt

@@ -1,3 +1,4 @@
 setuptools>=67.3.2
 wheel==0.38.4
 yapf==0.30.0
+pybind11-stubgen==2.5.1

util/ci_utils.sh

@@ -30,9 +30,6 @@ TORCH_VER="2.2.2"
 TORCH_REPO_URL="https://download.pytorch.org/whl/torch/"
 # Python
 PIP_VER="23.2.1"
-WHEEL_VER="0.38.4"
-STOOLS_VER="67.3.2"
-YAPF_VER="0.30.0"
 PROTOBUF_VER="4.24.0"
 
 OPEN3D_INSTALL_DIR=~/open3d_install
@@ -42,10 +39,10 @@ install_python_dependencies() {
 
     echo "Installing Python dependencies"
     options="$(echo "$@" | tr ' ' '|')"
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install --upgrade pip=="$PIP_VER"
+    python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt"
     if [[ "with-unit-test" =~ ^($options)$ ]]; then
-        python -m pip install -U -r python/requirements_test.txt
+        python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt"
     fi
     if [[ "with-cuda" =~ ^($options)$ ]]; then
         TF_ARCH_NAME=tensorflow
@@ -64,7 +61,6 @@ install_python_dependencies() {
         TORCH_GLNX="torch==${TORCH_VER}+cpu"
     fi
 
-    # TODO: modify other locations to use requirements.txt
     python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt"
     if [[ "with-jupyter" =~ ^($options)$ ]]; then
         python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt"
@@ -88,7 +84,6 @@ install_python_dependencies() {
         fi
     fi
     if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then
-        python -m pip install -U yapf=="$YAPF_VER"
         # Fix Protobuf compatibility issue
         # https://stackoverflow.com/a/72493690/1255535
         # https://github.com/protocolbuffers/protobuf/issues/10051
@@ -241,8 +236,8 @@ test_wheel() {
     python -m venv open3d_test.venv
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install -U pip=="$PIP_VER"
+    python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt"
     echo -n "Using python: $(command -v python)"
     python --version
     echo -n "Using pip: "
@@ -285,7 +280,7 @@ test_wheel() {
 run_python_tests() {
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install -U -r python/requirements_test.txt
+    python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt"
     echo Add --randomly-seed=SEED to the test command to reproduce test order.
     pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/)
     if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then
@@ -356,9 +351,8 @@ install_docs_dependencies() {
     echo Install Python dependencies for building docs
     command -v python
     python -V
-    python -m pip install -U -q "wheel==$WHEEL_VER" \
-        "pip==$PIP_VER"
-    python -m pip install -U -q "yapf==$YAPF_VER"
+    python -m pip install -U -q "pip==$PIP_VER"
+    python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt"
     if [[ -d "$1" ]]; then
         OPEN3D_ML_ROOT="$1"
         echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}"