Refactor docker setup (#478)

* remove poetry and simplify docker file

* run docker on self hosted for GPU tests

* ensure scikit build

* updating README

* fix: missing scikit-build

* adding trt requirement for local tests

* revert pyproject.toml to support skbuild automatically again

* remove docker from makefile

* good luck

* einer geht noch

* remove requirements.txt deps

* solving how to install torch as buidl req

* add some magic

* update setuptools

* upgrade pip

* update readme

Author: gedoensmax

.github/workflows/build.yml

@@ -42,8 +42,10 @@ jobs:
           libavformat-dev \
           libavcodec-dev \
           libswresample-dev \
-          libavutil-dev\
+          libavutil-dev \
+          python3-testresources\
           -y
+        python -mpip install -U setuptools wheel pip
 
     - name: Install dependencies (Windows)
       if: ${{ matrix.os == 'windows-2022' }}

.github/workflows/docker.yml

@@ -17,11 +17,20 @@ jobs:
     # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
     # You can convert this to a matrix build if you need cross-platform coverage.
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: ubuntu-latest 
+    runs-on: self-hosted 
 
     steps:
     - uses: actions/checkout@v3
     - name: Build docker containers
       run: | 
-        make vpf-gpu
-        make vpf-gpu-all
+        DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --tag vpf-gpu --build-arg PIP_INSTALL_EXTRAS=torch .
+        DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.tensorrt --build-arg PIP_INSTALL_EXTRAS=samples --tag vpf-gpu-trt .
+
+    - name: Run tests
+      run: | 
+        docker run --gpus all --rm -v $(shell pwd):/cwd --entrypoint "python3" vpf-gpu  -m unittest discover /repo/tests
+
+    - name: Run samples
+      run: | 
+        docker run --gpus all --rm -v $(shell pwd):/cwd --entrypoint "make" vpf-gpu-trt run_samples_without_docker
+    

.github/workflows/test.yml

@@ -23,26 +23,28 @@ jobs:
         cd ~/FFmpeg/
         ./prepare.sh  
       
-    - name: Setup Poetry
-      run: poetry install --extras vpf-samples        
-
+    - name: Setup pip
+      run: |
+        python3 -m venv /tmp/venv_vpf
+        source /tmp/venv_vpf/bin/activate   
+        
     - name: Build and Install Package
       env:
         PKG_CONFIG_PATH: /home/gh-runner/FFmpeg/build_x64_release_shared/lib/pkgconfig
         CXX: /usr/lib/ccache/c++
         CC: /usr/lib/ccache/cc
         SKBUILD_CONFIGURE_OPTIONS: "-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache"
       run: | 
-        source $(poetry env info --path)/bin/activate
-        python3 -mpip install . --verbose
-        pip install src/PytorchNvCodec --verbose
+        source /tmp/venv_vpf/bin/activate
+        python3 -mpip install .[samples] --verbose
+        python3 -mpip install src/PytorchNvCodec --verbose
 
     - name: Run Tests
       run: |
-        source $(poetry env info --path)/bin/activate
+        source /tmp/venv_vpf/bin/activate
         python3 -munittest discover tests
 
     - name: Run Samples
       run: |
-        source $(poetry env info --path)/bin/activate
+        source /tmp/venv_vpf/bin/activate
         make run_samples_without_docker

.gitignore

@@ -104,13 +104,6 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
 #pdm.lock

Makefile

@@ -1,24 +1,9 @@
-vpf-gpu:
-	DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.gpu --tag vpf-gpu .
-
-vpf-gpu-all:
-	DOCKER_BUILDKIT=1 docker build \
-					--tag vpf-gpu-all \
-					-f docker/Dockerfile.gpu \
-					--build-arg GEN_PYTORCH_EXT=1 \
-					--build-arg GEN_OPENGL_EXT=1 \
-					.
-
-run_tests: vpf-gpu
-	docker run --rm -v $(shell pwd):/repo --entrypoint "python3" vpf-gpu  -m unittest discover /repo/tests /repo/tests/test.mp4
-
-run_samples: vpf-gpu-all
-	docker run --rm -v $(shell pwd):/repo --entrypoint "python3" vpf-gpu-all -m unittest discover /repo/tests /repo/tests/test.mp4
-	docker run --rm -v $(shell pwd):/repo --workdir /repo --entrypoint "make" vpf-gpu-all run_samples_without_docker
+run_tests:
+	python -m unittest discover tests
 
 run_samples_without_docker:
 	wget http://www.scikit-video.org/stable/_static/bikes.mp4
-
+	
 	python ./samples/SampleDecode.py -g 0 -e ./bikes.mp4 -r ./tests/test.raw       
 	python ./samples/SampleDecodeSw.py -e ./bikes.mp4 -r ./tests/test.raw
 	python ./samples/SampleEncodeMultiThread.py 0 848 464 ./tests/test.raw 10
@@ -43,7 +28,7 @@ generate-stubs:
 	stubgen -mPyNvCodec._PyNvCodec
 	cp out/PyNvCodec/_PyNvCodec.pyi src/PyNvCodec/__init__.pyi
 
-.PHONY: run_tests vpf-gpu generate-stubs
+.PHONY: run_tests generate-stubs
 
 # vim:ft=make
 #

README.md

@@ -2,7 +2,7 @@
 
 VPF stands for Video Processing Framework. It’s set of C++ libraries and Python bindings which provides full HW acceleration for video processing tasks such as decoding, encoding, transcoding and GPU-accelerated color space and pixel format conversions.
 
-VPF also supports exporting GPU memory objects such as decoded video frames to PyTorch tensors without Host to Device copies. Check the [Wiki page](https://github.com/NVIDIA/VideoProcessingFramework/wiki/Building-from-source) on how to build from source.
+VPF also supports exporting GPU memory objects such as decoded video frames to PyTorch tensors without Host to Device copies. 
 
 ## Prerequisites
 VPF works on Linux(Ubuntu 20.04 and Ubuntu 22.04 only) and Windows
@@ -61,8 +61,8 @@ please make sure to enable the `video` driver capability: https://docs.nvidia.co
 the `NVIDIA_DRIVER_CAPABILITIES` environment variable in the container or the `--gpus` command line parameter (e.g.
 `docker run -it --rm --gpus 'all,"capabilities=compute,utility,video"' nvidia/cuda:12.1.0-base-ubuntu22.04`).
 
-Please note that some examples have additional dependencies https://github.com/NVIDIA/VideoProcessingFramework/blob/73a14683a17c8f1c7fa6dd73952f8813bd34a11f/setup.py#L26-L31
-that need to be installed via pip. Samples using PyTorch will require an optional extension which can be installed via
+Please note that some examples have additional dependencies that need to be installed via pip (`pip install .[samples]`). 
+Samples using PyTorch will require an optional extension which can be installed via
 ```bash
 pip install src/PytorchNvCodec  # install Torch extension if needed (optional), requires "torch" to be installed before
 ```
@@ -78,15 +78,14 @@ After resolving those you should be able to run `make run_samples_without_docker
 ```pwsh
 # Indicate path to your FFMPEG installation (with subfolders `bin` with DLLs, `include`, `lib`)
 $env:SKBUILD_CONFIGURE_OPTIONS="-DTC_FFMPEG_ROOT=C:/path/to/your/ffmpeg/installation/ffmpeg/" 
-
 pip install .
 ```
 To check whether VPF is correctly installed run the following Python script
 ```python
 import PyNvCodec
 ```
-Please note that some examples have additional dependencies https://github.com/NVIDIA/VideoProcessingFramework/blob/73a14683a17c8f1c7fa6dd73952f8813bd34a11f/setup.py#L26-L31
-that need to be installed via pip. Samples using PyTorch will require an optional extension which can be installed via
+Please note that some examples have additional dependencies (`pip install .[sampels]`) that need to be installed via pip. 
+Samples using PyTorch will require an optional extension which can be installed via
 
 ```bash
 pip install src/PytorchNvCodec  # install Torch extension if needed (optional), requires "torch" to be installed before
@@ -101,14 +100,15 @@ are required)
 
 ```bash
 DOCKER_BUILDKIT=1 docker build \
-                --tag vpf-gpu-all \
-                -f docker/Dockerfile.gpu \
-                --build-arg GEN_PYTORCH_EXT=1 \
-                --build-arg GEN_OPENGL_EXT=1 \
+                --tag vpf-gpu \
+                -f docker/Dockerfile \
+                --build-arg PIP_INSTALL_EXTRAS=torch \
                 .
-docker run -it --rm --gpus=all vpf-gpu-all
+docker run -it --rm --gpus=all vpf-gpu
 ```
 
+`PIP_INSTALL_EXTRAS` can be any subset listed under `project.optional-dependencies` in [pyproject.toml](pyproject.toml).
+
 ## Documentation
 
 A documentation for Video Processing Framework can be generated from this repository:

docker/Dockerfile

@@ -0,0 +1,35 @@
+FROM nvidia/cuda:12.1.1-devel-ubuntu22.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,video
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install \
+    libavfilter-dev \
+    libavformat-dev \
+    libavcodec-dev \
+    libswresample-dev \
+    libavutil-dev\
+    wget \
+    build-essential \
+    ninja-build \
+    cmake \
+    git \
+    python3 \
+    python3-pip \
+    python-is-python3
+
+ARG PIP_INSTALL_EXTRAS=""
+# Build vpf
+RUN mkdir /src
+WORKDIR /src
+COPY src src
+COPY setup.py setup.py
+COPY pyproject.toml pyproject.toml
+COPY CMakeLists.txt CMakeLists.txt
+RUN python3 -m pip install --no-cache-dir setuptools wheel
+RUN python3 -m pip install --no-cache-dir .[$PIP_INSTALL_EXTRAS] 
+
+RUN mkdir /cwd
+WORKDIR /cwd
+
+ENTRYPOINT ["/bin/bash"]