diff --git a/.gitignore b/.gitignore index a1b0e63..481b43a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ pynvvl/*.cpp !examples/sample.png pynvvl/_lib/ docker/include/ +examples/*.txt # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index 5353bec..4ff879f 100644 --- a/README.md +++ b/README.md @@ -5,21 +5,28 @@ PyNVVL is a thin wrapper of [NVIDIA Video Loader (NVVL)](https://github.com/NVID ## Requirements +- Python 2.7.6+, 3.4.7+, 3.5.1+, 3.6.0+ - [CuPy](https://github.com/cupy/cupy) v4.0.0 -## Install +## Tested Environment -``` -pip install [WHEEL URL] -``` +- Ubuntu 16.04 +- Python 2.7.6+, 3.4.7+, 3.5.1+, 3.6.0+ + +## Install the pre-built binary -Replace `[WHEEL URL]` with one of the wheels below: +Please choose a right package depending on your CUDA version. -### Python 3.5 +```bash +# [For CUDA 8.0] +pip install pynvvl-cuda80 -- CUDA 8.0: `https://github.com/mitmul/pynvvl/releases/download/v0.0.1/pynvvl_cuda80-0.0.1-cp35-cp35m-linux_x86_64.whl` -- CUDA 9.0: `https://github.com/mitmul/pynvvl/releases/download/v0.0.1/pynvvl_cuda90-0.0.1-cp35-cp35m-linux_x86_64.whl` -- CUDA 9.1: `https://github.com/mitmul/pynvvl/releases/download/v0.0.1/pynvvl_cuda91-0.0.1-cp35-cp35m-linux_x86_64.whl` +# [For CUDA 9.0] +pip install pynvvl-cuda90 + +# [For CUDA 9.1] +pip install pynvvl-cuda91 +``` ## Usage @@ -103,7 +110,7 @@ Loads the video from disk and returns it as a CuPy ndarray. yuv 4:2:0 to 4:4:4. It should be 'Linear' currently. ``` -## Build wheels +## How to build wheels ### Requirements for build @@ -114,5 +121,6 @@ Loads the video from disk and returns it as a CuPy ndarray. bash docker/build_docker.sh sudo rm -rf docker/lib bash docker/build_nvvl.sh -bash docker/build_wheels.sh +sudo rm -rf build dist *.egg-info +python docker/build_wheels.py ``` diff --git a/docker/Dockerfile.develop b/docker/Dockerfile.develop index e90b065..1b25d3b 100644 --- a/docker/Dockerfile.develop +++ b/docker/Dockerfile.develop @@ -12,4 +12,6 @@ RUN pip3 install \ cython \ numpy \ matplotlib -RUN echo ${CUDA_VERSION} | sed -E "s/\.//g" | awk '{print substr ($0, 0, 2)}' | xargs -I{} pip3 install cupy-cuda{} + +ARG CUPY_PACKAGE_NAME +RUN pip3 install ${CUPY_PACKAGE_NAME} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index f260d31..bfe8aa0 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,18 +1,65 @@ -ARG CUDA_VERSION=9.1 -FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu16.04 +ARG cuda_version +FROM nvidia/cuda:${cuda_version}-devel-ubuntu16.04 -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,video,utility +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility RUN apt-get update && apt-get install -y \ - python3 \ - python3-dev \ - python3-pip \ - python3-tk \ - gawk - -RUN pip3 install \ - cython \ - numpy \ - matplotlib -RUN echo ${CUDA_VERSION} | sed -E "s/\.//g" | awk '{print substr ($0, 0, 2)}' | xargs -I{} pip3 install cupy-cuda{} + python \ + python-dev \ + python-pip \ + python-wheel \ + python-setuptools \ + gawk \ + make \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + git + +WORKDIR /root + +ENV HOME /root + +# Install pyenv. +RUN git clone https://github.com/pyenv/pyenv.git ${HOME}/.pyenv +ENV PYENV_ROOT ${HOME}/.pyenv +ENV PATH ${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PYENV_ROOT}/libexec:${PATH} +RUN eval "$(pyenv init -)" +RUN pyenv versions + +# Install Python. +ARG python_versions +# RUN pyenv install 3.5.1 +RUN for VERSION in ${python_versions}; do \ + echo "Installing Python ${VERSION}..." && \ + pyenv install ${VERSION} && \ + pyenv global ${VERSION} && \ + pyenv rehash && \ + echo "Finished"; \ + done; +RUN pyenv versions + +# Install Python libraries. +ARG cython_version +ARG cupy_version +ARG cupy_package_name +RUN for VERSION in ${python_versions}; do \ + echo "Installing libraries on Python ${VERSION}..." && \ + pyenv global ${VERSION} && \ + pip install -U pip setuptools && \ + pip install argparse && \ + pip install Cython==${cython_version} wheel auditwheel && \ + pip install numpy matplotlib ${cupy_package_name}; \ + done; + +ENV MPLBACKEND Agg diff --git a/docker/Dockerfile.wheels b/docker/Dockerfile.wheels new file mode 100644 index 0000000..60d4b30 --- /dev/null +++ b/docker/Dockerfile.wheels @@ -0,0 +1,65 @@ +ARG cuda_version +FROM mitmul/pynvvl:cuda-${cuda_version} + +ENV NVIDIA_DRIVER_CAPABILITIES=video,compute,utility + +RUN apt-get update && apt-get install -y \ + python \ + python-dev \ + python-pip \ + python-wheel \ + python-setuptools \ + gawk \ + make \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + git + +WORKDIR /root + +ENV HOME /root + +# Install pyenv. +RUN git clone https://github.com/pyenv/pyenv.git ${HOME}/.pyenv +ENV PYENV_ROOT ${HOME}/.pyenv +ENV PATH ${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PYENV_ROOT}/libexec:${PATH} +RUN eval "$(pyenv init -)" +RUN pyenv versions + +# Install Python. +ARG python_versions +# RUN pyenv install 3.5.1 +RUN for VERSION in ${python_versions}; do \ + echo "Installing Python ${VERSION}..." && \ + pyenv install ${VERSION} && \ + pyenv global ${VERSION} && \ + pyenv rehash && \ + echo "Finished"; \ + done; +RUN pyenv versions + +# Install Python libraries. +ARG cython_version +ARG cupy_package_name +ARG cupy_version +RUN for VERSION in ${python_versions}; do \ + echo "Installing libraries on Python ${VERSION}..." && \ + pyenv global ${VERSION} && \ + pip install -U pip setuptools && \ + pip install argparse && \ + pip install Cython==${cython_version} wheel auditwheel && \ + pip install ${cupy_package_name}==${cupy_version} && \ + pip freeze; \ + done; + diff --git a/docker/build_docker.sh b/docker/build_docker.sh index ae2bf00..9a93845 100644 --- a/docker/build_docker.sh +++ b/docker/build_docker.sh @@ -1,12 +1,18 @@ #!/bin/bash build_docker_image() { - docker build -t mitmul/pynvvl:cuda-$1 --build-arg CUDA_VERSION=$1 -f docker/Dockerfile.build-nvvl docker + docker build -t mitmul/pynvvl:cuda-$1 \ + --build-arg CUDA_VERSION=$1 \ + -f docker/Dockerfile.build-nvvl docker docker push mitmul/pynvvl:cuda-$1 - docker build -t mitmul/pynvvl:cuda-$1-dev --build-arg CUDA_VERSION=$1 -f docker/Dockerfile.develop docker + + docker build -t mitmul/pynvvl:cuda-$1-dev \ + --build-arg CUDA_VERSION=$1 \ + --build-arg CUPY_PACKAGE_NAME=$2 \ + -f docker/Dockerfile.develop docker docker push mitmul/pynvvl:cuda-$1-dev } -build_docker_image 8.0 -build_docker_image 9.0 -build_docker_image 9.1 +build_docker_image 8.0 cupy-cuda80 +build_docker_image 9.0 cupy-cuda90 +build_docker_image 9.1 cupy-cuda91 diff --git a/docker/build_nvvl.sh b/docker/build_nvvl.sh index 761bf32..9d5770c 100644 --- a/docker/build_nvvl.sh +++ b/docker/build_nvvl.sh @@ -4,9 +4,14 @@ build_libnvvl() { nvidia-docker run \ --rm \ -v $PWD/docker:/root/build -t mitmul/pynvvl:cuda-$1 \ - bash -c "find / -name \"*libnvcuvid.so.1\" | \ - xargs -I{} ln -s {} /usr/local/lib/libnvcuvid.so && \ - if [ ! -d /root/build/lib/cuda-$1 ]; then mkdir -p /root/build/lib/cuda-$1; fi && \ + bash -c " \ + if [ ! -f /usr/local/lib/libnvcuvid.so ]; then \ + find / -name \"*libnvcuvid.so.1\" | \ + xargs -I{} ln -s {} /usr/local/lib/libnvcuvid.so; \ + fi && \ + if [ ! -d /root/build/lib/cuda-$1 ]; then \ + mkdir -p /root/build/lib/cuda-$1; \ + fi && \ cp -r /usr/local/lib /root/build/lib/cuda-$1 && \ mv /root/build/lib/cuda-$1/lib/* /root/build/lib/cuda-$1/ && \ rm -rf /root/build/lib/cuda-$1/lib && \ diff --git a/docker/build_wheels.py b/docker/build_wheels.py new file mode 100644 index 0000000..2655230 --- /dev/null +++ b/docker/build_wheels.py @@ -0,0 +1,162 @@ +import os +import subprocess + +CYTHON_VERSION = '0.27.3' +CUPY_VERSION = '4.0.0' +PYNVVL_VERSION = '0.0.2a1' + +WHEEL_CONFIGS = { + '8.0': { + 'lib': 'docker/lib/cuda-8.0', + 'tag': 'mitmul/pynvvl:cuda-8.0-wheels', + 'test': 'mitmul/pynvvl:cuda-8.0-test', + }, + '9.0': { + 'lib': 'docker/lib/cuda-9.0', + 'tag': 'mitmul/pynvvl:cuda-9.0-wheels', + 'test': 'mitmul/pynvvl:cuda-9.0-test', + }, + '9.1': { + 'lib': 'docker/lib/cuda-9.1', + 'tag': 'mitmul/pynvvl:cuda-9.1-wheels', + 'test': 'mitmul/pynvvl:cuda-9.1-test', + }, +} + +PYTHON_VERSIONS = { + '2.7.6': { + 'python_tag': 'cp27', + 'linux_abi_tag': 'cp27mu', + }, + '3.4.7': { + 'python_tag': 'cp34', + 'linux_abi_tag': 'cp34m', + }, + '3.5.1': { + 'python_tag': 'cp35', + 'linux_abi_tag': 'cp35m', + }, + '3.6.0': { + 'python_tag': 'cp36', + 'linux_abi_tag': 'cp36m', + }, +} + + +def build_docker_image(cuda_version, tag, test): + python_versions = ' '.join(PYTHON_VERSIONS.keys()) + cudda_version_no_dot = cuda_version.replace('.', '') + subprocess.call([ + 'docker', 'build', + '--build-arg', 'cuda_version={}'.format(cuda_version), + '--build-arg', 'python_versions={}'.format(python_versions), + '--build-arg', 'cython_version={}'.format(CYTHON_VERSION), + '--build-arg', 'cupy_version={}'.format(CUPY_VERSION), + '--build-arg', 'cupy_package_name=cupy-cuda{}'.format( + cudda_version_no_dot), + '-t', tag, + '-f', 'docker/Dockerfile.wheels', 'docker' + ]) + subprocess.call([ + 'docker', 'build', + '--build-arg', 'cuda_version={}'.format(cuda_version), + '--build-arg', 'python_versions={}'.format(python_versions), + '--build-arg', 'cython_version={}'.format(CYTHON_VERSION), + '--build-arg', 'cupy_version={}'.format(CUPY_VERSION), + '--build-arg', 'pynvvl_version={}'.format(PYNVVL_VERSION), + '--build-arg', 'cupy_package_name=cupy-cuda{}'.format( + cudda_version_no_dot), + '-t', test, + '-f', 'docker/Dockerfile.test', 'docker' + ]) + + +def build_wheels(cuda_version): + for python_version in PYTHON_VERSIONS.keys(): + print('-' * 10, + 'Building for Python {}'.format(python_version), + '-' * 10) + subprocess.call( + 'nvidia-docker run' + ' --rm' + ' -v {source_dir}:/pynvvl' + ' -t {tag}' + ' bash -c' + ' " \ + find / -name \"*libnvcuvid.so.1\" | \ + xargs -IXXX ln -s XXX /usr/local/lib/libnvcuvid.so && \ + pyenv global {python_version} && pyenv rehash && \ + cd /pynvvl && python setup.py bdist_wheel \ + -d dist/cuda-{cuda_version} \ + --package-name {package_name} \ + "'.format( + source_dir=os.getcwd(), + tag=WHEEL_CONFIGS[cuda_version]['tag'], + python_version=python_version, + cuda_version=cuda_version, + package_name='pynvvl_cuda{}'.format( + cuda_version.replace('.', '')), + ), shell=True) + + subprocess.call( + 'nvidia-docker run' + ' --rm' + ' -v {source_dir}:/pynvvl' + ' -t {tag}' + ' bash -c' + ' " \ + for file in \$(ls /pynvvl/dist/cuda-{cuda_version}/*.whl); \ + do \ + echo \$file | \ + sed --expression=\"s/linux/manylinux1/g\" | \ + xargs -IXXX mv \$file XXX; \ + done; \ + "'.format( + source_dir=os.getcwd(), + tag=WHEEL_CONFIGS[cuda_version]['tag'], + cuda_version=cuda_version, + ), shell=True) + + for python_version, tags in PYTHON_VERSIONS.items(): + print('-' * 10, + 'Testing wheel for Python {}'.format(python_version), + '-' * 10) + package_python = '{}-{}'.format( + tags['python_tag'], tags['linux_abi_tag']) + + # Test the wheel + wheel_name = '{}-{}-{}-manylinux1_x86_64.whl'.format( + 'pynvvl_cuda{}'.format(cuda_version.replace('.', '')), + PYNVVL_VERSION, + package_python + ) + subprocess.call( + 'nvidia-docker run' + ' --rm' + ' -v {source_dir}/examples:/examples' + ' -v {source_dir}/dist/cuda-{cuda_version}:/wheels' + ' -t {tag}' + ' bash -c' + ' " \ + pyenv global {python_version} && pyenv rehash && \ + pip install /wheels/{wheel_name} && \ + cd / && python examples/simple_load.py \ + > /examples/cuda-{cuda_version}_python-{python_version}.txt \ + "'.format( + source_dir=os.getcwd(), + tag=WHEEL_CONFIGS[cuda_version]['test'], + cuda_version=cuda_version, + python_version=python_version, + wheel_name=wheel_name, + ), shell=True) + + +# Build Docker images +for cuda_version, wheel_config in WHEEL_CONFIGS.items(): + build_docker_image(cuda_version, wheel_config['tag'], wheel_config['test']) + +# Build wheels +for cuda_version, wheel_config in WHEEL_CONFIGS.items(): + print('-' * 10, 'Building for CUDA {}'.format(cuda_version), '-' * 10) + build_wheels(cuda_version) + print('=' * 30) diff --git a/docker/build_wheels.sh b/docker/build_wheels.sh deleted file mode 100644 index 1f5a559..0000000 --- a/docker/build_wheels.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -build_wheels () { - nvidia-docker run \ - --rm \ - -v $PWD:/root/pynvvl -ti mitmul/pynvvl:cuda-$1-dev \ - bash -c "find / -name \"*libnvcuvid.so.1\" | \ - xargs -I{} ln -s {} /usr/local/lib/libnvcuvid.so && \ - cd /root/pynvvl && python3 setup.py bdist_wheel && \ - if [ ! -d dist/cuda-$1 ]; then mkdir -p dist/cuda-$1; fi && \ - ls dist/*.whl | xargs -I{} mv {} dist/cuda-$1/" -} - -build_wheels 8.0 -build_wheels 9.0 -build_wheels 9.1 diff --git a/docker/upload_wheels.sh b/docker/upload_wheels.sh new file mode 100644 index 0000000..a62bdb7 --- /dev/null +++ b/docker/upload_wheels.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +twine upload --repository pypi dist/cuda-8.0/*.whl +twine upload --repository pypi dist/cuda-9.0/*.whl +twine upload --repository pypi dist/cuda-9.1/*.whl diff --git a/pynvvl/__init__.pxd b/pynvvl/__init__.pxd new file mode 100644 index 0000000..e69de29 diff --git a/pynvvl/__init__.py b/pynvvl/__init__.py new file mode 100644 index 0000000..3846218 --- /dev/null +++ b/pynvvl/__init__.py @@ -0,0 +1,9 @@ +from pynvvl._nvvl import NVVLVideoLoader # NOQA + + +__copyright__ = 'Copyright (C) 2018 Shunta Saito' +__version__ = '0.0.2a1' +__license__ = 'MIT License' +__author__ = 'Shunta Saito' +__author_email__ = 'shunta.saito@gmail.com' +__url__ = 'http://github.com/mitmul/pynvvl' diff --git a/pynvvl/nvvl.pyx b/pynvvl/_nvvl.pyx similarity index 100% rename from pynvvl/nvvl.pyx rename to pynvvl/_nvvl.pyx diff --git a/setup.py b/setup.py index f49f1ca..d25f1ef 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,17 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import argparse import os import shutil +import subprocess +import sys +import sysconfig from Cython.Distutils import build_ext -import numpy as np +from pkg_resources import get_distribution from setuptools import Extension from setuptools import setup -import subprocess CUDA_VERSION = subprocess.check_output( 'nvcc -V | grep -oP "release\s([0-9\.]+)" | grep -oP "([0-9\.]+)"', @@ -17,37 +20,42 @@ def create_extensions(): sourcefiles = [ - 'pynvvl/nvvl.pyx', + 'pynvvl/_nvvl.pyx', ] - cpath = os.environ['CPATH'] if 'CPATH' in os.environ else '' - includefiles = [ - 'docker/include', + # List up include paths + include_dirs = [ + os.path.join(os.getcwd(), 'docker/include'), '/usr/local/cuda/include', - cpath, - np.get_include() + sysconfig.get_config_var('INCLUDEPY'), ] + if 'CPATH' in os.environ: + include_dirs.insert(1, os.environ['CPATH']) - ld_library_path = os.environ['LD_LIBRARY_PATH'] \ - if 'LD_LIBRARY_PATH' in os.environ else '' + # List up library paths library_dirs = [ - 'docker/lib/cuda-{}'.format(CUDA_VERSION), - ld_library_path + os.path.join(os.getcwd(), 'docker/lib/cuda-{}'.format(CUDA_VERSION)), ] + if 'LD_LIBRARY_PATH' in os.environ: + library_dirs.append(os.environ['LD_LIBRARY_PATH']) + if 'LIBRARY_PATH' in os.environ: + library_dirs.append(os.environ['LIBRARY_PATH']) + # List up libraries libraries = [ 'nvvl', ] + # RPATH which will be set to pynvvl.so rpath = [ - '$ORIGIN/pynvvl/_lib', + '$ORIGIN/_lib', ] extensions = [ Extension( - 'pynvvl', + 'pynvvl._nvvl', sourcefiles, - include_dirs=includefiles, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries, language='c++', @@ -83,19 +91,40 @@ def prepare_package_data(): return package_data +parser = argparse.ArgumentParser() +parser.add_argument('--package-name', type=str, default='pynvvl') +args, sys.argv = parser.parse_known_args(sys.argv) + package_data = prepare_package_data() extensions = create_extensions() +cupy_package_name = None +try: + cupy_package_name = get_distribution( + 'cupy-cuda{}'.format(CUDA_VERSION.replace('.', ''))) + cupy_package_name = cupy_package_name.project_name +except Exception: + cupy_package_name = 'cupy' + +print('=' * 30) +print('CuPy Package Name:', cupy_package_name) +print('=' * 30) setup( - name='pynvvl-cuda{}'.format(CUDA_VERSION.replace('.', '')), + name=args.package_name, url='https://github.com/mitmul/pynvvl', - version='0.0.1', + version='0.0.2a1', author='Shunta Saito', author_email='shunta.saito@gmail.com', - license='MIT', + license='MIT License', packages=['pynvvl'], package_data=package_data, - cmdclass={'build_ext': build_ext}, + install_requires=[ + '{}>=4.0.0'.format(cupy_package_name), + ], + setup_requires=[ + 'cython>=0.27.3', + ], ext_modules=extensions, + cmdclass={'build_ext': build_ext}, )