setup.py

# -*- coding: utf-8 -*-
from __future__ import absolute_import, division

import argparse
import os
import platform
import re
import shutil
import subprocess
import sys
import warnings

import setuptools
from distutils.core import Command

from Cython.Distutils import build_ext as _build_ext


__package__ = 'pydriver'

# set to True to skip automatic PCL_HELPER compilation (in this case you have to compile it manually before invoking setup.py)
SKIP_PCL_HELPER = False
if platform.system() == 'Windows':
    # requires manual compilation on Windows
    SKIP_PCL_HELPER = True

# current working directory (directory of setup.py)
cwd = os.path.abspath(os.path.dirname(__file__))

# pcl_helper directories
pcl_helper_dir = os.path.join(__package__, 'pcl', 'pcl_helper')
pcl_helper_dir_build = os.path.join(pcl_helper_dir, 'build')
pcl_helper_dir_lib = os.path.join(pcl_helper_dir, 'lib')

# version.py file path
version_py_path = os.path.join(cwd, __package__, 'version.py')
# source code template for version.py
version_py_src = """# this file was created automatically by setup.py
__version__ = '{version}'
__version_info__ = {{
    'full': __version__,
    'short': '.'.join(__version__.split('.')[:2])
}}
"""

def read(fname):
    return open(os.path.join(cwd, fname)).read()

def update_version_py():
    """Update version.py using "git describe" command"""
    if not os.path.isdir('.git'):
        print('This does not appear to be a Git repository, leaving version.py unchanged.')
        return False
    try:
        describe_output = subprocess.check_output(['git', 'describe', '--long', '--dirty']).decode('ascii').strip()
    except:
        print('Unable to run Git, leaving version.py unchanged.')
        return False
    # output looks like <version tag>-<commits since tag>-g<hash> and can end with '-dirty', e.g. v0.1.0-14-gd9f10e2-dirty
    # our version tags look like 'v0.1.0' or 'v0.1' and optionally additional segments (e.g. v0.1.0rc1), see PEP 0440
    describe_parts = re.match('^v([0-9]+\.[0-9]+(?:\.[0-9]+)?\S*)-([0-9]+)-g([0-9a-f]+)(?:-(dirty))?$', describe_output)
    assert describe_parts is not None, 'Unexpected output from "git describe": {}'.format(describe_output)
    version_tag, n_commits, commit_hash, dirty_flag = describe_parts.groups()
    version_parts = re.match('^([0-9]+)\.([0-9]+)(?:\.([0-9]+))?(\S*)$', version_tag)
    assert version_parts is not None, 'Unexpected version format: {}'.format(version_tag)
    version_major, version_minor, version_micro, version_segments = version_parts.groups()
    version_major = int(version_major)
    version_minor = int(version_minor)
    version_micro = int(version_micro) if version_micro is not None else 0
    n_commits = int(n_commits)
    if dirty_flag is not None:
        print('WARNING: Uncommitted changes detected.')
    if n_commits > 0:
        # non-exact match, dev version
        version_micro += 1
        version_segments += '.dev{}+{}'.format(n_commits, commit_hash)
    # final version string
    if version_micro > 0:
        version = '{}.{}.{}{}'.format(version_major, version_minor, version_micro, version_segments)
    else:
        version = '{}.{}{}'.format(version_major, version_minor, version_segments)
    with open(version_py_path, 'w') as f:
        f.write(version_py_src.format(version=version))
    print('Set version to: {}'.format(version))
    # success
    return True

# update version.py (if we're in a Git repository)
update_version_py()
# "import" version information without importing the package
exec(open(version_py_path).read())

class build_pcl_helper(Command):
    description = 'build pcl_helper library (inplace)'
    user_options = []
    def initialize_options(self):
        self.cwd_pcl_helper_dir_build = None
    def finalize_options(self):
        # build inplace
        self.cwd_pcl_helper_dir_build = os.path.join(cwd, pcl_helper_dir_build)
    def run(self):
        # create build dir if it doesn't exist
        if not os.path.exists(self.cwd_pcl_helper_dir_build):
            os.makedirs(self.cwd_pcl_helper_dir_build)
        # build pcl_helper
        if platform.system() == 'Windows':
            self._build_pcl_helper_windows(self.cwd_pcl_helper_dir_build)
        else:
            self._build_pcl_helper_linux(self.cwd_pcl_helper_dir_build)
    def _build_pcl_helper_linux(self, build_dir):
        subprocess.check_call(['cmake', '..'], cwd=build_dir)
        subprocess.check_call('make', cwd=build_dir)
    def _build_pcl_helper_windows(self, build_dir):
        raise NotImplementedError

class build_ext(_build_ext):
    user_options = _build_ext.user_options + [
        ('skip-pcl-helper', None, 'skip pcl_helper compilation (assume manual compilation)'),
    ]
    boolean_options = _build_ext.boolean_options + ['skip-pcl-helper']

    def initialize_options(self):
        _build_ext.initialize_options(self)
        # don't skip pcl helper by default
        self.skip_pcl_helper = False
        # pcl_helper location in source directory
        self.cwd_pcl_helper_dir_lib = None
        # pcl_helper location in package build directory
        self.build_pcl_helper_dir_lib = None

    def finalize_options(self):
        _build_ext.finalize_options(self)
        # prevent numpy from thinking it is still in its setup process:
        __builtins__.__NUMPY_SETUP__ = False
        import numpy as np
        self.include_dirs.append(np.get_include())
        # finalize pcl_helper directories
        self.cwd_pcl_helper_dir_lib = os.path.join(cwd, pcl_helper_dir_lib)
        self.build_pcl_helper_dir_lib = os.path.join(self.build_lib, pcl_helper_dir_lib)
        # check global flag SKIP_PCL_HELPER
        self.skip_pcl_helper = self.skip_pcl_helper or SKIP_PCL_HELPER

    def build_extensions(self, *args, **kwargs):
        compiler_type = self.compiler.compiler_type
        if compiler_type not in extra_args:
            compiler_type = 'unix'  # probably some unix-like compiler
        # merge compile and link arguments with global arguments for current compiler
        for e in self.extensions:
            e.extra_compile_args = list(set(e.extra_compile_args + extra_args[compiler_type]['extra_compile_args']))
            e.extra_link_args = list(set(e.extra_link_args + extra_args[compiler_type]['extra_link_args']))
        _build_ext.build_extensions(self, *args, **kwargs)

    def run(self):
        if not self.skip_pcl_helper:
            # build pcl_helper first
            try:
                self.run_command('build_pcl_helper')
            except:
                print('Error: pcl_helper could not be compiled automatically')
                print('Please compile pcl_helper manually (see %s/pcl/pcl_helper/README.rst for instructions)' % __package__ + \
                    ' and set SKIP_PCL_HELPER in setup.py to True.')
                raise
        # copy pcl_helper library to package build directory
        self.copy_tree(self.cwd_pcl_helper_dir_lib, self.build_pcl_helper_dir_lib)
        _build_ext.run(self)

    def get_outputs(self):
        # add contents of pcl_helper library directory to outputs (so they can be uninstalled)
        outputs = []
        for dirpath, dirnames, filenames in os.walk(self.build_pcl_helper_dir_lib):
            outputs.extend([os.path.join(dirpath, f) for f in filenames])
        return _build_ext.get_outputs(self) + outputs

class CleanCommand(Command):
    """Custom clean command to tidy up the project root."""
    user_options = []
    def initialize_options(self):
        pass
    def finalize_options(self):
        pass
    def run(self):
        self._remove_dirs('__pycache__')

        self._remove_dir(cwd, 'build')
        self._remove_dir(cwd, 'build_c')
        self._remove_dir(cwd, 'dist')
        self._remove_dir(cwd, '.eggs')
        self._remove_dir(cwd, '{}.egg-info'.format(__package__))
        self._remove_dir(cwd, pcl_helper_dir_build)
        self._remove_dir(cwd, pcl_helper_dir_lib)

        self._remove_files('pyc')
        self._remove_files('pyo')
        self._remove_files('pyd')
        self._remove_files('so')
    def _remove_dirs(self, dirname, parent_dir=None):
        if parent_dir is None:
            full_parent_dir = cwd
        else:
            full_parent_dir = os.path.join(cwd, parent_dir)
        matches = []
        for dirpath, dirnames, filenames in os.walk(full_parent_dir):
            matches.extend([os.path.join(dirpath, d) for d in dirnames if d==dirname])
        for d in matches:
            self._remove_dir(d)
    def _remove_dir(self, *args):
        dirpath = os.path.abspath(os.path.join(*args))
        # sanity checks
        if not os.path.exists(dirpath):
            # nothing to do
            return
        if not os.path.isdir(dirpath):
            print('"{}" is not a directory, aborting...'.format(dirpath))
            sys.exit()
        path_check = True
        if not dirpath.startswith(cwd):
            path_check = False
        if path_check and len(dirpath) > len(cwd):
            # first character after cwd should be a slash or a backslash
            if dirpath[len(cwd)] != os.sep:
                path_check = False
        if not path_check:
            print('The directory "{}" appears to be outside of main directory ({}), aborting...'.format(dirpath, cwd))
            sys.exit()
        # all sanity checks ok
        if not os.path.islink(dirpath):
            print('Removing directory: ' + dirpath)
            shutil.rmtree(dirpath, ignore_errors=True)
        else:
            print("Can't remove symlink to directory: " + dirpath)
    def _remove_files(self, ext, parent_dir=None):
        if parent_dir is None:
            full_parent_dir = cwd
        else:
            full_parent_dir = os.path.join(cwd, parent_dir)
        matches = []
        for dirpath, dirnames, filenames in os.walk(full_parent_dir):
            matches.extend([os.path.join(dirpath, f) for f in filenames if f.endswith('.'+ext)])
        for f in matches:
            self._remove_file(f)
    def _remove_file(self, *args):
        filepath = os.path.abspath(os.path.join(*args))
        # sanity checks
        if not os.path.exists(filepath):
            # nothing to do
            return
        if not os.path.isfile(filepath):
            print('"{}" is not a file, aborting...'.format(filepath))
            sys.exit()
        filepath_dir = os.path.abspath(os.path.dirname(filepath))
        path_check = True
        if not filepath_dir.startswith(cwd):
            path_check = False
        if path_check and len(filepath_dir) > len(cwd):
            # first character after cwd should be a slash or a backslash
            if filepath_dir[len(cwd)] != os.sep:
                path_check = False
        if not path_check:
            print('The file "{}" appears to be outside of main directory ({}), aborting...'.format(filepath, cwd))
            sys.exit()
        # all sanity checks ok
        print('Removing file: ' + filepath)
        os.remove(filepath)

class lazy_cythonize(list):
    # cythonize only if needed (e.g. not for "clean" command)
    def __init__(self, extensions, *args, **kwargs):
        self._list = None
        self.extensions = extensions
        self.args = args
        self.kwargs = kwargs
    def c_list(self):
        if self._list is None:
            self._list = self._cythonize()
        return self._list
    def __iter__(self):
        for e in self.c_list(): yield e
    def __getitem__(self, ii): return self.c_list()[ii]
    def __len__(self): return len(self.c_list())
    def _cythonize(self):
        from Cython.Build import cythonize
        return cythonize(self.extensions, *self.args, **self.kwargs)

# setup argument parser
parser = argparse.ArgumentParser(
    description = '%s setup script, basic install: python setup.py install' % __package__,
    epilog = 'Other arguments will be passed to setuptools, use --help-setup for more information.',
)
# add arguments which we will parse and pass to setuptools
parser.add_argument('command', nargs = '?',
                    help = 'command to pass to setuptools, use "install" to install package')
parser.add_argument('--debug', '-g', action = 'store_true',
                    help = 'compile/link with debugging information')
parser.add_argument('--force', '-f', action = 'store_true',
                    help = 'forcibly build everything (ignore file timestamps)')
parser.add_argument('--help-setup', action = 'store_true',
                    help = 'show setuptools help and exit')
# add own arguments
parser.add_argument('--annotate', action = 'store_true',
                    help = 'let Cython generate HTML files with performance information')
parser.add_argument('--cython-build-dir', default = 'build_c',
                    help = 'directory for C/C++ sources and HTML files generated by Cython (default: build_c)')
parser.add_argument('--inplace', action = 'store_true',
                    help = 'build inplace')
parser.add_argument('--no-openmp', dest = 'openmp', action = 'store_false',
                    help = 'compile/link without OpenMP support')
parser.add_argument('--profile', action = 'store_true',
                    help = 'enable profiling with cProfile')
parser.add_argument('--skip-pcl-helper', action = 'store_true',
                    help = 'skip pcl_helper compilation (assume manual compilation)')

# parse command line arguments
cmdargs, unknown_args = parser.parse_known_args()

if cmdargs.help_setup:
    # show setuptools help and exit
    sys.argv = [sys.argv[0], '--help']
    setuptools.setup()
    sys.exit()

# construct new command line arguments for setuptools
# leave the script name
setuptools_argv = [sys.argv[0]]
# pass setuptools options which we already have parsed
if cmdargs.command: setuptools_argv.append(cmdargs.command)
if cmdargs.force: setuptools_argv.append('--force')
if cmdargs.debug: setuptools_argv.append('--debug')
# add all unknown args
setuptools_argv += unknown_args
# replace sys.argv by arguments which will be passed to setuptools
sys.argv = setuptools_argv

# initialize setuptools arguments
setup_args = {
    'name': __package__,
    'version': __version__,
    'url': 'http://github.com/lpltk/pydriver',
    'license': 'MIT',
    'author': 'Leonard Plotkin',
    'author_email': 'git@leonard-plotkin.de',
    'description': 'A framework for training and evaluating object detectors and classifiers in road traffic environment.',
    'long_description': read('README.rst'),
    'zip_safe': False,
    'package_dir': {__package__: __package__},
    'packages': setuptools.find_packages(),
    'package_data': {__package__+'.pcl': ['pcl_helper/lib/*']},
    'include_package_data': True,
    'platforms': 'any',
    'setup_requires': [
        'numpy>=1.8.1',
        'cython>=0.22.1',
    ],
    'install_requires': [
        'numpy>=1.8.1',
        'cython>=0.22.1',
        'scipy>=0.13.3',
        'scikit-image',
        'scikit-learn',
    ],
    'classifiers': [
        'Development Status :: 3 - Alpha',
        'Environment :: Console',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Cython',
        'Programming Language :: C++',
        'Topic :: Scientific/Engineering :: Image Recognition',
    ],
}
# common include directories
setup_args['include_dirs'] = [
    os.path.join(__package__, 'common'),    # common constants, structs and typedefs
]

extra_args = {}
# arguments for unix compilers
extra_args['unix'] = {'extra_compile_args': [], 'extra_link_args': []}
if cmdargs.openmp:
    extra_args['unix']['extra_compile_args'].append('-fopenmp')
    extra_args['unix']['extra_link_args'].append('-fopenmp')
if not cmdargs.debug:
    extra_args['unix']['extra_compile_args'].append('-O3')  # maximum optimization
    extra_args['unix']['extra_compile_args'].append('-w')   # suppress warnings
# arguments for MSVC
# PYDs compiled with MSVC and /openmp could not be embedded in a MSVC project (regardless of the project's /openmp setting).
# They crash with the error R6034: An application has made an attempt to load the C runtime library incorrectly.
# The problem seems to be that the resulting exe and pyd binaries had incompatible manifests. It could also be caused by using
# the Visual Studio Express Edition which has limited support for extended features such as OpenMP.
extra_args['msvc'] = {'extra_compile_args': [], 'extra_link_args': []}
extra_args['msvc']['extra_compile_args'].append('/EHsc')    # exception handling option
if cmdargs.openmp:
    extra_args['msvc']['extra_compile_args'].append('/openmp')
if not cmdargs.debug:
    extra_args['msvc']['extra_compile_args'].append('/O2')  # optimize for speed
    extra_args['msvc']['extra_compile_args'].append('/W0')  # suppress warnings

# helper function for creating extensions with standard options
def create_extension(*args, **kwargs):
    def add_package_path(kwarg_key):
        # prepend package directory to every element in list in kwargs[kwarg_key]
        kwargs[kwarg_key] = [os.path.join(__package__, e) for e in kwargs.get(kwarg_key, [])]

    # add package name to extension module name and paths
    args = (__package__ + '.' + args[0],) + args[1:]
    add_package_path('sources')
    add_package_path('include_dirs')
    add_package_path('library_dirs')
    # generate C++ code (instead of C) by default
    if 'language' not in kwargs:
        kwargs['language'] = 'c++'
    return setuptools.Extension(*args, **kwargs)

extensions = [
        # common
        create_extension(
            'common.constants',
            sources = ['common/constants.pyx'],
        ),
        create_extension(
            'common.functions',
            sources = ['common/functions.pyx'],
        ),

        # geometry
        create_extension(
            'geometry.geometry',
            sources = ['geometry/geometry.pyx'],
        ),

        # stereo
        create_extension(
            'stereo.stereo',
            sources = ['stereo/stereo.pyx'],
        ),

        # pcl
        create_extension(
            'pcl.pcl',
            sources = ['pcl/pcl.pyx'],
            language = 'c++',
            include_dirs = ['pcl/pcl_helper'],
            libraries = ['pcl_helper'],
            library_dirs = ['pcl/pcl_helper/lib'],
            extra_link_args = ['-Wl,-rpath,$ORIGIN/pcl_helper/lib'] if platform.system() != 'Windows' else [], # handle paths in __init__.py on Windows
        ),

        # preprocessing
        create_extension(
            'preprocessing.preprocessing',
            sources = ['preprocessing/preprocessing.pyx'],
        ),

        # keypoints
        create_extension(
            'keypoints.base',
            sources = ['keypoints/base.pyx'],
        ),
        create_extension(
            'keypoints.harris',
            sources = ['keypoints/harris.pyx'],
        ),
        create_extension(
            'keypoints.iss',
            sources = ['keypoints/iss.pyx'],
        ),

        # features
        create_extension(
            'features.shot',
            sources = ['features/shot.pyx'],
        ),
        create_extension(
            'features.base',
            sources = ['features/base.pyx'],
        ),

        # detectors
        create_extension(
            'detectors.vocabularies',
            sources = ['detectors/vocabularies.pyx'],
        ),
        create_extension(
            'detectors.detectors',
            sources = ['detectors/detectors.pyx'],
        ),
    ]

# setup commands
setup_args['cmdclass'] = {
    'build_pcl_helper': build_pcl_helper,
    'build_ext': build_ext,
    'clean': CleanCommand,
}
# keyword arguments for cythonize()
cython_kwargs = {
    'build_dir': cmdargs.cython_build_dir,  # build directory
    'compiler_directives': {
        'embedsignature': True,             # embed signatures for documentation tools
    },
}
if cmdargs.force:
    cython_kwargs['force'] = True           # enforce full recompilation
if cmdargs.annotate:
    cython_kwargs['annotate'] = True        # generate HTML reports (in cmdargs.cython_build_dir)
if cmdargs.profile:
    if cmdargs.debug:
        warnings.warn(UserWarning('You should only profile in release mode with full optimization.'))
    cython_kwargs['profile'] = True         # globally enable profiling with cProfile
setup_args['ext_modules'] = lazy_cythonize(extensions, **cython_kwargs)
setup_args['options'] = {
    'build_ext': {
        'inplace': cmdargs.inplace,
        'skip_pcl_helper': cmdargs.skip_pcl_helper,
    },
}

try:
    setuptools.setup(**setup_args)
except Exception as e:
    print('Compilation errors encountered, aborting...')
    print('Exception information:')
    print(e)
    sys.exit(1)