work around for build_ext config #352

Distribution.reinitialize_command implies that you get the same command
that get_command_obj() but fails to set options that were supplied in
config files or on the command line like Distribution.get_command_obj()
does.

This includes a monkeypatch to fix this issue.

.ycm_extra_conf.py

@@ -1,25 +1,27 @@
+"""
+khmer specific YouCompleteMe configuration
+"""
+# pylint: disable=missing-docstring
+
 import os
-import ycm_core
+import sys
+from distutils import core
+from distutils import ccompiler
 
 # These are the compilation flags that will be used in case there's no
 # compilation database set (by default, one is not set).
 # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR.
-flags = [
-    '-Wall',
-    #'-Wextra',
+FLAGS = [
+    # '-Wextra',
     '-Werror',
-    #'-Wc++98-compat',
-    #'-Wno-long-long',
-    #'-Wno-variadic-macros',
-    #'-fexceptions',
-    '-DNDEBUG',
-    '-fno-strict-aliasing',
-    '-pthread',
-    '-fwrapv',
-    # THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know which
-    # language to use when compiling headers. So it will guess. Badly. So C++
-    # headers will be compiled as C headers. You don't want that so ALWAYS specify
-    # a "-std=<something>".
+    # '-Wc++98-compat',
+    # '-Wno-long-long',
+    # '-Wno-variadic-macros',
+    # '-fexceptions',
+    # THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know
+    # which language to use when compiling headers. So it will guess. Badly. So
+    # C++ headers will be compiled as C headers. You don't want that so ALWAYS
+    # specify a "-std=<something>".
     # For a C project, you would set this to something like 'c99' instead of
     # 'c++11'.
     '-std=c++11',
@@ -29,42 +31,14 @@
     # For a C project, you would set this to 'c' instead of 'c++'.
     '-x',
     'c++',
-    '-I',
-    '.',
-    '-isystem',
-    '/usr/include',
-    '-isystem',
-    '/usr/local/include',
-    '-system',
-    '/usr/include/c++/4.8',
-    '-I',
-    '/usr/include/python2.7',
-    '-I',
-    'lib'
 ]
 
 
-# Set this to the absolute path to the folder (NOT the file!) containing the
-# compile_commands.json file to use that instead of 'flags'. See here for
-# more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html
-#
-# Most projects will NOT need to set this to anything; you can just change the
-# 'flags' list of compilation flags. Notice that YCM itself uses that approach.
-compilation_database_folder = ''
-
-if os.path.exists(compilation_database_folder):
-    database = ycm_core.CompilationDatabase(compilation_database_folder)
-else:
-    database = None
-
-SOURCE_EXTENSIONS = ['.cpp', '.cxx', '.cc', '.c', '.m', '.mm']
-
-
-def DirectoryOfThisScript():
+def directory_of_this_script():
     return os.path.dirname(os.path.abspath(__file__))
 
 
-def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
+def make_relative_paths_in_flags_absolute(flags, working_directory):
     if not working_directory:
         return list(flags)
     new_flags = []
@@ -93,44 +67,65 @@ def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
     return new_flags
 
 
-def IsHeaderFile(filename):
+def is_header_file(filename):
     extension = os.path.splitext(filename)[1]
     return extension in ['.h', '.hxx', '.hpp', '.hh']
 
 
-def GetCompilationInfoForFile(filename):
-    # The compilation_commands.json file generated by CMake does not have entries
-    # for header files. So we do our best by asking the db for flags for a
-    # corresponding source file, if any. If one exists, the flags for that file
-    # should be good enough.
-    if IsHeaderFile(filename):
-        basename = os.path.splitext(filename)[0]
-        for extension in SOURCE_EXTENSIONS:
-            replacement_file = basename + extension
-            if os.path.exists(replacement_file):
-                compilation_info = database.GetCompilationInfoForFile(
-                    replacement_file)
-                if compilation_info.compiler_flags_:
-                    return compilation_info
-        return None
-    return database.GetCompilationInfoForFile(filename)
-
-
-def FlagsForFile(filename, **kwargs):
-    if database:
-        # Bear in mind that compilation_info.compiler_flags_ does NOT return a
-        # python list, but a "list-like" StringVec object
-        compilation_info = GetCompilationInfoForFile(filename)
-        if not compilation_info:
-            return None
-
-        final_flags = MakeRelativePathsInFlagsAbsolute(
-            compilation_info.compiler_flags_,
-            compilation_info.compiler_working_dir_)
-
-    else:
-        relative_to = DirectoryOfThisScript()
-        final_flags = MakeRelativePathsInFlagsAbsolute(flags, relative_to)
+def get_prepared_build_ext():
+    sys.path.insert(1, directory_of_this_script())
+    core._setup_stop_after = "commandline"  # pylint: disable=protected-access
+    env = {'__file__': "setup.py"}
+    sys.argv = ["setup.py", "build_ext"]
+    try:
+        script = open(sys.argv[0])
+        try:
+            exec script in env, env  # pylint: disable=exec-used
+        finally:
+            script.close()
+    except SystemExit:
+        pass
+    except:
+        raise
+
+    core._setup_stop_after = None  # pylint: disable=protected-access
+    distribution = core._setup_distribution  # pylint: disable=protected-access
+    distribution.dry_run = True
+    build_ext = distribution.get_command_obj(command="build_ext")
+    build_ext.ensure_finalized()
+    build_ext.run()
+    return build_ext
+
+
+def get_setuptools_options():
+    build_ext = get_prepared_build_ext()
+    compiler = build_ext.compiler
+    options = []
+    if compiler:
+        if compiler.compiler_so:
+            options.extend(compiler.compiler_so[1:])
+        options.extend(ccompiler.gen_preprocess_options(
+            macros=compiler.macros,
+            include_dirs=compiler.include_dirs))
+        options.extend(ccompiler.gen_lib_options(
+            compiler=compiler,
+            library_dirs=compiler.library_dirs,
+            runtime_library_dirs=compiler.runtime_library_dirs,
+            libraries=compiler.libraries))
+    if build_ext.extensions:
+        for ext in build_ext.extensions:
+            options.extend(ext.extra_compile_args)
+    return options
+
+FLAGS.extend(get_setuptools_options())
+
+SOURCE_EXTENSIONS = ['.cpp', '.cxx', '.cc', '.c', '.m', '.mm']
+
+
+def FlagsForFile(  # pylint: disable=unused-argument,invalid-name
+        filename, **kwargs):
+    relative_to = directory_of_this_script()
+    final_flags = make_relative_paths_in_flags_absolute(FLAGS, relative_to)
 
     return {
         'flags': final_flags,

ChangeLog

@@ -1,23 +1,35 @@
-2014-03-25  Michael R. Crusoe <mcrusoe@msu.edu>
+2013-03-30  Michael R. Crusoe  <mcrusoe@msu.edu>
 
-    * Makefile: update cppcheck command to match new version of Jenkins
-    plugin. Now ignores the lib/test*.cc files.
+    * setup.py: monkeypatched distutils.Distribution.reinitialize_command() so
+    that it matches the behavior of Distribution.get_command_obj(). This fixes
+    issues with 'pip install -e' and './setup.py nosetests' not respecting the
+    setup.cfg configuration directives for the build_ext command. Also
+    enhanced our build_ext command to respect the dry_run mode.
+
+    * .ycm_extra_conf.py: Update our custom YouCompleteMe configuration to
+    query the package configuration for the proper compilation flags.
 
 2013-03-27  Michael R. Crusoe  <mcrusoe@msu.edu>
 
     * The system zlib and bzip2 libraries are now used instead of the bundled
     versions if specified in setup.cfg or the command line.
 
+2014-03-25  Michael R. Crusoe  <mcrusoe@msu.edu>
+
+    * Makefile: update cppcheck command to match new version of Jenkins
+    plugin. Now ignores the lib/test*.cc files.
+
 2013-03-20  Michael R. Crusoe  <mcrusoe@msu.edu>
 
     * lib/storage.hh,khmer/_khmermodule.cc,lib/{readtable,read_parsers}.hh:
     remove unused storage.hh
 
-2014-03-19  Qingpeng Zhang <qingpeng@msu.edu>
+2014-03-19  Qingpeng Zhang  <qingpeng@msu.edu>
 
     * hashbits.cc: fix a bug of 'Division or modulo by zero' described in #182
     * test_scripts.py: add test code for count-overlap.py
-    * count-overlap.py: (fix a bug because of a typo and hashsize was replaced by min_hashsize)
+    * count-overlap.py: (fix a bug because of a typo and hashsize was replaced
+    by min_hashsize)
     * count-overlap.py: needs hashbits table generated by load-graph.py. 
     This information is added to the "usage:" line.
     * count-overlap.py: fix minor PyLint issues

Makefile

@@ -13,10 +13,10 @@ dist: FORCE
 	./setup.py sdist
 
 clean: FORCE
-	./setup.py clean --all
 	cd lib && ${MAKE} clean
 	cd tests && rm -rf khmertest_*
 	rm -f khmer/_khmermodule.so
+	./setup.py clean --all
 
 debug:
 	export CFLAGS="-pg -fprofile-arcs"; python setup.py build_ext --debug \
@@ -76,7 +76,6 @@ lib:
 	$(MAKE)
 
 test: all
-	python -m nose # match the coverage command, work around bug in the setuptools
-	# nose command that wipes out the build_ext config
+	./setup.py nosetests
 
 FORCE:

setup.cfg

@@ -12,7 +12,7 @@ undef = NO_UNIQUE_RC
 ## if using system libraries
 include-dirs = lib:lib/zlib:lib/bzip2
 # include-dirs = lib
-## if using system libraries
+## if using system libraries (broken)
 
 # define = NDEBUG
 # is not needed for most Linux installs

setup.py

@@ -8,33 +8,31 @@
 import ez_setup
 ez_setup.use_setuptools(version="0.6c11")
 
+import os
+from os import listdir as os_listdir
+from os.path import join as path_join
+
 from setuptools import setup
 from setuptools import Extension
+from setuptools.command.build_ext import build_ext as _build_ext
+from distutils.spawn import spawn
+from distutils.sysconfig import get_config_vars
+from distutils.dist import Distribution
 
 import versioneer
 versioneer.versionfile_source = 'khmer/_version.py'
 versioneer.versionfile_build = 'khmer/_version.py'
 versioneer.tag_prefix = 'v'  # tags are like v1.2.0
 versioneer.parentdir_prefix = '.'
-
-from os.path import (
-    join as path_join,
-)
-
-from os import (
-    listdir as os_listdir
-)
-
-from subprocess import call
+CMDCLASS = versioneer.get_cmdclass()
 
 # strip out -Wstrict-prototypes; a hack suggested by
 # http://stackoverflow.com/a/9740721
 # proper fix coming in http://bugs.python.org/issue1222585
 # numpy has a "nicer" fix:
 # https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py
-import os
-from distutils.sysconfig import get_config_vars
-(OPT,) = get_config_vars('OPT')
+# pylint: disable=unpacking-non-sequence
+(OPT,) = get_config_vars('OPT')  # pylint: disable=unbalanced-tuple-unpacking
 os.environ['OPT'] = " ".join(
     flag for flag in OPT.split() if flag != '-Wstrict-prototypes'
 )
@@ -123,10 +121,8 @@
         ],
     }
 
-from setuptools.command.build_ext import build_ext as _build_ext
 
-
-class BuildExt(_build_ext):  # pylint: disable=R0904
+class KhmerBuildExt(_build_ext):  # pylint: disable=R0904
     """Specialized Python extension builder for khmer project.
 
     Only run the library setup when needed, not on every invocation.
@@ -137,19 +133,39 @@ class BuildExt(_build_ext):  # pylint: disable=R0904
 
     def run(self):
         if "z" and "bz2" not in self.libraries:
-            call('cd ' + ZLIBDIR + ' && ( test Makefile -nt configure || bash'
-                 ' ./configure --static ) && make -f Makefile.pic PIC',
-                 shell=True)
-            call('cd ' + BZIP2DIR + ' && make -f Makefile-libbz2_so all',
-                 shell=True)
+            spawn(cmd=['bash', '-c', 'cd ' + ZLIBDIR + ' && ( test Makefile '
+                       '-nt configure || bash ./configure --static ) && make '
+                       '-f Makefile.pic PIC'],
+                  dry_run=self.dry_run)
+            spawn(cmd=['bash', '-c', 'cd ' + BZIP2DIR + ' && make -f '
+                       'Makefile-libbz2_so all'],
+                  dry_run=self.dry_run)
         else:
             for ext in self.extensions:
                 ext.extra_objects = []
 
         _build_ext.run(self)
 
-CMDCLASS = versioneer.get_cmdclass()
-CMDCLASS.update({'build_ext': BuildExt})
+CMDCLASS.update({'build_ext': KhmerBuildExt})
+
+_DISTUTILS_REINIT = Distribution.reinitialize_command
+
+
+def reinitialize_command(self, command, reinit_subcommands):
+    '''
+    Monkeypatch distutils.Distribution.reinitialize_command() to match behavior
+    of Distribution.get_command_obj()
+
+    This fixes issues with 'pip install -e' and './setup.py nosetests' not
+    respecting the setup.cfg configuration directives for the build_ext command
+    '''
+    cmd_obj = _DISTUTILS_REINIT(self, command, reinit_subcommands)
+    options = self.command_options.get(command)
+    if options:
+        self._set_command_options(  # pylint: disable=protected-access
+            cmd_obj, options)
+    return cmd_obj
+Distribution.reinitialize_command = reinitialize_command
 
 # pylint: disable=W0142
 setup(cmdclass=CMDCLASS,