for possible cython extensions

Author: jonathan-taylor

cythexts.py

@@ -0,0 +1,288 @@
+import os
+from os.path import splitext, sep as filesep, join as pjoin, relpath
+from hashlib import sha1
+
+from distutils.command.build_ext import build_ext
+from distutils.command.sdist import sdist
+from distutils.version import LooseVersion
+
+
+def derror_maker(klass, msg):
+    """ Decorate distutils class to make run method raise error """
+    class K(klass):
+        def run(self):
+            raise RuntimeError(msg)
+    return K
+
+
+def stamped_pyx_ok(exts, hash_stamp_fname):
+    """ Check for match of recorded hashes for pyx, corresponding c files
+
+    Parameters
+    ----------
+    exts : sequence of ``Extension``
+        distutils ``Extension`` instances, in fact only need to contain a
+        ``sources`` sequence field.
+    hash_stamp_fname : str
+        filename of text file containing hash stamps
+
+    Returns
+    -------
+    tf : bool
+        True if there is a corresponding c file for each pyx or py file in
+        `exts` sources, and the hash for both the (pyx, py) file *and* the c
+        file match those recorded in the file named in `hash_stamp_fname`.
+    """
+    # Calculate hashes for pyx and c files.  Check for presence of c files.
+    stamps = {}
+    for mod in exts:
+        for source in mod.sources:
+            base, ext = splitext(source)
+            if not ext in ('.pyx', '.py'):
+                continue
+            source_hash = sha1(open(source, 'rb').read()).hexdigest()
+            c_fname = base + '.c'
+            try:
+                c_file = open(c_fname, 'rb')
+            except IOError:
+                return False
+            c_hash = sha1(c_file.read()).hexdigest()
+            stamps[source_hash] = source
+            stamps[c_hash] = c_fname
+    # Read stamps from hash_stamp_fname; check in stamps dictionary
+    try:
+        stamp_file = open(hash_stamp_fname, 'rt')
+    except IOError:
+        return False
+    for line in stamp_file:
+        if line.startswith('#'):
+            continue
+        fname, hash = [e.strip() for e in line.split(',')]
+        if not hash in stamps:
+            return False
+        # Compare path made canonical for \/
+        fname = fname.replace(filesep, '/')
+        if not stamps[hash].replace(filesep, '/') == fname:
+            return False
+        stamps.pop(hash)
+    # All good if we found all hashes we need
+    return len(stamps) == 0
+
+
+def cyproc_exts(exts, cython_min_version,
+                hash_stamps_fname = 'pyx-stamps',
+                build_ext=build_ext):
+    """ Process sequence of `exts` to check if we need Cython.  Return builder
+
+    Parameters
+    ----------
+    exts : sequence of distutils ``Extension``
+        If we already have good c files for any pyx or py sources, we replace
+        the pyx or py files with their compiled up c versions inplace.
+    cython_min_version : str
+        Minimum cython version neede for compile
+    hash_stamps_fname : str, optional
+        filename with hashes for pyx/py and c files known to be in sync. Default
+        is 'pyx-stamps'
+    build_ext : distutils command
+        default build_ext to return if not cythonizing.  Default is distutils
+        ``build_ext`` class
+
+    Returns
+    -------
+    builder : ``distutils`` ``build_ext`` class or similar
+        Can be ``build_ext`` input (if we have good c files) or cython
+        ``build_ext`` if we have a good cython, or a class raising an informative
+        error on ``run()``
+    need_cython : bool
+        True if we need Cython to build extensions, False otherwise.
+    """
+    if stamped_pyx_ok(exts, hash_stamps_fname):
+        # Replace pyx with c files, use standard builder
+        for mod in exts:
+            sources = []
+            for source in mod.sources:
+                base, ext = splitext(source)
+                if ext in ('.pyx', '.py'):
+                    sources.append(base + '.c')
+                else:
+                    sources.append(source)
+            mod.sources = sources
+        return build_ext, False
+    # We need cython
+    try:
+        from Cython.Compiler.Version import version as cyversion
+    except ImportError:
+        return derror_maker(build_ext,
+                            'Need cython>={0} to build extensions '
+                            'but cannot import "Cython"'.format(
+                            cython_min_version)), True
+    if LooseVersion(cyversion) >= cython_min_version:
+        from Cython.Distutils import build_ext as extbuilder
+        return extbuilder, True
+    return derror_maker(build_ext,
+                        'Need cython>={0} to build extensions'
+                        'but found cython version {1}'.format(
+                        cython_min_version, cyversion)), True
+
+
+def build_stamp(pyxes, include_dirs=()):
+    """ Cythonize files in `pyxes`, return pyx, C filenames, hashes
+
+    Parameters
+    ----------
+    pyxes : sequence
+        sequence of filenames of files on which to run Cython
+    include_dirs : sequence
+        Any extra include directories in which to find Cython files.
+
+    Returns
+    -------
+    pyx_defs : dict
+        dict has key, value pairs of <pyx_filename>, <pyx_info>, where
+        <pyx_info> is a dict with key, value pairs of "pyx_hash", <pyx file SHA1
+        hash>; "c_filename", <c filemane>; "c_hash", <c file SHA1 hash>.
+    """
+    pyx_defs = {}
+    from Cython.Compiler.Main import compile
+    from Cython.Compiler.CmdLine import parse_command_line
+    includes = sum([['--include-dir', d] for d in include_dirs], [])
+    for source in pyxes:
+        base, ext = splitext(source)
+        pyx_hash = sha1(open(source, 'rt').read().encode('utf-8')).hexdigest()
+        c_filename = base + '.c'
+        options, sources = parse_command_line(includes + [source])
+        result = compile(sources, options)
+        if result.num_errors > 0:
+            raise RuntimeError('Cython failed to compile ' + source)
+        c_hash = sha1(open(c_filename, 'rt').read().encode('utf-8')).hexdigest()
+        pyx_defs[source] = dict(pyx_hash=pyx_hash,
+                                c_filename=c_filename,
+                                c_hash=c_hash)
+    return pyx_defs
+
+
+def write_stamps(pyx_defs, stamp_fname='pyx-stamps'):
+    """ Write stamp information in `pyx_defs` to filename `stamp_fname`
+
+    Parameters
+    ----------
+    pyx_defs : dict
+        dict has key, value pairs of <pyx_filename>, <pyx_info>, where
+        <pyx_info> is a dict with key, value pairs of "pyx_hash", <pyx file SHA1
+        hash>; "c_filename", <c filemane>; "c_hash", <c file SHA1 hash>.
+    stamp_fname : str
+        filename to which to write stamp information
+    """
+    with open(stamp_fname, 'wt') as stamp_file:
+        stamp_file.write('# SHA1 hashes for pyx files and generated c files\n')
+        stamp_file.write('# Auto-generated file, do not edit\n')
+        for pyx_fname, pyx_info in pyx_defs.items():
+            stamp_file.write('%s, %s\n' % (pyx_fname,
+                                           pyx_info['pyx_hash']))
+            stamp_file.write('%s, %s\n' % (pyx_info['c_filename'],
+                                           pyx_info['c_hash']))
+
+
+def find_pyx(root_dir):
+    """ Recursively find files with extension '.pyx' starting at `root_dir`
+
+    Parameters
+    ----------
+    root_dir : str
+        Directory from which to search for pyx files.
+
+    Returns
+    -------
+    pyxes : list
+        list of filenames relative to `root_dir`
+    """
+    pyxes = []
+    for dirpath, dirnames, filenames in os.walk(root_dir):
+        for filename in filenames:
+            if not filename.endswith('.pyx'):
+                continue
+            base = relpath(dirpath, root_dir)
+            pyxes.append(pjoin(base, filename))
+    return pyxes
+
+
+def get_pyx_sdist(sdist_like=sdist, hash_stamps_fname='pyx-stamps',
+                  include_dirs=()):
+    """ Add pyx->c conversion, hash recording to sdist command `sdist_like`
+
+    Parameters
+    ----------
+    sdist_like : sdist command class, optional
+        command that will do work of ``distutils.command.sdist.sdist``.  By
+        default we use the distutils version
+    hash_stamps_fname : str, optional
+        filename to which to write hashes of pyx / py and c files.  Default is
+        ``pyx-stamps``
+    include_dirs : sequence
+        Any extra include directories in which to find Cython files.
+
+    Returns
+    -------
+    modified_sdist : sdist-like command class
+        decorated `sdist_like` class, for compiling pyx / py files to c, putting
+        the .c files in the the source archive, and writing hashes for these
+        into the file named from `hash_stamps_fname`
+    """
+    class PyxSDist(sdist_like):
+        """ Custom distutils sdist command to generate .c files from pyx files.
+
+        Running the command object ``obj.run()`` will compile the pyx / py files
+        in any extensions, into c files, and add them to the list of files to
+        put into the source archive, as well as the usual behavior of distutils
+        ``sdist``.  It will also take the sha1 hashes of the pyx / py and c
+        files, and store them in a file ``pyx-stamps``, and put this file in the
+        release tree.  This allows someone who has the archive to know that the
+        pyx and c files that they have are the ones packed into the archive, and
+        therefore they may not need Cython at install time.  See
+        ``cython_process_exts`` for the build-time command.
+        """
+
+        def make_distribution(self):
+            """ Compile pyx to c files, add to sources, stamp sha1s """
+            pyxes = []
+            for mod in self.distribution.ext_modules:
+                for source in mod.sources:
+                    base, ext = splitext(source)
+                    if ext in ('.pyx', '.py'):
+                        pyxes.append(source)
+            self.pyx_defs = build_stamp(pyxes, include_dirs)
+            for pyx_fname, pyx_info in self.pyx_defs.items():
+                self.filelist.append(pyx_info['c_filename'])
+            sdist_like.make_distribution(self)
+
+        def make_release_tree(self, base_dir, files):
+            """ Put pyx stamps file into release tree """
+            sdist_like.make_release_tree(self, base_dir, files)
+            stamp_fname = pjoin(base_dir, hash_stamps_fname)
+            write_stamps(self.pyx_defs, stamp_fname)
+
+    return PyxSDist
+
+
+def build_stamp_source(root_dir=None, stamp_fname='pyx-stamps',
+                       include_dirs=None):
+    """ Build cython c files, make stamp file in source tree `root_dir`
+
+    Parameters
+    ----------
+    root_dir : None or str, optional
+        Directory from which to find ``.pyx`` files.  If None, use current
+        working directory.
+    stamp_fname : str, optional
+        Filename for stamp file we will write
+    include_dirs : None or sequence
+        Any extra Cython include directories
+    """
+    if root_dir is None:
+        root_dir = os.getcwd()
+    if include_dirs is None:
+        include_dirs = [pjoin(root_dir, 'src')]
+    pyxes = find_pyx(root_dir)
+    pyx_defs = build_stamp(pyxes, include_dirs=include_dirs)
+    write_stamps(pyx_defs, stamp_fname)