dabuild

#!/usr/bin/python

import dabuildsys
from dabuildsys import reprepro, BuildError, all_arch

from collections import OrderedDict
from functools import partial
from itertools import groupby
from pprint import pprint as pp
import argparse
import datetime
import os.path
import subprocess
import sys

# FIXME: there should be a sbuild-update -udcar invocation before
def build_package(distro, release, package, arch):
    """
    Build the binary package for the specified release, and include it into
    the specified APT repository.
    """

    try:
        dsc_path, = [f.path for f in package.files if f.name.endswith('.dsc')]
    except ValueError:
        raise BuildError("Source package %s has more than one dsc file" % package.name)

    tag = '~' + dabuildsys.release_tags[release]

    # Determine all arguments for sbuild invocation
    sbuild_cmd = ['sbuild']
    sbuild_cmd += ['--append-to-version', tag]
    sbuild_cmd += ['-d', release]
    sbuild_cmd += ['--arch', arch]
    if arch == all_arch:
        sbuild_cmd += ['-A']
    sbuild_cmd += ['-v']
    sbuild_cmd += ['--resolve-alternatives']
    sbuild_cmd += ['--setup-hook', dabuildsys.setup_hook_path]
    sbuild_cmd += [dsc_path]

    # Indicate which package we are currently building
    fullname = "%s_%s%s_%s" % (package.name, package.version, tag, arch)
    timestamp = datetime.datetime.now().strftime("%H:%M:%S")
    sys.stdout.write("[%s] Building %s... " % (timestamp, fullname))
    sys.stdout.flush()

    # Determine the location where we expect the output
    build_dir = dabuildsys.binary_package_dir
    changes_file = os.path.join(build_dir, fullname + '.changes')

    # Actual build happens here
    subprocess.check_output(sbuild_cmd, cwd = build_dir)
    sys.stdout.write("success! Including into %s... " % distro.name)
    sys.stdout.flush()
    # Import package into the repository.  Note that this has to happen in-loop,
    # because build dependencies are fetched from the APT repository
    reprepro.include_changes(distro.name, changes_file)
    print "done."

def union(s):
    """
    Given an iterator over sets, or objects which can be converted to sets,
    create their union.
    """

    return reduce(lambda x, y: x | y, s, frozenset())

def resolve_build_order_core(sources, binary_map, build_deps, bin_deps):
    """
    Given a list of source packages, map of source packages to
    binary packages they provide, map of build dependencies and map
    of dependencies of binaries, resolve the order in which the source
    packages need to be built.  Dependencies name are of string -> set()
    format.
    """

    def resolve_dependencies_recursively(package, stack):
        """
        Recursively resolve binary dependencies for a single package.
        The stack parameter is used to avoid dependency loops.
        """

        if package in stack:
            raise BuildError("Dependency loop detected with package %s" % package)

        direct_deps = frozenset(bin_deps[package])
        # Expanded dependencies of package X is union of its direct dependencies and
        # expanded dependencies of each of its direct dependency
        return direct_deps | union(
                resolve_dependencies_recursively(pkg, stack | {package})
                for pkg in direct_deps
            )

    bin_deps_rec_cache = {}
    def get_bin_deps(pkg):
        """Cached build dependency resolver."""

        if pkg not in bin_deps_rec_cache:
            bin_deps_rec_cache[pkg] = resolve_dependencies_recursively(pkg, frozenset())
        return bin_deps_rec_cache[pkg]

    # Construct { source package : its binary dependencies } map
    build_deps_rec = {
            srcpkg : union(
                get_bin_deps(binpkg) | {binpkg}
                for binpkg in build_deps[srcpkg]
                )
            for srcpkg in build_deps
        }

    # Compute { binary package : source package providing it } map
    bin_inverse = dict()
    for srcpkg, binpkgs in binary_map.iteritems():
        for binpkg in binpkgs:
            bin_inverse[binpkg] = srcpkg

    # Construct { source package : source packages providing its binary dependencies } map
    # The values are filtered to contain only source packages we actually intend building
    build_deps_src = {
            srcpkg : frozenset(
                bin_inverse[binpkg]
                for binpkg in build_deps_rec[srcpkg]
            ) & sources
            for srcpkg in sources
        }

    # Order has (package, packages to build before it) format
    order = OrderedDict()
    working_set = [
            (source, build_deps_src[source])
            for source in sources
        ]
    # Heuristic: sort packages by the number of dependencies
    working_set.sort(key=lambda (a,b): (len(b),a))

    prev = -1
    # Order resolution: in each pass, we move only those elements of working set
    # into the order which have their dependency already in order.
    # If during the pass nothing is added, we consider resolution failed.
    while working_set:
        if len(working_set) == prev:
            raise BuildError("Unable to resolve build dependencies")
        prev = len(working_set)

        for pkg, deps in working_set[:]:
            if all(dep in order for dep in deps):
                order[pkg] = deps
                working_set.remove( (pkg, deps) )
            else:
                pass

    return order

def resolve_build_order(distro, build_targets, arches, bindep_distro=None):
    """Given the distribution and a list of build targets in it,
    attempts to construct a list of tuples of format
    (source_name, architecture) """

    # List all source packages
    sources = union(
            {target for target in build_targets[arch]}
            for arch in arches
        )

    # Create source -> provided binaries map
    binary_map = {
            srcpkg : frozenset(distro.sources[srcpkg].binaries)
            for srcpkg in distro.sources
        }

    # Construct list of all known binaries
    target_binaries = union(binary_map.values())

    def simplify_deps(deps):
        """
        Flatten the complicated structure of package build-dependencies.  It takes
        the horrible dict python-debian gives us and returns the set of all possible
        dependencies.
        """

        # Note that here we transform "a or b" dependencies into "a and b"
        # This might break in some cases, but in general provides us legitimate
        # way to create a build ordering
        names = union({d["name"] for d in dep} for dep in deps)
        # Filter names to contain only known binaries
        return names & target_binaries

    # Construct { source package : direct binary dependencies } map
    build_deps = {
            srcpkg : simplify_deps(
                distro.sources[srcpkg].relations['build-depends']
            )
            for srcpkg in distro.sources
        }
    # Construct { binary package : direct binary dependencies } map
    bin_deps = {
            binpkg : union(
                simplify_deps(distro.binaries[binpkg][arch].relations['depends'])
                for arch in distro.binaries[binpkg])
            for binpkg in distro.binaries
        }

    # Mix-in binary dependencies information from other distribution if specified
    if bindep_distro:
        # Construct { binary package : direct binary dependencies } map for extra deps
        extra = {
                binpkg : union(
                    simplify_deps(bindep_distro.binaries[binpkg][arch].relations['depends'])
                    for arch in bindep_distro.binaries[binpkg]
                )
                for binpkg in bindep_distro.binaries
            }
        # Merge two dependency maps
        extra.update(bin_deps)
        bin_deps = extra

    # Actually resolve everything
    return resolve_build_order_core(sources, binary_map, build_deps, bin_deps)

def create_build_schedule(distro, build_targets, arches, bindep_distro=None):
    """
    Given the distribution and a list of build targets in it, get the tuples
    of package build commands in an appropriate order.  The tuples are of form
    (repo, chroot, arch, source_name)
    """

    # If we are building -dev pocket, resolve build dependencies based on -proposed as well
    if not bindep_distro and distro.pocket == 'development':
        _, bindep_distro, _ = dabuildsys.get_release(distro.release)

    order = resolve_build_order(distro, build_targets, arches, bindep_distro)
    schedule = OrderedDict()
    for package, deps in order.iteritems():
        for arch in arches:
            if package in build_targets[arch]:
                key = (distro.name, distro.release, package, deps)
                if key not in schedule:
                    schedule[key] = list()
                schedule[key].append(arch)

    for archlist in schedule.values():
        if 'all' in archlist:
            archlist.remove('all')
            if all_arch not in archlist:
                archlist.append(all_arch)

    return [k + (v,) for k, v in schedule.iteritems()]

def main():
    argparser = argparse.ArgumentParser(description="Build source packages from given repository which need building")
    argparser.add_argument("repository", help="Specifier of the repository which needs to be built")
    argparser.add_argument("architecture", nargs='*', help="List of architectures to build for")
    argparser.add_argument("--on-production-repository", action="store_true", help="Build even if non-development repository is specified")
    argparser.add_argument("--bindep-base", "-B", help="Release on which binary dependency resolution is based")
    argparser.add_argument("--keep-going", "-k", action="store_true", help="Continue building on errors")
    args = argparser.parse_args()

    repos = [args.repository]
    arches = args.architecture
    unknown_arches = set(arches) - set(dabuildsys.arches)
    if unknown_arches:
        raise BuildError("Unknown architectures: " + ', '.join(unknown_arches))
    if all_arch in arches:
        arches.insert(0, 'all')

    if repos == ['all']:
        repos = [release + '-development' for release in dabuildsys.releases]
    if not all(repo.endswith('-development') or repo.endswith('-bleeding') or repo.endswith('-staging') for repo in repos):
        if args.on_production_repository:
            print "WARNING: some of the repositories here are production repositories"
        else:
            raise BuildError("Attempting to run dabuild on a production repository")

    print "Checking the out-of-date packages in following suites: " + ", ".join(repos)
    build_targets = {}
    distros = {}
    distro_arches = {}
    for repo in repos:
        distros[repo] = dabuildsys.APTDistribution(repo)
        distro_arches[repo] = arches if arches else (['all'] + dabuildsys.release_arches[distros[repo].release])
        build_targets[repo] = { arch : list() for arch in distro_arches[repo] }
        for arch in distro_arches[repo]:
            for package in distros[repo].out_of_date_binaries(arch):
                build_targets[repo][arch].append(package)

    print "Attempting to resolve the build order"
    build_list = []
    bindep_distro = dabuildsys.APTDistribution(args.bindep_base) if args.bindep_base else None
    for repo in repos:
        build_list += create_build_schedule(distros[repo], build_targets[repo], distro_arches[repo], bindep_distro)

    print "Resolved the order"
    print
    for repo, order in groupby(build_list, lambda x: x[0]):
        print "Order in %s:" % repo
        for _, _, package, _, arches in order:
            print "* %s %s [%s]" % (package, distros[repo].sources[package].version, ' '.join(arches))
        print

    sys.stdout.write("Go ahead with the build? [y/N] ")
    sys.stdout.flush()
    answer = raw_input().lower().strip()
    if answer not in {'y', 'yes'}:
        return

    failures = OrderedDict()

    for repo, release, source_name, deps, pkg_arches in build_list:
        previous_failures = [dep for dep in deps if (repo, release, dep) in failures]
        if previous_failures:
            print "Skipping %s in %s due to previous failure of %s" % \
                (source_name, repo, ' '.join(previous_failures))
            failures[repo, release, source_name] = pkg_arches[:]
            continue

        for arch in pkg_arches:
            try:
                build_package(distros[repo], release, distros[repo].sources[source_name], arch)
            except subprocess.CalledProcessError if args.keep_going else () as err:
                print "FAILED"
                print err.output
                failures.setdefault((repo, release, source_name), []).append(arch)

    if failures:
        for repo, release, source_name in failures:
            print "Build failed: %s in %s" % (source_name, repo)
        raise RuntimeError("some builds failed")

if __name__ == '__main__':
    if not dabuildsys.claim_lock():
        print >>sys.stderr, "The lock is in place; unable to proceed"
        sys.exit(1)
    try:
        try:
            main()
        except subprocess.CalledProcessError as err:
            print err.output
            raise
    finally:
        dabuildsys.release_lock()