mixin-update

#!/usr/bin/env python

import argparse
import ConfigParser
import sys
import os
import re
import argparse
import collections
sys.path.insert(1,'external/pystache')
import pystache
pystache.defaults.MISSING_TAGS = 'strict'

# We're assuming '#' is valid to start a comment in all the configs we are amending
# which so far is true.

template = {
None: {
"header" : "# ----------------- BEGIN MIX-IN DEFINITIONS -----------------\n",
"footer" : "# ------------------ END MIX-IN DEFINITIONS ------------------\n",
"source" : "# Source: {}\n",
"line"   : "##############################################################\n",
"warn"   : "# Mix-In definitions are auto-generated by {}\n"
},
"xml" : {
"header" : "<!-- ############# BEGIN MIX-IN DEFINITIONS ############## -->\n",
"footer" : "<!-- ############## END MIX-IN DEFINITIONS ############### -->\n",
"source" : "<!-- Source: {} -->\n",
"line"   : "<!-- ##################################################### -->\n",
"warn"   : "<!-- Mix-In definitions are auto-generated by {} -->\n"
}}


# These are the set of product configuration files that are modified by mixins.
# If they are named something else in the actual product directory, the mixin spec
# file should setup the mapping in the "mapping" section
_FILE_LIST = ["BoardConfig.mk", "init.rc", "init.recovery.rc", "fstab", "fstab.recovery", "product.mk",
              "ueventd.rc", "AndroidBoard.mk", "gpt.ini"]

_PRODUCT_SPEC_FN = "mixins.spec"
_GROUP_SPEC_FN = "mixinfo.spec"
_OPTION_SPEC_FN = "option.spec"
_FILES_SPEC_FN = "files.spec"
_XML_SPEC_FN = "xml.spec"

abort_on_errors = True
policy_errors_found = False


class ReadTrackingDict(dict):
    def __init__(self, *args, **kwargs):
        super(ReadTrackingDict, self).__init__(*args, **kwargs)
        self.reads = collections.defaultdict(int)
    def __getitem__(self, key):
        self.reads[key] += 1
        return super(ReadTrackingDict, self).__getitem__(key)
    def unused_keys(self):
        return list(set(self.keys()) - set(self.reads.keys()))


def warning(s):
    sys.stderr.write(s + "\n")
    sys.stderr.flush()


def policy_error(s):
    global policy_errors_found
    sys.stderr.write("ERROR: ")
    warning(s)
    if abort_on_errors:
        sys.exit(1)
    else:
        policy_errors_found = True


def read_spec_file(specfile, cp=None):
    """read a mixin spec file and return a ConfigParser object with its
    definitions. If a ConfigParser is supplied as an argument, it will
    be augmented with the new data"""
    if not cp:
        cp = ConfigParser.SafeConfigParser()

    try:
        with open(specfile) as fp:
            cp.readfp(fp)
    except IOError:
        if os.path.islink(specfile):
            policy_error("reading specfile {} which is a symlink,"
                         " maybe it is broken".format(specfile))
        else:
            policy_error("reading specfile {}".format(specfile))

    return cp


def read_mixin_tree(basedir, mixin_tree=None):
    """return a dictionary mapping mixin groups found in a particular
    basedir with a dictionary mapping option names to the directory
    containing their configuration fragments as well as other group-level
    metadata"""

    if not mixin_tree:
        mixin_tree = {}

    groups = [i for i in os.listdir(basedir) if not i.startswith(".")]
    for group in groups:
        assert group not in mixin_tree
        mixin_tree[group] = {}
        groupdir = os.path.join(basedir, group)
        mixin_tree[group]["groupdir"] = groupdir
        mixin_tree[group]["options"] = {}
        mixin_tree[group]["deps"] = set()

        # Check for a mixinfo file in the root of the group directory
        # This is for metadata about the group as a whole. Its presence
        # is optional, many groups won't need it.
        mixinfo = os.path.join(groupdir, _GROUP_SPEC_FN)
        if os.path.exists(mixinfo):
            cp = read_spec_file(mixinfo)

            # "mixinfo.deps" is the set of groups which must be inherited
            # prior to inheriting this mixin, typically because we need the
            # other groups to define certain variables for us
            if cp.has_option("mixinfo", "deps"):
                mixin_tree[group]["deps"] = set(cp.get("mixinfo", "deps").split())

        options = [i for i in os.listdir(groupdir) if not i.startswith(".")]
        for option in options:
            assert option not in mixin_tree[group]["options"]

            mixin_tree[group]["options"][option] = {}
            cur_opt = mixin_tree[group]["options"][option]

            opt_dir = os.path.join(groupdir, option)
            if os.path.isdir(opt_dir):
                cur_opt["listdir"] = os.listdir(opt_dir)
            cur_opt["optiondir"] = opt_dir

            cur_opt["deps"] = set()
            cur_opt["defaults"] = {}

            optioninfo = os.path.join(cur_opt["optiondir"], _OPTION_SPEC_FN)
            if option == "default" and os.path.islink(cur_opt["optiondir"]):
                cur_opt["realname"] = os.path.basename(os.path.realpath(cur_opt["optiondir"]))
            else:
                cur_opt["realname"] = option

            if os.path.exists(optioninfo):
                cp = read_spec_file(optioninfo)

                # Options may define their own dependencies just like at the
                # group-level
                if cp.has_option("mixinfo", "deps"):
                    cur_opt["deps"] = set(cp.get("mixinfo", "deps").split())

                if cp.has_section("defaults"):
                    for k, v in cp.items("defaults"):
                        cur_opt["defaults"][k] = v

    return mixin_tree


def get_deps(mixin_tree, group, option):
    return list(mixin_tree[group]["deps"] | mixin_tree[group]["options"][option]["deps"])


def sort_selections(selections, mixin_tree, verbose):
    """sort a group selection based on their dependencies.  Parent groups
    are moved before their childs."""
    sorted_selections = []

    # Re-order the selections so that the "parent" groups appear
    # before their childs. This sort algorithm is stable.
    while selections:
        (group, option, params) = selections.pop(0)
        deps = get_deps(mixin_tree, group, option)
        missing_parents = []
        while deps:
            dep = deps.pop()
            parent_group = [x for x in selections if x[0] == dep]
            if not parent_group:
                continue
            parent_group = parent_group[0]
            missing_parents.append(parent_group)
            for d in get_deps(mixin_tree, parent_group[0], parent_group[1]):
                if d not in deps:
                    deps.append(d)

        while missing_parents:
            parent = missing_parents.pop()
            if verbose:
                print "moving %s before %s" % (parent[0], group)
            if parent not in sorted_selections:
                sorted_selections.append(parent)
            if parent in selections:
                selections.remove(parent)

        sorted_selections.append((group, option, params))

    return sorted_selections


def validate_selections(selections, mixin_tree, verbose=False):
    """enforce that the set of mixin selections is sane by checking
    the following:
    1) For each group selected, verify that the group exists and that
    the particular selection made within that group also exists
    2) If any groups exist for which there is no selection made, make
    the default selection for that group. If no default exists, report an
    error.
    The selections list may be altered by this function to include
    group default selections"""

    groups_seen = []
    returned_selections = []
    default_groups = []

    # The spec file may have omitted some groups. If so, pull in their
    # default options, or generate an error if there is no default
    selected = []
    for (group, option, params) in selections:
        selected.append(group)
    unspecified_groups = sorted(set(mixin_tree.keys()) - set(selected))
    for group in unspecified_groups:
        default_groups.append(group)
        if "default" not in mixin_tree[group]["options"]:
            if verbose:
                policy_error("group {} doesn't have a default option!".format(group))
        else:
            params = mixin_tree[group]["options"]["default"]["defaults"]
            coerce_boolean(params)
            selections.append((group, "default", ReadTrackingDict(params)))

    selections = sort_selections(selections, mixin_tree, verbose)

    for (group, option, params) in selections:
        if group in groups_seen:
            policy_error("selection already made for group {}".format(group))
            continue

        groups_seen.append(group)

        if group not in mixin_tree:
            policy_error("no definition found for group {}".format(group))
            continue

        if option not in mixin_tree[group]["options"]:
            policy_error("unknown option {} for group {}".format(option, group))
            continue

        deps = get_deps(mixin_tree, group, option)
        for dep in deps:
            if dep not in groups_seen:
                policy_error("group {} option {} requires that group {} be selected first".format(group, option, dep))

        defaults = mixin_tree[group]["options"][option]["defaults"]
        for k, v in defaults.iteritems():
            if k not in params:
                params[k] = v

        # Inherit the dependency parameters
        for dep_group, dep_option, dep_params in returned_selections:
            if dep_group in deps:
                dep_params[group] = mixin_tree[group]["options"][option]["realname"]
                coerce_boolean(dep_params)
                for k, v in dep_params.iteritems():
                    if k not in params:
                        params[k] = v
                params[dep_group] = mixin_tree[dep_group]["options"][dep_option]["realname"]

        coerce_boolean(params)
        returned_selections.append((group, option, params))

    return returned_selections


def clear_file(dest):
    """Return of list of string lines based on the destination file.
    Clear out any existing mixin defintions from the specified file; after this
    is done only the header/footer will remain. If the dest file never had anything
    in it, add the header/footer"""
    output = []
    in_mixin = False
    ever_in_mixin = False
    ftype = get_file_type(dest)

    if os.path.exists(dest):
        with open(dest) as dfile:
            dlines = dfile.readlines()
    else:
        dlines = []

    orig_dlines = dlines[:]

    for line in dlines:
        if not in_mixin:
            output.append(line)
            if line == template[ftype]["header"]:
                in_mixin = True
                ever_in_mixin = True
                output.append(template[ftype]["warn"].format(os.path.basename(sys.argv[0])))
        else:
            if line == template[ftype]["footer"]:
                output.append(line)
                in_mixin = False

    if in_mixin:
        # header with no footer? ok whatever
        output.append(template[ftype]["footer"])

    if not ever_in_mixin:
        output.append(template[ftype]["header"])
        output.append(template[ftype]["warn"].format(os.path.basename(sys.argv[0])))
        output.append(template[ftype]["footer"])

    return orig_dlines, output

warn_vars = [
        "ADDITIONAL_BUILD_PROPERTIES",
        "ADDITIONAL_DEFAULT_PROPERTIES",
        "BOARD_KERNEL_CMDLINE",
        "DEVICE_PACKAGE_OVERLAYS",
        "PRODUCT_COPY_FILES",
        "PRODUCT_DEFAULT_PROPERTY_OVERRIDES",
        "PRODUCT_PACKAGES",
        "PRODUCT_PACKAGES_DEBUG",
        "PRODUCT_PACKAGES_ENG",
        "PRODUCT_PACKAGES_TESTS",
        "PRODUCT_PACKAGE_OVERLAYS",
        "PRODUCT_PROPERTY_OVERRIDES",
        ]


def get_file_type(fname):
    if fname.endswith(".xml"):
        return "xml"
    return None


def amend_file(dlines, src, mixinsbase, params):
    """Augment the destination lines list with data from the source file provided.
    Assumes we have run clear_file() on dlines at some point beforehand"""
    with open(src) as sfile:
        src_contents = sfile.read()
        if src_contents[-1] != '\n':
            policy_error("No Newline at the end of {}".format(src))

        try:
            slines = pystache.render(src_contents, params)
        except pystache.context.KeyNotFoundError as e:
            policy_error("{} depends on undefined mixin parameter '{}'".format(src, e.key))

    # sanity checks
    if slines.find(mixinsbase) >= 0:
        policy_error("build-time references to paths inside mixin directory are not allowed; {} is invalid".format(src))

    # much faster than previous regexp (\w*)
    m = re.findall('(' + '|'.join(warn_vars) + ')\s*:=', slines)
    if m:
        warning("Non-accumulative assignment to '{}' found in {}".format(m[0], src))

    ftype = get_file_type(src)
    idx = dlines.index(template[ftype]["footer"])
    output = dlines[:idx]
    output.insert(idx, template[ftype]["line"])
    output.insert(idx, template[ftype]["source"].format(src))
    output.insert(idx, template[ftype]["line"])
    output.extend(slines.splitlines(True))
    output.extend(dlines[idx:])
    return output


def coerce_boolean(d):
    for k in d:
        if type(d[k]) == str and d[k].lower() == 'false':
            d[k] = False


def split_params(selections):
    regx_val = re.compile(r"(?<!\\),")  # param's value can contain comma (escaped with backslash)
    res = []
    for group, option_params in selections:
        m = re.match('([^\s(]+)\s*\(([^)]*)\)\s*$', option_params)
        if m is not None:
            option = m.group(1)
            params = dict(map(str.strip, x.replace('\,',',').split('=', 1)) for x in regx_val.split(m.group(2)))
            coerce_boolean(params)
            res.append((group, option, ReadTrackingDict(params)))
        else:
            res.append((group, option_params, ReadTrackingDict()))
    return res


def check_section(section, cp, specfile):
    if not cp.has_section(section):
        policy_error('specfile missing {} section: {}'.format(section, specfile))
        return False
    return True


def extrafiles_from_selections(basedir, selections):

    extrafiles = []
    for (group, option, params) in selections:
        groupdir = os.path.join(basedir, group)

        # Check for a files.spec file in the mixin directory.
        # The section [extrafiles] can be used to expand the
        # _FILE_LIST
        specfile = os.path.join(groupdir, option, _FILES_SPEC_FN)
        if os.path.isfile(specfile) or os.path.islink(specfile):
            cp = read_spec_file(specfile)
            # append all files found in the section avoiding duplicate
            # entries
            if cp.has_section("extrafiles"):
                for newfile, comment in cp.items("extrafiles"):
                    if newfile not in extrafiles + _FILE_LIST:
                        extrafiles.append(newfile)

    return extrafiles


def handle_include(specfile):
    """ check for include section in a spec file.
    handle multiple include by recursion.
    return the list of file in the order they must be parsed."""

    cp = read_spec_file(specfile)

    if cp.has_section("include"):
        relative_path = cp.get("include", "file")
        include_path = os.path.join(os.path.dirname(specfile), relative_path)
        return handle_include(include_path) + [specfile]
    return [specfile]


def process_spec_file(specfile, dry_run, mixin_tree):

    filelist = handle_include(specfile)
    cp = None
    for f in filelist:
        cp = read_spec_file(f, cp)

    if False in (
            check_section("main", cp, specfile),
            check_section("groups", cp, specfile)
            ):
        return

    basedir = cp.get("main", "mixinsdir")
    selections = split_params(cp.items("groups"))

    if not dry_run:
        print("processing {}".format(specfile))

    # After this, all default selections should be populated
    # and the selections should be sane
    selections = validate_selections(selections, mixin_tree, not dry_run)

    product_dir = os.path.dirname(specfile)
    file_map = {}
    retval = True
    extrafiles = extrafiles_from_selections(basedir, selections)

    # If a file is in [extrafiles] section of mixins.spec
    # but not already in _FILE_LIST or extrafiles, add it to extrafile list
    if cp.has_section("extrafiles"):
        for newfile, comment in cp.items("extrafiles"):
            if newfile not in extrafiles + _FILE_LIST:
                extrafiles.append(newfile)

    # Set up the file map since the config files in the product directory
    # may have slightly different names than what is in _FILE_LIST
    for fname in _FILE_LIST + extrafiles:
        if cp.has_option("mapping", fname):
            file_map[fname] = cp.get("mapping", fname).split()
        else:
            file_map[fname] = [fname]

    for src, dests in file_map.iteritems():
        for dest in dests:
            dest_fn = os.path.join(product_dir, dest)

            orig_dest_lines, dest_lines = clear_file(dest_fn)

            # Now check all the groups to see if they have a configuration
            # fragment to insert into the destination file
            for group, option, params in selections:
                cur_opt = mixin_tree[group]["options"][option]

                optdir = cur_opt["optiondir"]

                # Any given file can have multiple fragments. We first
                # look for <frag>.1, <frag>.2, ... <frag>.9, <frag>
                # This is useful for when several mixin options have mostly
                # the same data except for maybe a few lines; you can
                # avoid copypasting a lot of stuff by using this feature
                # and symbolic links for the common bits.
                dsrc, fsrc = os.path.split(src)
                flist = [f for f in cur_opt["listdir"] if f.startswith(fsrc) and (dsrc in optdir)]
                fraglist = [f for f in flist if len(f) == len(fsrc) + 2 and f[-2] == '.' and f[-1].isdigit()]
                if fraglist:
                    fraglist.sort()
                    for f in fraglist:
                        dest_lines = amend_file(dest_lines, os.path.join(optdir, f), basedir, params)
                if fsrc in flist:
                    dest_lines = amend_file(dest_lines, os.path.join(optdir, fsrc), basedir, params)

            if dry_run:
                if cmp(dest_lines, orig_dest_lines) != 0:
                    warning("{} is out of date".format(dest_fn))
                    retval = False
            else:
                if cmp(dest_lines, orig_dest_lines) != 0:
                    print("updating {}".format(dest_fn))
                    with open(dest_fn, "w") as fp:
                        fp.writelines(dest_lines)

    for group, option, params in selections:
        unused = params.unused_keys()
        if unused:
            policy_error('Unnecessary parameters {} given for mixin option'
                         ' "{}: {}"'.format(unused, group, option))
    return retval


def find_all_spec_files(basepath):
    """find all the mixin spec files underneath the specified
    directory (typically device/) and return a list of paths to
    them"""
    ret = []
    for root, dirs, files in os.walk(basepath):
        if "mixins.spec" in files:
            ret.append(os.path.join(root, _PRODUCT_SPEC_FN))
    return ret


def main(dry_run=False, sfiles=None, warn_only=False):
    global abort_on_errors

    ret = 0
    abort_on_errors = not warn_only
    out_of_sync = False

    if not sfiles:
        sfiles = find_all_spec_files("device/intel/project-celadon")

    mixin_tree = read_mixin_tree('device/intel/mixins/groups')
    for sf in sfiles:
        if not process_spec_file(sf, dry_run, mixin_tree):
            out_of_sync = True

    if out_of_sync:
        warning("Board configs are out of sync with mixins, please run mixin-update")
        ret = 3

    if policy_errors_found:
        warning("Some spec files have policy issues")
        ret = 2

    return ret


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Update board configurations with mixin data based on spec file")
    parser.add_argument("-d", "--dry-run",
            help="Don't make any actual changes, exit nonzero if something needs to be updated",
            action="store_true")
    parser.add_argument("-s", "--spec",
            help="Read a specific spec file. Can be called multiple times. Defaults to scanning the tree under device/intel/ for spec files",
            action="append")
    parser.add_argument("-w", "--warn-only",
            help="Generate warnings instead of fatal errors for spec file policy violations",
            action="store_true")

    args = parser.parse_args()

    ret = main(args.dry_run, args.spec, args.warn_only)
    sys.exit(ret)