squad-local-bisect

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: set ts=4
#
# Copyright 2022-present Linaro Limited
#
# SPDX-License-Identifier: MIT


import argparse
import logging
import os
import re
import sys
from pathlib import Path

import requests
from squad_client.core.api import SquadApi
from squad_client.core.models import Build, Squad, TestRun
from squad_client.shortcuts import get_build
from squad_client.utils import first, getid

squad_host_url = "https://qa-reports.linaro.org/"
SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url))

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def get_file(path):
    print(f"Getting file from {path}")
    if re.search(r"https?://", path):
        request = requests.get(path, allow_redirects=True)
        request.raise_for_status()
        filename = path.split("/")[-1]
        with open(filename, "wb") as f:
            f.write(request.content)
        return filename
    elif os.path.exists(path):
        return path
    else:
        raise Exception(f"Path {path} not found")


def find_good_build(
    base_build, project, environment, build_name, suite_name, test_name
):
    builds = project.builds(id__lt=base_build.id, ordering="-id", count=10).values()
    for build in builds:
        logger.debug(f'Trying to find good test in build "{build.version}"', flush=True)
        for testrun in build.testruns(environment=environment.id).values():
            logger.debug(
                f"  - Trying to find {build_name} in {testrun.job_url}", flush=True
            )
            if build_name == testrun.metadata.build_name:
                logger.debug(
                    f"    - Yay, found it, now looking for a passing {suite_name}/{test_name}",
                    flush=True,
                )
                candidate_test = first(
                    testrun.tests(metadata__suite=suite_name, metadata__name=test_name)
                )
                if candidate_test is None:
                    logger.debug("      - no test in here :(", flush=True)
                    continue
                if candidate_test.result:
                    logger.debug("************** FOUND IT *************", flush=True)
                    return build
    return None


def parse_args():
    parser = argparse.ArgumentParser(
        description="Get a local bisect script within SQUAD"
    )

    parser.add_argument(
        "--group",
        required=True,
        help="squad group",
    )

    parser.add_argument(
        "--project",
        required=True,
        help="squad project",
    )

    parser.add_argument(
        "--build",
        required=True,
        help="squad build",
    )

    parser.add_argument(
        "--fail",
        required=True,
        help="Failing string in the format 'device/build_name/suite_name/test_name'.",
    )

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="Display debug messages",
    )

    return parser.parse_args()


def run():
    args = parse_args()
    if args.debug:
        logger.setLevel(level=logging.DEBUG)

    base_group = Squad().group(args.group)
    if base_group is None:
        logger.error(f"Get group failed. Group not found: '{args.group}'.")
        return -1

    base_project = base_group.project(args.project)
    if base_project is None:
        logger.error(f"Get project failed. project not found: '{args.project}'.")
        return -1

    base_build = get_build(args.build, base_project)
    if base_build is None:
        logger.error(f"Get build failed. build not found: '{args.build}'.")
        return -1

    device_name, build_name, suite_name, test_name = args.fail.split("/")
    metadata = first(
        Squad().suitemetadata(suite=suite_name, name=test_name, kind="test")
    )
    if metadata is None:
        print('There is no test named "{suite_name}/{test_name}"')
        return -1

    environment = base_project.environment(device_name)
    build = Build(base_build.id)
    test = first(build.tests(metadata=metadata.id, environment=environment.id))
    if test is None:
        print(
            f'Build "{build.version}" has no test "{suite_name}/{test_name}" available on "{device_name}"'
        )
        return -1

    # In theory there should only be one of those
    testrun = TestRun(getid(test.test_run))
    logger.debug(f"Testrun id {testrun.id}")
    download_url = testrun.metadata.download_url
    if download_url is None:
        if testrun.metadata.config is None:
            print("There is no way to determine download_url")
            return -1
        download_url = testrun.metadata.config.replace("config", "")

    tuxmake = get_file(f"{download_url}tuxmake_reproducer.sh")
    build_cmdline = ""
    for line in Path(tuxmake).read_text(encoding="utf-8").split("\n"):
        logger.debug(line)
        if "# tuxmake --runtime" in line:
            line = re.sub(r"^# ", "", line)
            build_cmdline = os.path.join(build_cmdline + line).strip()

    good_build = None
    if suite_name in ["boot", "build"]:
        suite = base_project.suite(suite_name)
        print("Trying to find good tests")
        good_tests = Squad().tests(
            build__id__lt=base_build.id,
            suite__id=suite.id,
            environment__id=environment.id,
            metadata__name=build_name,
            result=True,
            ordering="-build_id",
            count=1,
        )
        if len(good_tests) > 0:
            good = first(good_tests)
            good_build = Build(getid(good.build))

    else:
        good_build = find_good_build(
            build, base_project, environment, build_name, suite_name, test_name
        )

    if good_build is None:
        print("No good build found")
        return

    if suite_name not in "build":
        tuxrun = get_file(f"{testrun.job_url}/reproducer")
        for line in Path(tuxrun).read_text(encoding="utf-8").split("\n"):
            if "tuxrun --runtime" in line:
                line = re.sub(r"--kernel \S+ ", "", line)
                line = re.sub(r"--modules \S+ ", "", line)
                if suite_name in "boot" or "ltp-" in suite_name:
                    line = re.sub(r"--tests \S+ ", "", line)
                build_cmdline = os.path.join(
                    build_cmdline
                    + ' --results-hook "'
                    + line.strip()
                    + ' --tuxmake ./ --save-outputs --log-file -"'
                ).strip()

        if "ltp-" in suite_name:
            build_cmdline = build_cmdline.replace(
                '-"', f"- -- 'cd /opt/ltp && ./runltp -s {test_name}'\""
            )
    print(f"Found good test on build {good_build.version}")

    bisect_script = f"""#!/bin/bash
# Set GOOD_SHA
bad_sha="{build.version}"
good_sha=$(git merge-base {build.version} {good_build.version})
git checkout $good_sha
{build_cmdline}
if [[ $? -eq 0 ]]; then
    git bisect start $bad_sha $good_sha && git bisect run {build_cmdline}
else
    echo "merge base $good_sha didn't work either aborting."
fi
"""

    filename = f"bisect-badsha-{build.version}-{device_name}-{build_name}-{suite_name}-{test_name}.sh"

    Path(filename).write_text(bisect_script, encoding="utf-8")

    print(f"{bisect_script}")
    print(f"file created: {filename}")


if __name__ == "__main__":
    sys.exit(run())