squad-compare-compat

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: set ts=4
#
# Copyright 2024-present Linaro Limited
#
# SPDX-License-Identifier: MIT


import argparse
import logging
import os
import sys
from urllib import request

import yaml
from squad_client.core.api import SquadApi
from squad_client.core.models import Squad
from squad_client.shortcuts import download_tests as download
from squad_client.shortcuts import get_build

squad_host_url = "https://qa-reports.linaro.org/"
SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url))

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()


def parse_args():
    parser = argparse.ArgumentParser(description="Compare builds within SQUAD")

    parser.add_argument(
        "--gpb",
        required=True,
        action="append",
        nargs=3,
        help="squad group,project,build",
    )

    parser.add_argument(
        "--environments", help="Filter on environments (separated by ',')"
    )

    parser.add_argument("--suites", help="Filter on suites (separated by ',')")

    parser.add_argument(
        "--filename", help="Name of the output file where results will be written"
    )

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="Display debug messages",
    )

    return parser.parse_args()


def download_tests(project, build, environments, suites, output_filename):
    download(
        project,
        build,
        environments,
        suites,
        "{test.environment.slug}/{test.test_run.metadata.build_name}/{test.name} {test.status} {test.test_run.log_file}",
        output_filename,
    )


def run():
    args = parse_args()
    if args.debug:
        logger.setLevel(level=logging.DEBUG)

    known_issue_files = "kselftests-production.yaml kvm-unit-tests.yaml libhugetlbfs-production.yaml ltp-production.yaml network-basic-tests.yaml packetdrill-tests.yaml perf.yaml spectre-meltdown-checker.yaml v4l2-compliance.yaml"
    known_issues_patterns = []
    group_name, project_name, build_name = args.gpb[0]
    for file in known_issue_files.split():

        x = request.urlopen(
            f"https://raw.githubusercontent.com/Linaro/qa-reports-known-issues/master/{file}"
        )
        issues = yaml.safe_load(x)
        for tmp in issues["projects"][0]["known_issues"]:
            environments = []
            projects = []
            for key, value in tmp.items():
                if "environments" in tmp.keys():
                    environments = tmp["environments"]
                else:
                    for matrix in tmp["matrix_apply"]:
                        environments.append(matrix["environments"])

                if "projects" in tmp.keys():
                    projects = tmp["projects"]
                else:
                    for matrix in tmp["matrix_apply"]:
                        projects.append(matrix["projects"])
                if "test_names" in key:
                    for argh in value:
                        if f"{group_name}/{project_name}" in projects:
                            for env in environments:
                                known_issues_patterns.append(
                                    f"{env}/*{argh.replace('.', '?')}"
                                )
                elif "test_name" in key:
                    if f"{group_name}/{project_name}" in projects:
                        for env in environments:
                            known_issues_patterns.append(
                                f"{env}/*{value.replace('.', '?')}"
                            )

    table_filename = "table-compat.txt"
    headings = "64-bit native | compat | 32-bit native | build_version/device/build_name/testsuite/test\n"
    headings = f"{headings}----------------------------------------------------------------------------\n"
    with open(table_filename, "w") as fp:
        fp.write(headings)
        for group_name, project_name, build_name in args.gpb:
            group = Squad().group(group_name)
            project = group.project(project_name)
            build = get_build(build_name, project)

            environments = None
            if args.environments:
                environments = [
                    project.environment(e) for e in args.environments.split(",")
                ]

            suites = None
            if args.suites:
                suites = []
                for s in args.suites.split(","):
                    suites += project.suites(slug__startswith=s).values()

            file = f"{group.slug}-{project.slug}-{build.version}".replace("~", "")
            file_with_ending = os.path.join(file + ".txt")
            download_tests(project, build, environments, suites, file_with_ending)
            file_open = open(file_with_ending, "r")
            file_lines = file_open.readlines()

            compatlist = {}
            native32list = {}
            for line in file_lines:
                test_name, test_result, logfile = line.split()
                device_name, build_name = test_name.split("/")[0:2]
                test_name = "/".join(test_name.split("/")[1:])

                if build_name.endswith("-compat"):
                    compatlist[test_name.replace("-compat", "")] = test_result
                elif device_name in ["qemu-armv7", "x15", "qemu-i386", "i386"]:
                    native32list[test_name] = test_result

            for line in file_lines:
                test_name, test_result, logfile = line.split()
                device_name, build_name = test_name.split("/")[0:2]
                test_name = "/".join(test_name.split("/")[1:])
                if test_name in compatlist.keys() and device_name not in [
                    "qemu-armv7",
                    "x15",
                    "qemu-i386",
                    "i386",
                ]:
                    if test_name not in native32list.keys():
                        native32list[test_name] = None
                    if (
                        test_result != compatlist[test_name]
                        or test_result != native32list[test_name]
                    ):
                        fp.write(
                            f"{test_result} | {compatlist[test_name]} | {native32list[test_name]} : {build.version}/{device_name}/{test_name}\n"
                        )

        logger.debug(f"group: {group}, project: {project}, build: {build}")


if __name__ == "__main__":
    sys.exit(run())