diff --git a/taskcluster/android-components-projects.json b/taskcluster/android-components-projects.json new file mode 100644 index 0000000000..fa5b44deb5 --- /dev/null +++ b/taskcluster/android-components-projects.json @@ -0,0 +1 @@ +["compose-awesomebar", "compose-browser-toolbar", "compose-engine", "compose-tabstray", "concept-awesomebar", "concept-base", "concept-fetch", "concept-toolbar", "concept-tabstray", "concept-engine", "concept-menu", "concept-push", "concept-storage", "concept-sync", "feature-autofill", "feature-awesomebar", "feature-accounts", "feature-accounts-push", "feature-app-links", "feature-containers", "feature-contextmenu", "feature-customtabs", "feature-intent", "feature-media", "feature-logins", "feature-webnotifications", "feature-readerview", "feature-search", "feature-serviceworker", "feature-session", "feature-tabs", "feature-tab-collections", "feature-recentlyclosed", "feature-toolbar", "feature-top-sites", "feature-share", "feature-downloads", "feature-privatemode", "feature-prompts", "feature-push", "feature-pwa", "feature-findinpage", "feature-syncedtabs", "feature-sitepermissions", "feature-qr", "feature-webauthn", "feature-webcompat", "feature-webcompat-reporter", "feature-addons", "browser-domains", "browser-engine-gecko", "browser-engine-system", "browser-errorpages", "browser-icons", "browser-menu", "browser-menu2", "browser-session-storage", "browser-state", "browser-storage-sync", "browser-tabstray", "browser-thumbnails", "browser-toolbar", "ui-autocomplete", "ui-colors", "ui-widgets", "ui-fonts", "ui-icons", "ui-tabcounter", "service-firefox-accounts", "service-location", "service-sync-autofill", "service-sync-logins", "service-digitalassetlinks", "service-glean", "service-nimbus", "service-pocket", "service-contile", "support-base", "support-images", "support-ktx", "support-migration", "support-test", "support-test-fakes", "support-test-libstate", "support-android-test", "support-test-appservices", "support-utils", "support-sync-telemetry", "support-rustlog", "support-rusthttp", "support-locale", "support-webextensions", "lib-crash", "lib-crash-sentry", "lib-crash-sentry-legacy", "lib-dataprotect", "lib-fetch-httpurlconnection", "lib-fetch-okhttp", "lib-jexl", "lib-publicsuffixlist", "lib-push-firebase", "lib-state", "tooling-detekt", "tooling-lint", "tooling-fetch-tests", "samples-compose-browser", "samples-crash", "samples-firefox-accounts", "samples-glean", "samples-glean-library", "samples-sync", "samples-sync-logins", "samples-toolbar", "samples-dataprotect", "tooling-glean-gradle", "tooling-nimbus-gradle"] \ No newline at end of file diff --git a/taskcluster/app_services_taskgraph/__init__.py b/taskcluster/app_services_taskgraph/__init__.py index bb1eb396d0..d01f0f6d24 100644 --- a/taskcluster/app_services_taskgraph/__init__.py +++ b/taskcluster/app_services_taskgraph/__init__.py @@ -22,7 +22,9 @@ def register(graph_config): extend_parameters_schema({ Optional('branch-build'): { + Optional('android-components-owner'): str, Optional('android-components-branch'): str, + Optional('fenix-owner'): str, Optional('fenix-branch'): str, }, 'nightly-build': bool, diff --git a/taskcluster/app_services_taskgraph/branch_builds.py b/taskcluster/app_services_taskgraph/branch_builds.py index 0ef2b0de34..e11263c5b8 100644 --- a/taskcluster/app_services_taskgraph/branch_builds.py +++ b/taskcluster/app_services_taskgraph/branch_builds.py @@ -7,8 +7,9 @@ from taskgraph.filter_tasks import filter_task -ANDROID_COMPONENTS_BRANCH_RE = re.compile(r'\[ac:\s*([\w-]+)\]') -FENIX_BRANCH_RE = re.compile(r'\[fenix:\s*([\w-]+)\]') +REPO_RE = r'((?P[\.\w-]+)/)?(?P[\.\w-]+)' +ANDROID_COMPONENTS_BRANCH_RE = re.compile(r'\[a-?c:\s*' + REPO_RE + r'\]') +FENIX_BRANCH_RE = re.compile(r'\[fenix:\s*' + REPO_RE + r'\]') def update_decision_parameters(parameters): parameters['branch-build'] = calc_branch_build_param(parameters) @@ -25,14 +26,22 @@ def calc_branch_build_param(parameters): ac_branch_match = ANDROID_COMPONENTS_BRANCH_RE.search(title) if ac_branch_match: - branch_build['android-components-branch'] = ac_branch_match.group(1) + branch_build['android-components-owner'] = calc_owner(ac_branch_match) + branch_build['android-components-branch'] = ac_branch_match.group('branch') fenix_branch_match = FENIX_BRANCH_RE.search(title) if fenix_branch_match: - branch_build['fenix-branch'] = fenix_branch_match.group(1) + branch_build['fenix-owner'] = calc_owner(fenix_branch_match) + branch_build['fenix-branch'] = fenix_branch_match.group('branch') return branch_build +def calc_owner(match): + if match.group('owner'): + return match.group('owner') + else: + return 'mozilla-mobile' + @filter_task("branch-build") def filter_branch_build_tasks(full_task_graph, parameters, graph_config): if parameters.get('branch-build'): diff --git a/taskcluster/app_services_taskgraph/transforms/branch_build.py b/taskcluster/app_services_taskgraph/transforms/branch_build.py index 60b5472d6b..2b4bc056a3 100644 --- a/taskcluster/app_services_taskgraph/transforms/branch_build.py +++ b/taskcluster/app_services_taskgraph/transforms/branch_build.py @@ -3,11 +3,15 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from copy import deepcopy +import os +import json from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import validate_schema, Schema from voluptuous import Optional, Required, In +TASKCLUSTER_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + # Schema for the job dictionary from kinds.yml branch_build_schema = Schema({ # Which repository are we working on @@ -45,6 +49,7 @@ 'desktop-linux-libs', 'desktop-macos-libs', 'desktop-win32-x86-64-libs', + 'robolectric', 'rust', ], }, @@ -52,8 +57,6 @@ 'using': 'gradlew', 'pre-gradlew': [ ["git", "submodule", "update", "--init"], - ["source", "taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh"], - ["rsync", "-a", "/builds/worker/fetches/libs/", "/builds/worker/checkouts/vcs/libs/"], ] }, } @@ -71,80 +74,112 @@ def validate(config, tasks): @transforms.add def setup(config, tasks): branch_build_params = config.params.get('branch-build', {}) - android_components_branch = branch_build_params.get('android-components-branch', 'main') - fenix_branch = branch_build_params.get('fenix-branch', 'main') for task in tasks: repo_name = task.pop('repository') operation = task['name'] task.update(deepcopy(TASK_COMMON)) + task['description'] = '{} {}'.format(operation, repo_name) if repo_name == 'application-services': setup_application_services(task) elif repo_name == 'android-components': - setup_android_components(task, android_components_branch) + setup_android_components(task, branch_build_params) elif repo_name == 'fenix': - setup_fenix(task, android_components_branch, fenix_branch) + setup_fenix(task, branch_build_params) else: raise ValueError("Invalid branch build repository: {}".format(repo_name)) if operation == 'build': - setup_build(task, repo_name) + for task in get_build_tasks(task, repo_name): + yield task elif operation == 'test': - setup_test(task, repo_name) + for task in get_test_tasks(task, repo_name): + yield task else: raise ValueError("Invalid branch build operation: {}".format(operation)) - task['description'] = '{} {}'.format(operation, repo_name) - yield task def setup_application_services(task): task['run']['pre-gradlew'].extend([ + ["source", "taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh"], + ["rsync", "-a", "/builds/worker/fetches/libs/", "/builds/worker/checkouts/vcs/libs/"], ["taskcluster/scripts/setup-branch-build.py"], ]) + task['fetches'] = { + 'toolchain': [ + 'android-libs', + 'desktop-linux-libs', + 'desktop-macos-libs', + 'desktop-win32-x86-64-libs', + 'rust', + ], + } -def setup_android_components(task, android_components_branch): +def setup_android_components(task, branch_build_params): task['dependencies'] = { 'branch-build-as': 'branch-build-as-build', } - task['fetches']['branch-build-as'] = [ 'application-services-m2.tar.gz' ] + task['fetches'] = { + 'toolchain': [ + 'robolectric', + ], + 'branch-build-as': [ 'application-services-m2.tar.gz' ], + } task['run']['pre-gradlew'].extend([ ['rsync', '-a', '/builds/worker/fetches/.m2/', '/builds/worker/.m2/'], - [ - "taskcluster/scripts/setup-branch-build.py", - '--android-components', android_components_branch, - ], + setup_branch_build_command_line(branch_build_params, setup_fenix=False), ['cd', 'android-components'], + ['git', 'rev-parse', '--short', 'HEAD'], # Building this up-front seems to make the build more stable. I think # having multiple components all try to execute the # Bootstrap_CONDA_'Miniconda3' task in parallel causes issues. ['./gradlew', ":browser-engine-gecko:Bootstrap_CONDA_'Miniconda3'"], ]) -def setup_fenix(task, android_components_branch, fenix_branch): +def setup_fenix(task, branch_build_params): task['dependencies'] = { 'branch-build-as': 'branch-build-as-build', 'branch-build-ac': 'branch-build-ac-build', } - task['fetches']['branch-build-as'] = ['application-services-m2.tar.gz' ] - task['fetches']['branch-build-ac'] = ['android-components-m2.tar.gz' ] + task['fetches'] = { + 'toolchain': [ + 'robolectric', + ], + 'branch-build-as': [ 'application-services-m2.tar.gz' ], + 'branch-build-ac': ['android-components-m2.tar.gz' ], + } task['run']['pre-gradlew'].extend([ ['rsync', '-a', '/builds/worker/fetches/.m2/', '/builds/worker/.m2/'], - [ - "taskcluster/scripts/setup-branch-build.py", - '--android-components', android_components_branch, - '--fenix', fenix_branch, - ], + setup_branch_build_command_line(branch_build_params, setup_fenix=True), ['cd', 'fenix'], + ['git', 'rev-parse', '--short', 'HEAD'], ]) -def setup_build(task, repo_name): +def setup_branch_build_command_line(branch_build_params, setup_fenix): + cmd_line = [ + 'taskcluster/scripts/setup-branch-build.py', + '--android-components-owner', + branch_build_params.get('android-components-owner', 'mozilla-mobile'), + '--android-components-branch', + branch_build_params.get('android-components-branch', 'main'), + ] + if setup_fenix: + cmd_line.extend([ + '--fenix-owner', + branch_build_params.get('fenix-owner', 'mozilla-mobile'), + '--fenix-branch', + branch_build_params.get('fenix-branch', 'main'), + ]) + return cmd_line + +def get_build_tasks(task, repo_name): if repo_name == 'fenix': - setup_fenix_build(task) + return get_fenix_build_tasks(task) else: - setup_maven_package_build(task, repo_name) + return get_maven_package_build_tasks(task, repo_name) -def setup_maven_package_build(task, repo_name): +def get_maven_package_build_tasks(task, repo_name): task['run']['gradlew'] = ['publishToMavenLocal'] task['run']['post-gradlew'] = [ [ @@ -161,8 +196,9 @@ def setup_maven_package_build(task, repo_name): 'type': 'file', } ] + yield task -def setup_fenix_build(task): +def get_fenix_build_tasks(task): task['run']['gradlew'] = ['assembleDebug'] task['worker']['artifacts'] = [ { @@ -171,6 +207,53 @@ def setup_fenix_build(task): 'type': 'file', } ] + yield task + +def get_test_tasks(task, repo_name): + if repo_name == 'android-components': + # Split up the android components tasks by project. Running them all at once in the same task tends to cause failures + # Also, use `testRelease` instead of `testDebugUnitTest`. I'm not sure what the difference is, but this is what the android-components CI runs. + for project in get_android_components_projects(): + project_task = deepcopy(task) + project_task['description'] += ' {}'.format(project) + project_task['run']['gradlew'] = android_components_test_gradle_tasks(project) + project_task['name'] = 'test-{}'.format(project) + yield project_task + else: + task['run']['gradlew'] = ['testDebugUnitTest'] + yield task + +def android_components_test_gradle_tasks(project): + # Gradle tasks to run to test an android-components project, this should + # match the android-components code in `taskcluster/ci/build/kind.yml` + if project == 'tooling-lint': + tasks = [ + ':{project}:assemble', + ':{project}:assembleAndroidTest', + ':{project}:test', + ':{project}:lint', + 'githubBuildDetails', + ] + elif project == 'tooling-detekt': + tasks = [ + ':{project}:assemble', + ':{project}:assembleAndroidTest', + ':{project}:test', + ':{project}:lintRelease', + 'githubBuildDetails', + ] + else: + tasks = [ + ':{project}:assemble', + ':{project}:assembleAndroidTest', + ':{project}:testRelease', + ':{project}:lintRelease', + 'githubBuildDetails', + ] + return [t.format(project=project) for t in tasks] + +def get_android_components_projects(): + path = os.path.join(TASKCLUSTER_DIR, 'android-components-projects.json') + with open(path) as f: + return json.load(f) -def setup_test(task, repo_name): - task['run']['gradlew'] = ['testDebugUnitTest'] diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml index 96b27fb646..6ff8f59f26 100644 --- a/taskcluster/ci/toolchain/kind.yml +++ b/taskcluster/ci/toolchain/kind.yml @@ -31,4 +31,5 @@ jobs-from: - android.yml - desktop.yml - resourcemonitor.yml + - robolectric.yml - rust.yml diff --git a/taskcluster/ci/toolchain/robolectric.yml b/taskcluster/ci/toolchain/robolectric.yml new file mode 100644 index 0000000000..b65ac79ede --- /dev/null +++ b/taskcluster/ci/toolchain/robolectric.yml @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +robolectric: + description: 'Build the Robolectric maven packages' + run: + script: build-robolectric-toolchain.sh + toolchain-alias: robolectric + toolchain-artifact: public/build/robolectric.tar.gz diff --git a/taskcluster/docker/linux/Dockerfile b/taskcluster/docker/linux/Dockerfile index 5fa9a9155d..0d9fa9de56 100644 --- a/taskcluster/docker/linux/Dockerfile +++ b/taskcluster/docker/linux/Dockerfile @@ -19,7 +19,7 @@ WORKDIR /builds/worker/ # Configuration ENV ANDROID_BUILD_TOOLS "30.0.3" -ENV ANDROID_PLATFORM_VERSION "30" +ENV ANDROID_PLATFORM_VERSION "31" ENV ANDROID_NDK_VERSION "21.3.6528147" # Set up the language variables to avoid problems (we run locale-gen later). @@ -93,6 +93,8 @@ RUN apt-get update -qq \ rsync \ # Required for creating a venv for glean_parser python3-venv \ + # Used by gradle and the robolectric toolchain task + maven \ && apt-get clean RUN pip3 install --upgrade pip diff --git a/taskcluster/scripts/setup-branch-build.py b/taskcluster/scripts/setup-branch-build.py index 2408d8264d..e74bf999c3 100755 --- a/taskcluster/scripts/setup-branch-build.py +++ b/taskcluster/scripts/setup-branch-build.py @@ -4,35 +4,40 @@ import os import subprocess -ANDROID_COMPONENTS_REPO_URL = 'https://github.com/bendk/android-components' -FENIX_REPO_URL = 'https://github.com/bendk/fenix' - def main(): args = parse_args() - local_properties = ["rust.targets=x86,linux-x86-64"] + local_properties = [] local_properties.extend(branch_build_properties('application-services', '.')) - if args.android_components: - git_checkout(ANDROID_COMPONENTS_REPO_URL, args.android_components) + if args.android_components_branch: + git_checkout(android_components_repo(args), args.android_components_branch) local_properties.extend(branch_build_properties('android-components', 'android-components')) - if args.fenix: - git_checkout(FENIX_REPO_URL, args.fenix) + if args.fenix_branch: + git_checkout(fenix_repo(args), args.fenix_branch) local_properties = '\n'.join(local_properties) print("Local properties:") print(local_properties) write_local_properties("local.properties", local_properties) - if args.android_components: + if args.android_components_branch: write_local_properties("android-components/local.properties", local_properties) - if args.fenix: + if args.fenix_branch: write_local_properties("fenix/local.properties", local_properties) def parse_args(): parser = argparse.ArgumentParser(description='Setup a branch build in taskcluster') - parser.add_argument('--android-components', help='Android components branch') - parser.add_argument('--fenix', help='Fenix branch') + parser.add_argument('--android-components-owner', help='Android components repository owner', default='mozilla-mobile') + parser.add_argument('--android-components-branch', help='Android components branch') + parser.add_argument('--fenix-owner', help='Fenix repository owner', default='mozilla-mobile') + parser.add_argument('--fenix-branch', help='Fenix branch') return parser.parse_args() +def android_components_repo(args): + return f'https://github.com/{args.android_components_owner}/android-components' + +def fenix_repo(args): + return f'https://github.com/{args.fenix_owner}/fenix' + def git_checkout(url, branch): subprocess.check_call(['git', 'clone', '--branch', branch, '--recurse-submodules', '--depth', '1', '--', url]) diff --git a/taskcluster/scripts/toolchain/build-robolectric-toolchain.sh b/taskcluster/scripts/toolchain/build-robolectric-toolchain.sh new file mode 100755 index 0000000000..79ef40598c --- /dev/null +++ b/taskcluster/scripts/toolchain/build-robolectric-toolchain.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Build the Robolectric maven packages for branch builds. +# +# For unknown reasons, these sometimes fail to automatically build when running +# gradle, so we build them upfront and distribute the maven packages. + +set -ex + +mvn dependency:get -Dartifact=org.robolectric:android-all::6.0.1_r3-robolectric-r1 +mvn dependency:get -Dartifact=org.robolectric:android-all:7.0.0_r1-robolectric-r1 +mvn dependency:get -Dartifact=org.robolectric:android-all:8.0.0_r4-robolectric-r1 +mvn dependency:get -Dartifact=org.robolectric:android-all:8.1.0-robolectric-4611349 +mvn dependency:get -Dartifact=org.robolectric:android-all:9-robolectric-4913185 + +# Tar everything into UPLOAD_DIR +cd "$HOME" +mkdir -p "$UPLOAD_DIR" +tar zc --directory=/builds/worker/ --file="$UPLOAD_DIR"/robolectric.tar.gz .m2/repository/org/robolectric/ diff --git a/taskcluster/scripts/update-branch-build-components.py b/taskcluster/scripts/update-branch-build-components.py new file mode 100755 index 0000000000..0f749e209e --- /dev/null +++ b/taskcluster/scripts/update-branch-build-components.py @@ -0,0 +1,47 @@ +#!/usr/bin/python3 +# +# Update the projects list for android-components and Fenix. This is required +# for branch builds to work properly. Run this whenever a new project is +# added. + +import argparse +import json +import os +import yaml + +TASKCLUSTER_DIR = os.path.dirname(os.path.dirname(__file__)) +PROJECTS_FILENAME = "android-components-projects.json" +# Don't try to run tests for these +EXCLUDED_PROJECTS = set([ + 'samples-browser', +]) + +def main(): + args = parse_args() + ac_projects = get_android_components_projects(args) + write_projects(ac_projects) + print("{} updated".format(PROJECTS_FILENAME)) + +def parse_args(): + parser = argparse.ArgumentParser( + description='Update the component list for android-components and fenix') + + parser.add_argument('android_components_dir') + return parser.parse_args() + +def get_android_components_projects(args): + path = os.path.join(args.android_components_dir, ".buildconfig.yml") + with open(path) as f: + build_config = yaml.safe_load(f.read()) + return [ + p for p in build_config['projects'].keys() + if p not in EXCLUDED_PROJECTS + ] + +def write_projects(ac_projects): + path = os.path.join(TASKCLUSTER_DIR, PROJECTS_FILENAME) + with open(path, "wt") as f: + json.dump(ac_projects, f) + +if __name__ == '__main__': + main() diff --git a/taskcluster/test/params/branch-build.yml b/taskcluster/test/params/branch-build.yml index 0d7278b3c1..8b007fcc88 100644 --- a/taskcluster/test/params/branch-build.yml +++ b/taskcluster/test/params/branch-build.yml @@ -20,4 +20,5 @@ repository_type: git target_tasks_method: pr-normal tasks_for: github-pull-request branch-build: + android-components-owner: somebody android-components-branch: some-commit-id