Fix a-c and fenix branch build tests [ac: main] [fenix: main]

taskcluster/android-components-projects.json

@@ -0,0 +1 @@
+["compose-awesomebar", "compose-browser-toolbar", "compose-engine", "compose-tabstray", "concept-awesomebar", "concept-base", "concept-fetch", "concept-toolbar", "concept-tabstray", "concept-engine", "concept-menu", "concept-push", "concept-storage", "concept-sync", "feature-autofill", "feature-awesomebar", "feature-accounts", "feature-accounts-push", "feature-app-links", "feature-containers", "feature-contextmenu", "feature-customtabs", "feature-intent", "feature-media", "feature-logins", "feature-webnotifications", "feature-readerview", "feature-search", "feature-serviceworker", "feature-session", "feature-tabs", "feature-tab-collections", "feature-recentlyclosed", "feature-toolbar", "feature-top-sites", "feature-share", "feature-downloads", "feature-privatemode", "feature-prompts", "feature-push", "feature-pwa", "feature-findinpage", "feature-syncedtabs", "feature-sitepermissions", "feature-qr", "feature-webauthn", "feature-webcompat", "feature-webcompat-reporter", "feature-addons", "browser-domains", "browser-engine-gecko", "browser-engine-system", "browser-errorpages", "browser-icons", "browser-menu", "browser-menu2", "browser-session-storage", "browser-state", "browser-storage-sync", "browser-tabstray", "browser-thumbnails", "browser-toolbar", "ui-autocomplete", "ui-colors", "ui-widgets", "ui-fonts", "ui-icons", "ui-tabcounter", "service-firefox-accounts", "service-location", "service-sync-autofill", "service-sync-logins", "service-digitalassetlinks", "service-glean", "service-nimbus", "service-pocket", "service-contile", "support-base", "support-images", "support-ktx", "support-migration", "support-test", "support-test-fakes", "support-test-libstate", "support-android-test", "support-test-appservices", "support-utils", "support-sync-telemetry", "support-rustlog", "support-rusthttp", "support-locale", "support-webextensions", "lib-crash", "lib-crash-sentry", "lib-crash-sentry-legacy", "lib-dataprotect", "lib-fetch-httpurlconnection", "lib-fetch-okhttp", "lib-jexl", "lib-publicsuffixlist", "lib-push-firebase", "lib-state", "tooling-detekt", "tooling-lint", "tooling-fetch-tests", "samples-compose-browser", "samples-crash", "samples-firefox-accounts", "samples-glean", "samples-glean-library", "samples-sync", "samples-sync-logins", "samples-toolbar", "samples-dataprotect", "tooling-glean-gradle", "tooling-nimbus-gradle"]

taskcluster/app_services_taskgraph/__init__.py

@@ -22,7 +22,9 @@ def register(graph_config):
 
  extend_parameters_schema({
  Optional('branch-build'): {
+ Optional('android-components-owner'): str,
  Optional('android-components-branch'): str,
+ Optional('fenix-owner'): str,
  Optional('fenix-branch'): str,
  },
  'nightly-build': bool,

taskcluster/app_services_taskgraph/branch_builds.py

@@ -7,8 +7,9 @@
 
 from taskgraph.filter_tasks import filter_task
 
-ANDROID_COMPONENTS_BRANCH_RE = re.compile(r'\[ac:\s*([\w-]+)\]')
-FENIX_BRANCH_RE = re.compile(r'\[fenix:\s*([\w-]+)\]')
+REPO_RE = r'((?P<owner>[\.\w-]+)/)?(?P<branch>[\.\w-]+)'
+ANDROID_COMPONENTS_BRANCH_RE = re.compile(r'\[a-?c:\s*' + REPO_RE + r'\]')
+FENIX_BRANCH_RE = re.compile(r'\[fenix:\s*' + REPO_RE + r'\]')
 
 def update_decision_parameters(parameters):
  parameters['branch-build'] = calc_branch_build_param(parameters)
@@ -25,14 +26,22 @@ def calc_branch_build_param(parameters):
 
  ac_branch_match = ANDROID_COMPONENTS_BRANCH_RE.search(title)
  if ac_branch_match:
- branch_build['android-components-branch'] = ac_branch_match.group(1)
+ branch_build['android-components-owner'] = calc_owner(ac_branch_match)
+ branch_build['android-components-branch'] = ac_branch_match.group('branch')
 
  fenix_branch_match = FENIX_BRANCH_RE.search(title)
  if fenix_branch_match:
- branch_build['fenix-branch'] = fenix_branch_match.group(1)
+ branch_build['fenix-owner'] = calc_owner(fenix_branch_match)
+ branch_build['fenix-branch'] = fenix_branch_match.group('branch')
 
  return branch_build
 
+def calc_owner(match):
+ if match.group('owner'):
+ return match.group('owner')
+ else:
+ return 'mozilla-mobile'
+
 @filter_task("branch-build")
 def filter_branch_build_tasks(full_task_graph, parameters, graph_config):
  if parameters.get('branch-build'):

taskcluster/app_services_taskgraph/transforms/branch_build.py

@@ -3,11 +3,15 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from copy import deepcopy
+import os
+import json
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema
 from voluptuous import Optional, Required, In
 
+TASKCLUSTER_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+
 # Schema for the job dictionary from kinds.yml
 branch_build_schema = Schema({
  # Which repository are we working on
@@ -45,15 +49,14 @@
  'desktop-linux-libs',
  'desktop-macos-libs',
  'desktop-win32-x86-64-libs',
+ 'robolectric',
  'rust',
  ],
  },
  'run': {
  'using': 'gradlew',
  'pre-gradlew': [
  ["git", "submodule", "update", "--init"],
- ["source", "taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh"],
- ["rsync", "-a", "/builds/worker/fetches/libs/", "/builds/worker/checkouts/vcs/libs/"],
  ]
  },
 }
@@ -71,80 +74,112 @@ def validate(config, tasks):
 @transforms.add
 def setup(config, tasks):
  branch_build_params = config.params.get('branch-build', {})
- android_components_branch = branch_build_params.get('android-components-branch', 'main')
- fenix_branch = branch_build_params.get('fenix-branch', 'main')
 
  for task in tasks:
  repo_name = task.pop('repository')
  operation = task['name']
  task.update(deepcopy(TASK_COMMON))
+ task['description'] = '{} {}'.format(operation, repo_name)
  if repo_name == 'application-services':
  setup_application_services(task)
  elif repo_name == 'android-components':
- setup_android_components(task, android_components_branch)
+ setup_android_components(task, branch_build_params)
  elif repo_name == 'fenix':
- setup_fenix(task, android_components_branch, fenix_branch)
+ setup_fenix(task, branch_build_params)
  else:
  raise ValueError("Invalid branch build repository: {}".format(repo_name))
 
  if operation == 'build':
- setup_build(task, repo_name)
+ for task in get_build_tasks(task, repo_name):
+ yield task
  elif operation == 'test':
- setup_test(task, repo_name)
+ for task in get_test_tasks(task, repo_name):
+ yield task
  else:
  raise ValueError("Invalid branch build operation: {}".format(operation))
- task['description'] = '{} {}'.format(operation, repo_name)
- yield task
 
 def setup_application_services(task):
  task['run']['pre-gradlew'].extend([
+ ["source", "taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh"],
+ ["rsync", "-a", "/builds/worker/fetches/libs/", "/builds/worker/checkouts/vcs/libs/"],
  ["taskcluster/scripts/setup-branch-build.py"],
  ])
+ task['fetches'] = {
+ 'toolchain': [
+ 'android-libs',
+ 'desktop-linux-libs',
+ 'desktop-macos-libs',
+ 'desktop-win32-x86-64-libs',
+ 'rust',
+ ],
+ }
 
-def setup_android_components(task, android_components_branch):
+def setup_android_components(task, branch_build_params):
  task['dependencies'] = {
  'branch-build-as': 'branch-build-as-build',
  }
 
- task['fetches']['branch-build-as'] = [ 'application-services-m2.tar.gz' ]
+ task['fetches'] = {
+ 'toolchain': [
+ 'robolectric',
+ ],
+ 'branch-build-as': [ 'application-services-m2.tar.gz' ],
+ }
  task['run']['pre-gradlew'].extend([
  ['rsync', '-a', '/builds/worker/fetches/.m2/', '/builds/worker/.m2/'],
- [
- "taskcluster/scripts/setup-branch-build.py",
- '--android-components', android_components_branch,
- ],
+ setup_branch_build_command_line(branch_build_params, setup_fenix=False),
  ['cd', 'android-components'],
+ ['git', 'rev-parse', '--short', 'HEAD'],
  # Building this up-front seems to make the build more stable. I think
  # having multiple components all try to execute the
  # Bootstrap_CONDA_'Miniconda3' task in parallel causes issues.
  ['./gradlew', ":browser-engine-gecko:Bootstrap_CONDA_'Miniconda3'"],
  ])
 
-def setup_fenix(task, android_components_branch, fenix_branch):
+def setup_fenix(task, branch_build_params):
  task['dependencies'] = {
  'branch-build-as': 'branch-build-as-build',
  'branch-build-ac': 'branch-build-ac-build',
  }
 
- task['fetches']['branch-build-as'] = ['application-services-m2.tar.gz' ]
- task['fetches']['branch-build-ac'] = ['android-components-m2.tar.gz' ]
+ task['fetches'] = {
+ 'toolchain': [
+ 'robolectric',
+ ],
+ 'branch-build-as': [ 'application-services-m2.tar.gz' ],
+ 'branch-build-ac': ['android-components-m2.tar.gz' ],
+ }
  task['run']['pre-gradlew'].extend([
  ['rsync', '-a', '/builds/worker/fetches/.m2/', '/builds/worker/.m2/'],
- [
- "taskcluster/scripts/setup-branch-build.py",
- '--android-components', android_components_branch,
- '--fenix', fenix_branch,
- ],
+ setup_branch_build_command_line(branch_build_params, setup_fenix=True),
  ['cd', 'fenix'],
+ ['git', 'rev-parse', '--short', 'HEAD'],
  ])
 
-def setup_build(task, repo_name):
+def setup_branch_build_command_line(branch_build_params, setup_fenix):
+ cmd_line = [
+ 'taskcluster/scripts/setup-branch-build.py',
+ '--android-components-owner',
+ branch_build_params.get('android-components-owner', 'mozilla-mobile'),
+ '--android-components-branch',
+ branch_build_params.get('android-components-branch', 'main'),
+ ]
+ if setup_fenix:
+ cmd_line.extend([
+ '--fenix-owner',
+ branch_build_params.get('fenix-owner', 'mozilla-mobile'),
+ '--fenix-branch',
+ branch_build_params.get('fenix-branch', 'main'),
+ ])
+ return cmd_line
+
+def get_build_tasks(task, repo_name):
  if repo_name == 'fenix':
- setup_fenix_build(task)
+ return get_fenix_build_tasks(task)
  else:
- setup_maven_package_build(task, repo_name)
+ return get_maven_package_build_tasks(task, repo_name)
 
-def setup_maven_package_build(task, repo_name):
+def get_maven_package_build_tasks(task, repo_name):
  task['run']['gradlew'] = ['publishToMavenLocal']
  task['run']['post-gradlew'] = [
  [
@@ -161,8 +196,9 @@ def setup_maven_package_build(task, repo_name):
  'type': 'file',
  }
  ]
+ yield task
 
-def setup_fenix_build(task):
+def get_fenix_build_tasks(task):
  task['run']['gradlew'] = ['assembleDebug']
  task['worker']['artifacts'] = [
  {
@@ -171,6 +207,53 @@ def setup_fenix_build(task):
  'type': 'file',
  }
  ]
+ yield task
+
+def get_test_tasks(task, repo_name):
+ if repo_name == 'android-components':
+ # Split up the android components tasks by project. Running them all at once in the same task tends to cause failures
+ # Also, use `testRelease` instead of `testDebugUnitTest`. I'm not sure what the difference is, but this is what the android-components CI runs.
+ for project in get_android_components_projects():
+ project_task = deepcopy(task)
+ project_task['description'] += ' {}'.format(project)
+ project_task['run']['gradlew'] = android_components_test_gradle_tasks(project)
+ project_task['name'] = 'test-{}'.format(project)
+ yield project_task
+ else:
+ task['run']['gradlew'] = ['testDebugUnitTest']
+ yield task
+
+def android_components_test_gradle_tasks(project):
+ # Gradle tasks to run to test an android-components project, this should
+ # match the android-components code in `taskcluster/ci/build/kind.yml`
+ if project == 'tooling-lint':
+ tasks = [
+ ':{project}:assemble',
+ ':{project}:assembleAndroidTest',
+ ':{project}:test',
+ ':{project}:lint',
+ 'githubBuildDetails',
+ ]
+ elif project == 'tooling-detekt':
+ tasks = [
+ ':{project}:assemble',
+ ':{project}:assembleAndroidTest',
+ ':{project}:test',
+ ':{project}:lintRelease',
+ 'githubBuildDetails',
+ ]
+ else:
+ tasks = [
+ ':{project}:assemble',
+ ':{project}:assembleAndroidTest',
+ ':{project}:testRelease',
+ ':{project}:lintRelease',
+ 'githubBuildDetails',
+ ]
+ return [t.format(project=project) for t in tasks]
+
+def get_android_components_projects():
+ path = os.path.join(TASKCLUSTER_DIR, 'android-components-projects.json')
+ with open(path) as f:
+ return json.load(f)
 
-def setup_test(task, repo_name):
- task['run']['gradlew'] = ['testDebugUnitTest']

taskcluster/ci/toolchain/kind.yml

@@ -31,4 +31,5 @@ jobs-from:
  - android.yml
  - desktop.yml
  - resourcemonitor.yml
+ - robolectric.yml
  - rust.yml

taskcluster/ci/toolchain/robolectric.yml

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+robolectric:
+ description: 'Build the Robolectric maven packages'
+ run:
+ script: build-robolectric-toolchain.sh
+ toolchain-alias: robolectric
+ toolchain-artifact: public/build/robolectric.tar.gz

taskcluster/docker/linux/Dockerfile

@@ -19,7 +19,7 @@ WORKDIR /builds/worker/
 # Configuration
 
 ENV ANDROID_BUILD_TOOLS "30.0.3"
-ENV ANDROID_PLATFORM_VERSION "30"
+ENV ANDROID_PLATFORM_VERSION "31"
 ENV ANDROID_NDK_VERSION "21.3.6528147"
 
 # Set up the language variables to avoid problems (we run locale-gen later).
@@ -93,6 +93,8 @@ RUN apt-get update -qq \
  rsync \
  # Required for creating a venv for glean_parser
  python3-venv \
+ # Used by gradle and the robolectric toolchain task
+ maven \
  && apt-get clean
 
 RUN pip3 install --upgrade pip

taskcluster/scripts/setup-branch-build.py

@@ -4,35 +4,40 @@
 import os
 import subprocess
 
-ANDROID_COMPONENTS_REPO_URL = 'https://github.com/bendk/android-components'
-FENIX_REPO_URL = 'https://github.com/bendk/fenix'
-
 def main():
  args = parse_args()
- local_properties = ["rust.targets=x86,linux-x86-64"]
+ local_properties = []
  local_properties.extend(branch_build_properties('application-services', '.'))
- if args.android_components:
- git_checkout(ANDROID_COMPONENTS_REPO_URL, args.android_components)
+ if args.android_components_branch:
+ git_checkout(android_components_repo(args), args.android_components_branch)
  local_properties.extend(branch_build_properties('android-components', 'android-components'))
- if args.fenix:
- git_checkout(FENIX_REPO_URL, args.fenix)
+ if args.fenix_branch:
+ git_checkout(fenix_repo(args), args.fenix_branch)
 
  local_properties = '\n'.join(local_properties)
  print("Local properties:")
  print(local_properties)
 
  write_local_properties("local.properties", local_properties)
- if args.android_components:
+ if args.android_components_branch:
  write_local_properties("android-components/local.properties", local_properties)
- if args.fenix:
+ if args.fenix_branch:
  write_local_properties("fenix/local.properties", local_properties)
 
 def parse_args():
  parser = argparse.ArgumentParser(description='Setup a branch build in taskcluster')
- parser.add_argument('--android-components', help='Android components branch')
- parser.add_argument('--fenix', help='Fenix branch')
+ parser.add_argument('--android-components-owner', help='Android components repository owner', default='mozilla-mobile')
+ parser.add_argument('--android-components-branch', help='Android components branch')
+ parser.add_argument('--fenix-owner', help='Fenix repository owner', default='mozilla-mobile')
+ parser.add_argument('--fenix-branch', help='Fenix branch')
  return parser.parse_args()
 
+def android_components_repo(args):
+ return f'https://github.com/{args.android_components_owner}/android-components'
+
+def fenix_repo(args):
+ return f'https://github.com/{args.fenix_owner}/fenix'
+
 def git_checkout(url, branch):
  subprocess.check_call(['git', 'clone', '--branch', branch, '--recurse-submodules', '--depth', '1', '--', url])
 

taskcluster/scripts/toolchain/build-robolectric-toolchain.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Build the Robolectric maven packages for branch builds.
+#
+# For unknown reasons, these sometimes fail to automatically build when running
+# gradle, so we build them upfront and distribute the maven packages.
+
+set -ex
+
+mvn dependency:get -Dartifact=org.robolectric:android-all::6.0.1_r3-robolectric-r1
+mvn dependency:get -Dartifact=org.robolectric:android-all:7.0.0_r1-robolectric-r1
+mvn dependency:get -Dartifact=org.robolectric:android-all:8.0.0_r4-robolectric-r1
+mvn dependency:get -Dartifact=org.robolectric:android-all:8.1.0-robolectric-4611349
+mvn dependency:get -Dartifact=org.robolectric:android-all:9-robolectric-4913185
+
+# Tar everything into UPLOAD_DIR
+cd "$HOME"
+mkdir -p "$UPLOAD_DIR"
+tar zc --directory=/builds/worker/ --file="$UPLOAD_DIR"/robolectric.tar.gz .m2/repository/org/robolectric/