diff --git a/app/build.gradle b/app/build.gradle index 72dd716e1de6..830d9807690d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -378,6 +378,12 @@ task printBuildVariants { } } +task printGeckoviewVersions { + doLast { + println "nightly: " + groovy.json.JsonOutput.toJson(GeckoVersions.nightly_version) + } +} + // Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we // can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best. apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle' diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 1f7bd3ae9ea5..46459d03c342 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -12,8 +12,13 @@ import os import taskcluster -from lib import build_variants -from lib.tasks import TaskBuilder, schedule_task_graph, get_architecture_and_build_type_from_variant +from lib.gradle import get_build_variants, get_geckoview_versions +from lib.tasks import ( + fetch_mozharness_task_id, + get_architecture_and_build_type_from_variant, + schedule_task_graph, + TaskBuilder, +) from lib.chain_of_trust import ( populate_chain_of_trust_task_graph, populate_chain_of_trust_required_but_unused_files @@ -22,6 +27,7 @@ REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '') +SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH') # If we see this text inside a pull request title then we will not execute any tasks for this PR. SKIP_TASKS_TRIGGER = '[ci skip]' @@ -31,7 +37,7 @@ task_id=os.environ.get('TASK_ID'), repo_url=REPO_URL, git_ref=os.environ.get('MOBILE_HEAD_BRANCH'), - short_head_branch=os.environ.get('SHORT_HEAD_BRANCH'), + short_head_branch=SHORT_HEAD_BRANCH, commit=COMMIT, owner="fenix-eng-notifications@mozilla.com", source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), @@ -42,30 +48,47 @@ ) -def pr_or_push(is_master_push): - if not is_master_push and SKIP_TASKS_TRIGGER in PR_TITLE: +def pr_or_push(is_push): + if not is_push and SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} + variants = get_build_variants() + geckoview_nightly_version = get_geckoview_versions()['nightly'] + mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) + gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] + build_tasks = {} signing_tasks = {} other_tasks = {} - for variant in build_variants.from_gradle(): + for variant in variants: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) - arch, build_type = get_architecture_and_build_type_from_variant(variant) + architecture, build_type = get_architecture_and_build_type_from_variant(variant) # autophone only supports arm and aarch64, so only sign/perftest those builds if ( + is_push and build_type == 'releaseRaptor' and - arch in ('arm', 'aarch64') and - is_master_push + architecture in ('arm', 'aarch64') and + SHORT_HEAD_BRANCH == 'master' ): - signing_tasks[taskcluster.slugId()] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant) - # raptor task will be added in follow-up + signing_task_id = taskcluster.slugId() + signing_tasks[signing_task_id] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant) + + ALL_RAPTOR_CRAFT_FUNCTIONS = [ + BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) + for i in range(1, 11) + ] + for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS: + args = (signing_task_id, mozharness_task_id, variant, gecko_revision) + other_tasks[taskcluster.slugId()] = craft_function(*args) + # we also want the arm APK to be tested on 64-bit-devices + if architecture == 'arm': + other_tasks[taskcluster.slugId()] = craft_function(*args, force_run_on_64_bit_device=True) for craft_function in ( BUILDER.craft_detekt_task, diff --git a/automation/taskcluster/lib/build_variants.py b/automation/taskcluster/lib/build_variants.py deleted file mode 100644 index f0d3e7c340be..000000000000 --- a/automation/taskcluster/lib/build_variants.py +++ /dev/null @@ -1,29 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function -import json -import subprocess - - -def from_gradle(): - print('Fetching build variants from gradle') - process = subprocess.Popen([ - "./gradlew", "--no-daemon", "--quiet", "printBuildVariants" - ], stdout=subprocess.PIPE) - (output, err) = process.communicate() - exit_code = process.wait() - - if exit_code != 0: - print("Gradle command returned error: {}".format(exit_code)) - - variants_line = [line for line in output.split('\n') if line.startswith('variants: ')][0] - variants_json = variants_line.split(' ', 1)[1] - variants = json.loads(variants_json) - - if len(variants) == 0: - raise RuntimeError('Expected at least one build variant from gradle') - - print("Got variants: " + ' '.join(variants)) - return variants diff --git a/automation/taskcluster/lib/gradle.py b/automation/taskcluster/lib/gradle.py new file mode 100644 index 000000000000..6242c91808cb --- /dev/null +++ b/automation/taskcluster/lib/gradle.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function +import json +import subprocess + + +def get_build_variants(): + print("Fetching build variants from gradle") + output = _run_gradle_process('printBuildVariants') + content = _extract_content_from_command_output(output, prefix='variants: ') + variants = json.loads(content) + + if len(variants) == 0: + raise ValueError("Could not get build variants from gradle") + + print("Got variants: {}".format(' '.join(variants))) + + return variants + + +def get_geckoview_versions(): + print("Fetching geckoview version from gradle") + output = _run_gradle_process('printGeckoviewVersions') + + versions = {} + for version_type in ('nightly',): + version = _extract_content_from_command_output(output, prefix='{}: '.format(version_type)) + version = version.strip('"') + versions[version_type] = version + print('Got {} version: "{}"'.format(version_type, version)) + + return versions + + +def _run_gradle_process(gradle_command): + process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command], stdout=subprocess.PIPE) + output, err = process.communicate() + exit_code = process.wait() + + if exit_code is not 0: + print("Gradle command returned error: {}".format(exit_code)) + + return output + + +def _extract_content_from_command_output(output, prefix): + variants_line = [line for line in output.split('\n') if line.startswith(prefix)][0] + return variants_line.split(' ', 1)[1] diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 4d04af2616f3..6f6a8bdda2e5 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -13,7 +13,9 @@ from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter DEFAULT_EXPIRES_IN = '1 year' +DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk' _OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix' +_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task' class TaskBuilder(object): @@ -252,14 +254,14 @@ def _craft_build_ish_task( } return self._craft_default_task_definition( - 'mobile-{}-b-fenix'.format(self.trust_level), - 'aws-provisioner-v1', - dependencies, - routes, - scopes, - name, - description, - payload, + worker_type='mobile-{}-b-fenix'.format(self.trust_level), + provisioner_id='aws-provisioner-v1', + name=name, + description=description, + payload=payload, + dependencies=dependencies, + routes=routes, + scopes=scopes, treeherder=treeherder, ) @@ -290,9 +292,20 @@ def _craft_signing_task(self, name, description, signing_type, assemble_task_id, ) def _craft_default_task_definition( - self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, - payload, treeherder=None + self, + worker_type, + provisioner_id, + name, + description, + payload, + dependencies=None, + routes=None, + scopes=None, + treeherder=None, ): + dependencies = [] if dependencies is None else dependencies + scopes = [] if scopes is None else scopes + routes = [] if routes is None else routes treeherder = {} if treeherder is None else treeherder created = datetime.datetime.now() @@ -333,21 +346,24 @@ def craft_master_commit_signing_task( ): architecture, build_type = get_architecture_and_build_type_from_variant(variant) build_type = convert_camel_case_into_kebab_case(build_type) - routes = [ - 'index.project.mobile.fenix.v2.branch.master.revision.{}.{}.{}'.format( - self.commit, build_type, architecture - ), - 'index.project.mobile.fenix.v2.branch.master.latest.{}.{}'.format( - build_type, architecture - ), - 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.{}.{}'.format( - self.date.year, self.date.month, self.date.day, self.commit, - build_type, architecture - ), - 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.{}.{}'.format( - self.date.year, self.date.month, self.date.day, build_type, architecture - ), - ] + + routes = [] + if self.repo_url == _OFFICIAL_REPO_URL: + routes = [ + 'index.project.mobile.fenix.v2.branch.master.revision.{}.{}.{}'.format( + self.commit, build_type, architecture + ), + 'index.project.mobile.fenix.v2.branch.master.latest.{}.{}'.format( + build_type, architecture + ), + 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.{}.{}'.format( + self.date.year, self.date.month, self.date.day, self.commit, + build_type, architecture + ), + 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.{}.{}'.format( + self.date.year, self.date.month, self.date.day, build_type, architecture + ), + ] return self._craft_signing_task( name='sign: {}'.format(variant), @@ -436,6 +452,115 @@ def craft_push_task( }, ) + def craft_raptor_tp6m_cold_task(self, for_suite): + + def craft_function(signing_task_id, mozharness_task_id, variant, gecko_revision, force_run_on_64_bit_device=False): + return self._craft_raptor_task( + signing_task_id, + mozharness_task_id, + variant, + gecko_revision, + name_prefix='raptor tp6m-cold-{}'.format(for_suite), + description='Raptor tp6m cold on Fenix', + test_name='raptor-tp6m-cold-{}'.format(for_suite), + job_symbol='tp6m-c-{}'.format(for_suite), + force_run_on_64_bit_device=force_run_on_64_bit_device, + ) + return craft_function + + def _craft_raptor_task( + self, + signing_task_id, + mozharness_task_id, + variant, + gecko_revision, + name_prefix, + description, + test_name, + job_symbol, + group_symbol=None, + extra_test_args=None, + force_run_on_64_bit_device=False, + ): + extra_test_args = [] if extra_test_args is None else extra_test_args + apk_location = '{}/{}/artifacts/{}'.format( + _DEFAULT_TASK_URL, signing_task_id, DEFAULT_APK_ARTIFACT_LOCATION + ) + architecture, _ = get_architecture_and_build_type_from_variant(variant) + worker_type = 'gecko-t-ap-perf-p2' if force_run_on_64_bit_device or architecture == 'aarch64' else 'gecko-t-ap-perf-g5' + + if force_run_on_64_bit_device: + treeherder_platform = 'android-hw-p2-8-0-arm7-api-16' + elif architecture == 'arm': + treeherder_platform = 'android-hw-g5-7-0-arm7-api-16' + elif architecture == 'aarch64': + treeherder_platform = 'android-hw-p2-8-0-aarch64' + else: + raise ValueError('Unsupported architecture "{}"'.format(architecture)) + + task_name = '{}: {} {}'.format( + name_prefix, variant, '(on 64-bit-device)' if force_run_on_64_bit_device else '' + ) + + return self._craft_default_task_definition( + worker_type=worker_type, + provisioner_id='proj-autophone', + dependencies=[signing_task_id], + name=task_name, + description=description, + payload={ + "artifacts": [{ + 'path': '/builds/worker/{}'.format(worker_path), + 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), + 'type': 'directory', + 'name': 'public/{}/'.format(public_folder) + } for worker_path, public_folder in ( + ('artifacts', 'test'), + ('workspace/build/logs', 'logs'), + ('workspace/build/blobber_upload_dir', 'test_info'), + )], + "command": [ + "./test-linux.sh", + '--installer-url={}'.format(apk_location), + "--test-packages-url={}/{}/artifacts/public/build/target.test_packages.json".format(_DEFAULT_TASK_URL, mozharness_task_id), + "--test={}".format(test_name), + "--app=fenix", + "--binary=org.mozilla.fenix", + "--activity=GeckoViewActivity", + "--download-symbols=ondemand" + ] + extra_test_args, + "env": { + "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/mozilla-central", + "GECKO_HEAD_REV": gecko_revision, + "MOZ_AUTOMATION": "1", + "MOZ_HIDE_RESULTS_TABLE": "1", + "MOZ_NO_REMOTE": "1", + "MOZ_NODE_PATH": "/usr/local/bin/node", + "MOZHARNESS_CONFIG": "raptor/android_hw_config.py", + "MOZHARNESS_SCRIPT": "raptor_script.py", + "MOZHARNESS_URL": "{}/{}/artifacts/public/build/mozharness.zip".format(_DEFAULT_TASK_URL, mozharness_task_id), + "MOZILLA_BUILD_URL": apk_location, + "NEED_XVFB": "false", + "NO_FAIL_ON_TEST_ERRORS": "1", + "TASKCLUSTER_WORKER_TYPE": 'proj-autophone/{}'.format(worker_type), + "WORKING_DIR": "/builds/worker", + "WORKSPACE": "/builds/worker/workspace", + "XPCOM_DEBUG_BREAK": "warn", + }, + "context": "https://hg.mozilla.org/mozilla-central/raw-file/{}/taskcluster/scripts/tester/test-linux.sh".format(gecko_revision) + }, + treeherder={ + 'jobKind': 'test', + 'groupSymbol': 'Rap' if group_symbol is None else group_symbol, + 'machine': { + 'platform': treeherder_platform, + }, + 'symbol': job_symbol, + 'tier': 2, + } + ) + + def _craft_treeherder_platform_from_variant(variant): architecture, build_type = get_architecture_and_build_type_from_variant(variant) @@ -449,7 +574,7 @@ def _craft_treeherder_group_symbol_from_variant(variant): def _craft_artifacts_from_variant(variant): return { - 'public/target.apk': { + DEFAULT_APK_ARTIFACT_LOCATION: { 'type': 'file', 'path': _craft_apk_full_path_from_variant(variant), 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), @@ -513,3 +638,13 @@ def schedule_task_graph(ordered_groups_of_tasks): } return full_task_graph + + +def fetch_mozharness_task_id(geckoview_nightly_version): + nightly_build_id = geckoview_nightly_version.split('.')[-1] + nightly_date = arrow.get(nightly_build_id, 'YYYYMMDDHHmmss') + + raptor_index = 'gecko.v2.mozilla-central.pushdate.{}.{:02}.{:02}.{}.firefox.linux64-debug'.format( + nightly_date.year, nightly_date.month, nightly_date.day, nightly_build_id + ) + return taskcluster.Index().findTask(raptor_index)['taskId'] diff --git a/buildSrc/src/main/java/Gecko.kt b/buildSrc/src/main/java/Gecko.kt index 70ca8b93324b..316cbc103409 100644 --- a/buildSrc/src/main/java/Gecko.kt +++ b/buildSrc/src/main/java/Gecko.kt @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -internal object GeckoVersions { +object GeckoVersions { const val nightly_version = "68.0.20190422094240" const val beta_version = "67.0.20190415085659" const val release_version = "66.0.20190322021635"