Jetstream: Adding a workload to execute the

jetstream suite of browser based tests.

Jetstream is a browser based benchmark.

This workload executes the full suite of Jetstream
tests and also includes the addition of the clearChromeTabs
function to the base ui automation class.

wa/framework/uiauto/app/src/main/java/com/arm/wa/uiauto/BaseUiAutomation.java

@@ -610,6 +610,26 @@ public void dismissChromePopup() throws Exception {
         }
     }
 
+    // Clear any existing Chrome tabs from the application
+    public void clearChromeTabs() throws Exception {
+        UiObject tabselector =
+            mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/tab_switcher_button")
+                .className("android.widget.ImageButton"));
+        if (!tabselector.exists()){
+            return;
+        }
+        tabselector.click();
+        UiObject menu =
+            mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/menu_button")
+                .className("android.widget.ImageButton"));
+        menu.click();
+        UiObject closetabs =
+            mDevice.findObject(new UiSelector().textContains("Close all tabs"));
+        if (closetabs.exists()){
+            closetabs.click();
+        }
+    }
+
     // Override getParams function to decode a url encoded parameter bundle before
     // passing it to workloads.
     public Bundle getParams() {

wa/workloads/jetstream/__init__.py

@@ -0,0 +1,84 @@
+#    Copyright 2014-2020 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import re
+
+from wa import ApkUiautoWorkload, Parameter
+from wa.framework.exception import ValidationError, WorkloadError
+
+class Jetstream(ApkUiautoWorkload):
+
+    name = 'jetstream'
+    package_names = ['com.android.chrome']
+    regex = re.compile(r'Jetstream Score ([\d.]+)')
+    tests = ['3d-cube-SP', '3d-raytrace-SP', 'acorn-wtb', 'ai-astar', 'Air', 'async-fs', 'Babylon', 'babylon-wtb', 'base64-SP',
+             'Basic', 'bomb-workers', 'Box2D', 'cdjs', 'chai-wtb', 'coffeescript-wtb', 'crypto', 'crypto-aes-SP', 'crypto-md5-SP',
+             'crypto-sha1-SP', 'date-format-tofte-SP', 'date-format-xparb-SP', 'delta-blue', 'earley-boyer','espree-wtb',
+             'first-inspector-code-load', 'FlightPlanner', 'float-mm.c', 'gaussian-blur', 'gbemu', 'hash-map', 'jshint-wtb',
+             'json-parse-inspector', 'json-stringify-inspector', 'lebab-wtb', 'mandreel', 'ML', 'multi-inspector-code-load',
+             'n-body-SP','navier-stokes', 'octane-code-load', 'octane-zlib', 'OfflineAssembler', 'pdfjs', 'prepack-wtb', 
+             'raytrace', 'regex-dna-SP', 'regexp', 'richards', 'segmentation', 'splay', 'stanford-crypto-aes', 'stanford-crypto-pbkdf2',
+             'stanford-crypto-sha256', 'string-unpack-code-SP', 'tagcloud-SP', 'typescript', 'uglify-js-wtb', 'UniPoker']
+    description = '''
+    A workload to execute the jetstream web based benchmark
+
+    Test description:
+    1. Open chrome
+    2. Navigate to the jetstream website - https://browserbench.org/JetStream/
+    3. Execute the benchmark
+
+    known working chrome version 80.0.3987.149
+    '''
+    requires_network = True
+
+    def __init__(self, target, **kwargs):
+        super(Jetstream, self).__init__(target, **kwargs)
+        self.gui.timeout = 2700
+        self.regex_tests = []
+        for test in self.tests:
+            formatted_string = 'text="([\d.]+)" resource-id="results-cell-({})-score"'.format(test)
+            self.regex_tests.append(re.compile(formatted_string))
+        # Add regex for tests with annoyingly different resource id's
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="wasm-score-id(gcc-loops-wasm)"'))
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="wasm-score-id(HashSet-wasm)"'))
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="wasm-score-id(quicksort-wasm)"'))
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="wasm-score-id(richards-wasm)"'))
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="wasm-score-id(tsf-wasm)"'))
+        self.regex_tests.append(re.compile(r'text="([\d.]+)" resource-id="(wsl)-score-score"'))
+
+    def extract_results(self, context):
+        self.target.execute('uiautomator dump')
+        self.target.pull(os.path.join('/sdcard/window_dump.xml'), os.path.join(context.output_directory, 'screenDump.xml'))
+
+    def update_output(self, context):
+        super(Jetstream, self).update_output(context)
+        screen_xml = os.path.join(context.output_directory, 'screenDump.xml')
+        total_score_regex = re.compile(r'text="([\d.]+)" resource-id=""')
+        with open(screen_xml, 'r') as fh:
+            xml_str = fh.read()
+            total_score_match = total_score_regex.search(xml_str)
+            if total_score_match:
+                total_score = float(total_score_match.group(1))
+                context.add_metric('jetstream', total_score, 'score', lower_is_better=False)
+            else:
+                raise WorkloadError('Total score for jetstream could not be found')
+            for regex in self.regex_tests:
+                match = regex.search(xml_str)
+                if match:
+                    result = float(match.group(1))
+                    test_name = match.group(2)
+                    context.add_metric(test_name, result, 'score', lower_is_better=False)
+                else:
+                    raise WorkloadError('score {} cannot be found'.format(regex))

wa/workloads/jetstream/uiauto/app/build.gradle

@@ -0,0 +1,41 @@
+apply plugin: 'com.android.application'
+
+def packageName = "com.arm.wa.uiauto.jetstream"
+
+android {
+    compileSdkVersion 28
+    buildToolsVersion "28.0.3"
+    defaultConfig {
+        applicationId "${packageName}"
+        minSdkVersion 18
+        targetSdkVersion 28
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+        applicationVariants.all { variant ->
+            variant.outputs.each { output ->
+                output.outputFile = file("$project.buildDir/apk/${packageName}.apk")
+            }
+        }
+    }
+}
+
+dependencies {
+    compile fileTree(dir: 'libs', include: ['*.jar'])
+    compile 'com.android.support.test:runner:0.5'
+    compile 'com.android.support.test:rules:0.5'
+    compile 'com.android.support.test.uiautomator:uiautomator-v18:2.1.2'
+    compile(name: 'uiauto', ext:'aar')
+}
+
+repositories {
+    flatDir {
+        dirs 'libs'
+    }
+}

wa/workloads/jetstream/uiauto/app/src/main/AndroidManifest.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.arm.wa.uiauto.jetstream"
+    android:versionCode="1"
+    android:versionName="1.0">
+
+
+    <instrumentation
+        android:name="android.support.test.runner.AndroidJUnitRunner"
+        android:targetPackage="${applicationId}"/>
+
+</manifest>
+

wa/workloads/jetstream/uiauto/app/src/main/java/com/arm/wa/uiauto/jetstream/UiAutomation.java

@@ -0,0 +1,124 @@
+/*    Copyright 2014-2018 ARM Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.arm.wa.uiauto.jetstream;
+
+import android.os.Bundle;
+import android.support.test.runner.AndroidJUnit4;
+import android.support.test.uiautomator.UiObject;
+import android.support.test.uiautomator.UiObjectNotFoundException;
+import android.support.test.uiautomator.UiSelector;
+import android.support.test.uiautomator.UiScrollable;
+
+import com.arm.wa.uiauto.BaseUiAutomation;
+import android.util.Log;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.concurrent.TimeUnit;
+
+@RunWith(AndroidJUnit4.class)
+public class UiAutomation extends BaseUiAutomation {
+
+    private int networkTimeoutSecs = 30;
+    private long networkTimeout =  TimeUnit.SECONDS.toMillis(networkTimeoutSecs);
+    public static String TAG = "UXPERF";
+
+    @Before
+    public void initialize(){
+        initialize_instrumentation();
+    }
+
+    @Test
+    public void setup() throws Exception{
+        setScreenOrientation(ScreenOrientation.NATURAL);
+        dismissChromePopup();
+        openJetstream();
+    }
+
+    @Test
+    public void runWorkload() throws Exception {
+        runBenchmark();
+    }
+
+    @Test
+    public void teardown() throws Exception{
+        clearChromeTabs();
+        unsetScreenOrientation();
+    }
+
+    public void runBenchmark() throws Exception {
+        UiObject start =
+            mDevice.findObject(new UiSelector().description("Start Test"));
+
+        UiObject starttext =
+            mDevice.findObject(new UiSelector().text("Start Test"));
+
+        // Run Jetstream test
+        if (start.waitForExists(20000)) {
+            start.click();
+        } else {
+            starttext.click();
+        }
+
+        UiObject scores =
+            mDevice.findObject(new UiSelector().resourceId("result-summary"));
+        scores.waitForExists(2100000);
+    }
+
+    public void openJetstream() throws Exception {
+        UiObject urlBar =
+            mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/url_bar"));
+
+        UiObject searchBox =  mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/search_box_text"));
+
+        if (!urlBar.waitForExists(5000)) {
+                searchBox.click();
+        }
+
+        String url = "http://browserbench.org/JetStream/";
+
+        // Clicking search box turns it into url bar on some deivces
+        if(urlBar.waitForExists(2000)) {
+            urlBar.click();
+            sleep(2);
+            urlBar.setText(url);
+        } else {
+            searchBox.setText(url);
+        }
+        pressEnter();
+    }
+
+    /*public void clearTabs() throws Exception {
+        UiObject tabselector =
+            mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/tab_switcher_button")
+                .className("android.widget.ImageButton"));
+        if (!tabselector.exists()){
+            return;
+        }
+        tabselector.click();
+        UiObject menu =
+            mDevice.findObject(new UiSelector().resourceId("com.android.chrome:id/menu_button")
+                .className("android.widget.ImageButton"));
+        menu.click();
+        UiObject closetabs =
+            mDevice.findObject(new UiSelector().textContains("Close all tabs"));
+        if (closetabs.exists()){
+            closetabs.click();
+        }
+    }*/
+}

wa/workloads/jetstream/uiauto/build.gradle

@@ -0,0 +1,23 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    repositories {
+        jcenter()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:2.3.2'
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        jcenter()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

wa/workloads/jetstream/uiauto/build.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# CD into build dir if possible - allows building from any directory
+script_path='.'
+if `readlink -f $0 &>/dev/null`; then
+    script_path=`readlink -f $0 2>/dev/null`
+fi
+script_dir=`dirname $script_path`
+cd $script_dir
+
+# Ensure gradelw exists before starting
+if [[ ! -f gradlew ]]; then
+    echo 'gradlew file not found! Check that you are in the right directory.'
+    exit 9
+fi
+
+# Copy base class library from wa dist
+libs_dir=app/libs
+base_class=`python3 -c "import os, wa; print(os.path.join(os.path.dirname(wa.__file__), 'framework', 'uiauto', 'uiauto.aar'))"`
+mkdir -p $libs_dir
+cp $base_class $libs_dir
+
+# Build and return appropriate exit code if failed
+# gradle build
+./gradlew clean :app:assembleDebug
+exit_code=$?
+if [[ $exit_code -ne 0 ]]; then
+    echo "ERROR: 'gradle build' exited with code $exit_code"
+    exit $exit_code
+fi
+
+# If successful move APK file to workload folder (overwrite previous)
+package=com.arm.wa.uiauto.jetstream
+rm -f ../$package
+if [[ -f app/build/apk/$package.apk ]]; then
+    cp app/build/apk/$package.apk ../$package.apk
+else
+    echo 'ERROR: UiAutomator apk could not be found!'
+    exit 9
+fi

wa/workloads/jetstream/uiauto/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed May 03 15:42:44 BST 2017
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip