CI: performance impact measurements

.github/workflows/integration-tests-benchmarks.yml

@@ -1,4 +1,4 @@
-name: "Integration Tests - Benchmarks"
+name: 'Integration Tests - Benchmarks'
 on:
   schedule:
     - cron: '0 0 * * *' # every night at midnight UTC
@@ -8,7 +8,7 @@ on:
       - '**/sentry-android/**'
       - '**/sentry-android-core/**'
       - '**/sentry-android-ndk/**'
-      - '**/sentry-android-integration-tests/sentry-uitest-android-benchmark/**'
+      - '**/sentry-android-integration-tests/**'
       - '**/.github/**'
 
 jobs:
@@ -20,11 +20,11 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v2
 
-      - name: "Set up Java: 11"
+      - name: 'Set up Java: 11'
         uses: actions/setup-java@v2
         with:
-          java-version: "11"
-          distribution: "adopt"
+          java-version: '11'
+          distribution: 'adopt'
 
       # Clean, build and release a test apk
       - name: Make assembleBenchmarks
@@ -50,3 +50,35 @@ jobs:
           sauce-username: ${{ secrets.SAUCE_USERNAME }}
           sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
           config-file: .sauce/sentry-uitest-android-benchmark-lite.yml
+
+  app-metrics:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+
+      - name: 'Set up Java: 11'
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - uses: actions/cache@v3
+        id: app-plain-cache
+        with:
+          path: sentry-android-integration-tests/test-app-plain/build/outputs/apk/release/test-app-plain-release.apk
+          key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('sentry-android-integration-tests/test-app-plain/**') }}
+
+      - name: Build Android app plain
+        if: steps.app-plain-cache.outputs['cache-hit'] != 'true'
+        run: ./gradlew :sentry-android-integration-tests:test-app-plain:assembleRelease
+
+      - name: Build Android app with Sentry
+        run: ./gradlew :sentry-android-integration-tests:test-app-sentry:assembleRelease
+
+      - name: Collect app metrics
+        uses: getsentry/action-app-sdk-overhead-metrics@v1
+        with:
+          config: sentry-android-integration-tests/metrics-test.yml
+          sauce-user: ${{ secrets.SAUCE_USERNAME }}
+          sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}

build.gradle.kts

@@ -60,6 +60,8 @@ apiValidation {
             "sentry-samples-netflix-dgs",
             "sentry-uitest-android",
             "sentry-uitest-android-benchmark",
+            "test-app-plain",
+            "test-app-sentry",
         )
     )
 }

sentry-android-integration-tests/README.md

@@ -0,0 +1,7 @@
+# Android performance-impact and integration tests
+
+* [Sample app without sentry](./test-app-plain) created with Android Studio -> New Project -> Basic Activity
+* [Same app, but with Sentry included](./test-app-sentry) - made part of the root project
+* [App metrics test specification (yaml)](./metrics-test.yml)
+* [Espresso-based benchmarks](./sentry-uitest-android-benchmark) - run within SauceLabs (see /.sauce/*.yml)
+* [Espresso-based UI tests](./sentry-uitest-android) - run within SauceLabs (see /.sauce/*.yml)

sentry-android-integration-tests/metrics-test.yml

@@ -0,0 +1,16 @@
+apps:
+  - name: io.sentry.java.tests.perf.appplain
+    activity: MainActivity
+    path: ./test-app-plain/build/outputs/apk/release/test-app-plain-release.apk
+  - name: io.sentry.java.tests.perf.appsentry
+    activity: MainActivity
+    path: ./test-app-sentry/build/outputs/apk/release/test-app-sentry-release.apk
+
+startupTimeTest:
+  runs: 50
+  diffMin: 0
+  diffMax: 150
+
+binarySizeTest:
+  diffMin: 500 KiB
+  diffMax: 700 KiB

sentry-android-integration-tests/test-app-plain/.gitignore

@@ -0,0 +1 @@
+/build

sentry-android-integration-tests/test-app-plain/build.gradle.kts

@@ -0,0 +1,56 @@
+plugins {
+    id("com.android.application")
+}
+
+android {
+    compileSdk = Config.Android.compileSdkVersion
+
+    defaultConfig {
+        applicationId = "io.sentry.java.tests.perf.appplain"
+        minSdk = Config.Android.minSdkVersionNdk
+        targetSdk = Config.Android.targetSdkVersion
+        versionCode = 1
+        versionName = "1.0"
+    }
+
+    buildTypes {
+        getByName("release") {
+            isMinifyEnabled = true
+            signingConfig = signingConfigs.getByName("debug") // to be able to run release mode
+            proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro")
+            ndk {
+                abiFilters.clear()
+                abiFilters.add("arm64-v8a")
+            }
+        }
+    }
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+    buildFeatures {
+        viewBinding = true
+    }
+    signingConfigs {
+        getByName("debug") {
+            storeFile = rootProject.file("debug.keystore")
+            storePassword = "android"
+            keyAlias = "androiddebugkey"
+            keyPassword = "android"
+        }
+    }
+
+    variantFilter {
+        if (Config.Android.shouldSkipDebugVariant(buildType.name)) {
+            ignore = true
+        }
+    }
+}
+
+dependencies {
+    implementation("androidx.appcompat:appcompat:1.3.0")
+    implementation("com.google.android.material:material:1.4.0")
+    implementation("androidx.constraintlayout:constraintlayout:2.0.4")
+    implementation("androidx.navigation:navigation-fragment:2.3.5")
+    implementation("androidx.navigation:navigation-ui:2.3.5")
+}

sentry-android-integration-tests/test-app-plain/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.kts.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="io.sentry.java.tests.perf.appplain">
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.TestAppPlain"
+        tools:targetApi="31">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@style/Theme.TestAppPlain.NoActionBar">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java

@@ -0,0 +1,42 @@
+package io.sentry.java.tests.perf.appplain;
+
+import android.os.Bundle;
+import android.view.LayoutInflater;
+import android.view.View;
+import android.view.ViewGroup;
+import androidx.annotation.NonNull;
+import androidx.fragment.app.Fragment;
+import androidx.navigation.fragment.NavHostFragment;
+import io.sentry.java.tests.perf.appplain.databinding.FragmentFirstBinding;
+
+public class FirstFragment extends Fragment {
+
+  private FragmentFirstBinding binding;
+
+  @Override
+  public View onCreateView(
+      LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
+
+    binding = FragmentFirstBinding.inflate(inflater, container, false);
+    return binding.getRoot();
+  }
+
+  public void onViewCreated(@NonNull View view, Bundle savedInstanceState) {
+    super.onViewCreated(view, savedInstanceState);
+
+    binding.buttonFirst.setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            NavHostFragment.findNavController(FirstFragment.this)
+                .navigate(R.id.action_FirstFragment_to_SecondFragment);
+          }
+        });
+  }
+
+  @Override
+  public void onDestroyView() {
+    super.onDestroyView();
+    binding = null;
+  }
+}

sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java

@@ -0,0 +1,74 @@
+package io.sentry.java.tests.perf.appplain;
+
+import android.os.Bundle;
+import android.view.Menu;
+import android.view.MenuItem;
+import android.view.View;
+import androidx.appcompat.app.AppCompatActivity;
+import androidx.navigation.NavController;
+import androidx.navigation.Navigation;
+import androidx.navigation.ui.AppBarConfiguration;
+import androidx.navigation.ui.NavigationUI;
+import com.google.android.material.snackbar.Snackbar;
+import io.sentry.java.tests.perf.appplain.databinding.ActivityMainBinding;
+
+public class MainActivity extends AppCompatActivity {
+
+  private AppBarConfiguration appBarConfiguration;
+  private ActivityMainBinding binding;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    binding = ActivityMainBinding.inflate(getLayoutInflater());
+    setContentView(binding.getRoot());
+
+    setSupportActionBar(binding.toolbar);
+
+    NavController navController =
+        Navigation.findNavController(this, R.id.nav_host_fragment_content_main);
+    appBarConfiguration = new AppBarConfiguration.Builder(navController.getGraph()).build();
+    NavigationUI.setupActionBarWithNavController(this, navController, appBarConfiguration);
+
+    binding.fab.setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            Snackbar.make(view, "Replace with your own action", Snackbar.LENGTH_LONG)
+                .setAction("Action", null)
+                .show();
+          }
+        });
+  }
+
+  @Override
+  public boolean onCreateOptionsMenu(Menu menu) {
+    // Inflate the menu; this adds items to the action bar if it is present.
+    getMenuInflater().inflate(R.menu.menu_main, menu);
+    return true;
+  }
+
+  @Override
+  public boolean onOptionsItemSelected(MenuItem item) {
+    // Handle action bar item clicks here. The action bar will
+    // automatically handle clicks on the Home/Up button, so long
+    // as you specify a parent activity in AndroidManifest.xml.
+    int id = item.getItemId();
+
+    //noinspection SimplifiableIfStatement
+    if (id == R.id.action_settings) {
+      return true;
+    }
+
+    return super.onOptionsItemSelected(item);
+  }
+
+  @Override
+  public boolean onSupportNavigateUp() {
+    NavController navController =
+        Navigation.findNavController(this, R.id.nav_host_fragment_content_main);
+    return NavigationUI.navigateUp(navController, appBarConfiguration)
+        || super.onSupportNavigateUp();
+  }
+}

sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java

@@ -0,0 +1,42 @@
+package io.sentry.java.tests.perf.appplain;
+
+import android.os.Bundle;
+import android.view.LayoutInflater;
+import android.view.View;
+import android.view.ViewGroup;
+import androidx.annotation.NonNull;
+import androidx.fragment.app.Fragment;
+import androidx.navigation.fragment.NavHostFragment;
+import io.sentry.java.tests.perf.appplain.databinding.FragmentSecondBinding;
+
+public class SecondFragment extends Fragment {
+
+  private FragmentSecondBinding binding;
+
+  @Override
+  public View onCreateView(
+      LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
+
+    binding = FragmentSecondBinding.inflate(inflater, container, false);
+    return binding.getRoot();
+  }
+
+  public void onViewCreated(@NonNull View view, Bundle savedInstanceState) {
+    super.onViewCreated(view, savedInstanceState);
+
+    binding.buttonSecond.setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            NavHostFragment.findNavController(SecondFragment.this)
+                .navigate(R.id.action_SecondFragment_to_FirstFragment);
+          }
+        });
+  }
+
+  @Override
+  public void onDestroyView() {
+    super.onDestroyView();
+    binding = null;
+  }
+}