Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SR] Capture gestures/motion events #3390

Merged
merged 12 commits into from
May 29, 2024
37 changes: 36 additions & 1 deletion sentry-android-replay/api/sentry-android-replay.api
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
public final fun rotate (J)V
}

public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable {
public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)V
public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
Expand All @@ -49,6 +49,7 @@ public final class io/sentry/android/replay/ReplayIntegration : android/content/
public fun onLowMemory ()V
public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V
public fun onScreenshotRecorded (Ljava/io/File;J)V
public fun onTouchEvent (Landroid/view/MotionEvent;)V
public fun pause ()V
public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V
public fun resume ()V
Expand Down Expand Up @@ -89,6 +90,40 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
}

public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
}

public class io/sentry/android/replay/util/FixedWindowCallback : android/view/Window$Callback {
public final field delegate Landroid/view/Window$Callback;
public fun <init> (Landroid/view/Window$Callback;)V
public fun dispatchGenericMotionEvent (Landroid/view/MotionEvent;)Z
public fun dispatchKeyEvent (Landroid/view/KeyEvent;)Z
public fun dispatchKeyShortcutEvent (Landroid/view/KeyEvent;)Z
public fun dispatchPopulateAccessibilityEvent (Landroid/view/accessibility/AccessibilityEvent;)Z
public fun dispatchTouchEvent (Landroid/view/MotionEvent;)Z
public fun dispatchTrackballEvent (Landroid/view/MotionEvent;)Z
public fun onActionModeFinished (Landroid/view/ActionMode;)V
public fun onActionModeStarted (Landroid/view/ActionMode;)V
public fun onAttachedToWindow ()V
public fun onContentChanged ()V
public fun onCreatePanelMenu (ILandroid/view/Menu;)Z
public fun onCreatePanelView (I)Landroid/view/View;
public fun onDetachedFromWindow ()V
public fun onMenuItemSelected (ILandroid/view/MenuItem;)Z
public fun onMenuOpened (ILandroid/view/Menu;)Z
public fun onPanelClosed (ILandroid/view/Menu;)V
public fun onPointerCaptureChanged (Z)V
public fun onPreparePanel (ILandroid/view/View;Landroid/view/Menu;)Z
public fun onProvideKeyboardShortcuts (Ljava/util/List;Landroid/view/Menu;I)V
public fun onSearchRequested ()Z
public fun onSearchRequested (Landroid/view/SearchEvent;)Z
public fun onWindowAttributesChanged (Landroid/view/WindowManager$LayoutParams;)V
public fun onWindowFocusChanged (Z)V
public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;)Landroid/view/ActionMode;
public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;I)Landroid/view/ActionMode;
}

public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer {
public abstract fun getVideoTime ()J
public abstract fun isStarted ()Z
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import android.content.Context
import android.content.res.Configuration
import android.graphics.Bitmap
import android.os.Build
import android.view.MotionEvent
import io.sentry.Hint
import io.sentry.IHub
import io.sentry.Integration
Expand Down Expand Up @@ -32,7 +33,7 @@ public class ReplayIntegration(
private val recorderProvider: (() -> Recorder)? = null,
private val recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)? = null,
private val replayCacheProvider: ((replayId: SentryId) -> ReplayCache)? = null
) : Integration, Closeable, ScreenshotRecorderCallback, ReplayController, ComponentCallbacks {
) : Integration, Closeable, ScreenshotRecorderCallback, TouchRecorderCallback, ReplayController, ComponentCallbacks {

// needed for the Java's call site
constructor(context: Context, dateProvider: ICurrentDateProvider) : this(
Expand Down Expand Up @@ -72,7 +73,7 @@ public class ReplayIntegration(
}

this.hub = hub
recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this)
recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this)
isEnabled.set(true)

try {
Expand Down Expand Up @@ -220,4 +221,8 @@ public class ReplayIntegration(
}

override fun onLowMemory() = Unit

override fun onTouchEvent(event: MotionEvent) {
captureStrategy?.onTouchEvent(event)
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
package io.sentry.android.replay

import android.annotation.TargetApi
import android.view.MotionEvent
import android.view.View
import android.view.Window
import io.sentry.SentryLevel.DEBUG
import io.sentry.SentryLevel.ERROR
import io.sentry.SentryOptions
import io.sentry.android.replay.util.FixedWindowCallback
import io.sentry.android.replay.util.gracefullyShutdown
import io.sentry.android.replay.util.scheduleAtFixedRateSafely
import java.lang.ref.WeakReference
Expand All @@ -16,7 +21,8 @@ import kotlin.LazyThreadSafetyMode.NONE
@TargetApi(26)
internal class WindowRecorder(
private val options: SentryOptions,
private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null
private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null,
private val touchRecorderCallback: TouchRecorderCallback? = null
) : Recorder {

internal companion object {
Expand All @@ -39,7 +45,11 @@ internal class WindowRecorder(
if (added) {
rootViews.add(WeakReference(root))
recorder?.bind(root)

root.startGestureTracking()
} else {
root.stopGestureTracking()

recorder?.unbind(root)
rootViews.removeAll { it.get() == root }

Expand Down Expand Up @@ -86,6 +96,60 @@ internal class WindowRecorder(
isRecording.set(false)
}

override fun close() {
stop()
capturer.gracefullyShutdown(options)
}

private fun View.startGestureTracking() {
val window = phoneWindow
if (window == null) {
options.logger.log(DEBUG, "Window is invalid, not tracking gestures")
return
}

if (touchRecorderCallback == null) {
options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures")
return
}

val delegate = window.callback
window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate)
}

private fun View.stopGestureTracking() {
val window = phoneWindow
if (window == null) {
options.logger.log(DEBUG, "Window was null in stopGestureTracking")
return
}

if (window.callback is SentryReplayGestureRecorder) {
val delegate = (window.callback as SentryReplayGestureRecorder).delegate
window.callback = delegate
}
}

private class SentryReplayGestureRecorder(
private val options: SentryOptions,
private val touchRecorderCallback: TouchRecorderCallback?,
delegate: Window.Callback?
) : FixedWindowCallback(delegate) {
override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
if (event != null) {
val copy: MotionEvent = MotionEvent.obtainNoHistory(event)
try {
touchRecorderCallback?.onTouchEvent(copy)
} catch (e: Throwable) {
options.logger.log(ERROR, "Error dispatching touch event", e)
} finally {
copy.recycle()
}
}
return super.dispatchTouchEvent(event)
}
}

private class RecorderExecutorServiceThreadFactory : ThreadFactory {
private var cnt = 0
override fun newThread(r: Runnable): Thread {
Expand All @@ -94,9 +158,8 @@ internal class WindowRecorder(
return ret
}
}
}

override fun close() {
stop()
capturer.gracefullyShutdown(options)
}
public interface TouchRecorderCallback {
fun onTouchEvent(event: MotionEvent)
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.sentry.android.replay.capture

import android.view.MotionEvent
import io.sentry.Breadcrumb
import io.sentry.DateUtils
import io.sentry.Hint
Expand All @@ -17,13 +18,19 @@ import io.sentry.android.replay.util.submitSafely
import io.sentry.protocol.SentryId
import io.sentry.rrweb.RRWebBreadcrumbEvent
import io.sentry.rrweb.RRWebEvent
import io.sentry.rrweb.RRWebIncrementalSnapshotEvent
import io.sentry.rrweb.RRWebInteractionEvent
import io.sentry.rrweb.RRWebInteractionEvent.InteractionType
import io.sentry.rrweb.RRWebInteractionMoveEvent
import io.sentry.rrweb.RRWebInteractionMoveEvent.Position
import io.sentry.rrweb.RRWebMetaEvent
import io.sentry.rrweb.RRWebSpanEvent
import io.sentry.rrweb.RRWebVideoEvent
import io.sentry.transport.ICurrentDateProvider
import io.sentry.util.FileUtils
import java.io.File
import java.util.Date
import java.util.LinkedList
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ThreadFactory
Expand Down Expand Up @@ -51,6 +58,10 @@ internal abstract class BaseCaptureStrategy(
"http.response_content_length",
"http.request_content_length"
)

// rrweb values
private const val TOUCH_MOVE_DEBOUNCE_THRESHOLD = 50
private const val CAPTURE_MOVE_EVENT_THRESHOLD = 500
}

protected var cache: ReplayCache? = null
Expand All @@ -60,6 +71,12 @@ internal abstract class BaseCaptureStrategy(
override val currentSegment = AtomicInteger(0)
override val replayCacheDir: File? get() = cache?.replayCacheDir

protected val currentEvents = LinkedList<RRWebEvent>()
private val currentEventsLock = Any()
private val currentPositions = mutableListOf<Position>()
private var touchMoveBaseline = 0L
private var lastCapturedMoveEvent = 0L

protected val replayExecutor: ScheduledExecutorService by lazy {
executor ?: Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory())
}
Expand Down Expand Up @@ -249,6 +266,12 @@ internal abstract class BaseCaptureStrategy(
}
}

rotateCurrentEvents(endTimestamp.time) { event ->
if (event.timestamp >= segmentTimestamp.time) {
recordingPayload += event
}
}

val recording = ReplayRecording().apply {
this.segmentId = segmentId
payload = recordingPayload.sortedBy { it.timestamp }
Expand All @@ -265,10 +288,33 @@ internal abstract class BaseCaptureStrategy(
this.recorderConfig = recorderConfig
}

override fun onTouchEvent(event: MotionEvent) {
val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
if (rrwebEvent != null) {
synchronized(currentEventsLock) {
currentEvents += rrwebEvent
}
}
}

override fun close() {
replayExecutor.gracefullyShutdown(options)
}

protected fun rotateCurrentEvents(
until: Long,
callback: ((RRWebEvent) -> Unit)? = null
) {
synchronized(currentEventsLock) {
var event = currentEvents.peek()
while (event != null && event.timestamp <= until) {
callback?.invoke(event)
currentEvents.remove()
event = currentEvents.peek()
}
}
}

private class ReplayExecutorServiceThreadFactory : ThreadFactory {
private var cnt = 0
override fun newThread(r: Runnable): Thread {
Expand Down Expand Up @@ -335,4 +381,63 @@ internal abstract class BaseCaptureStrategy(
data = breadcrumbData
}
}

private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
val event = this
return when (val action = event.actionMasked) {
MotionEvent.ACTION_MOVE -> {
// we only throttle move events as those can be overwhelming
val now = dateProvider.currentTimeMillis
if (lastCapturedMoveEvent != 0L && lastCapturedMoveEvent + TOUCH_MOVE_DEBOUNCE_THRESHOLD > now) {
return null
}
lastCapturedMoveEvent = now

// idk why but rrweb does it like dis
if (touchMoveBaseline == 0L) {
touchMoveBaseline = now
}

currentPositions += Position().apply {
x = event.x * recorderConfig.scaleFactorX
y = event.y * recorderConfig.scaleFactorY
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
timeOffset = now - touchMoveBaseline
}

val totalOffset = now - touchMoveBaseline
return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) {
RRWebInteractionMoveEvent().apply {
timestamp = now
positions = currentPositions.map { pos ->
pos.timeOffset -= totalOffset
pos
}
}.also {
currentPositions.clear()
touchMoveBaseline = 0L
}
} else {
null
}
}

MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
RRWebInteractionEvent().apply {
timestamp = dateProvider.currentTimeMillis
x = event.x * recorderConfig.scaleFactorX
y = event.y * recorderConfig.scaleFactorY
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
interactionType = when (action) {
MotionEvent.ACTION_UP -> InteractionType.TouchEnd
MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
else -> InteractionType.TouchMove_Departed // should not happen
}
}
}

else -> null
}
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.sentry.android.replay.capture

import android.view.MotionEvent
import io.sentry.DateUtils
import io.sentry.Hint
import io.sentry.IHub
Expand Down Expand Up @@ -171,4 +172,10 @@ internal class BufferCaptureStrategy(
captureStrategy.start(segmentId = currentSegment.get(), replayId = currentReplayId.get(), cleanupOldReplays = false)
return captureStrategy
}

override fun onTouchEvent(event: MotionEvent) {
super.onTouchEvent(event)
val bufferLimit = dateProvider.currentTimeMillis - options.experimental.sessionReplay.errorReplayDuration
rotateCurrentEvents(bufferLimit)
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.sentry.android.replay.capture

import android.view.MotionEvent
import io.sentry.Hint
import io.sentry.android.replay.ReplayCache
import io.sentry.android.replay.ScreenshotRecorderConfig
Expand Down Expand Up @@ -27,6 +28,8 @@ internal interface CaptureStrategy {

fun onConfigurationChanged(recorderConfig: ScreenshotRecorderConfig)

fun onTouchEvent(event: MotionEvent)

fun convert(): CaptureStrategy

fun close()
Expand Down
Loading
Loading