diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4cf89943..09ecbba6f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Features + +- Session Replay: Gesture/touch support for Flutter ([#3623](https://github.com/getsentry/sentry-java/pull/3623)) + ### Fixes - Avoid ArrayIndexOutOfBoundsException on Android cpu data collection ([#3598](https://github.com/getsentry/sentry-java/pull/3598)) diff --git a/sentry-android-replay/api/sentry-android-replay.api b/sentry-android-replay/api/sentry-android-replay.api index caf0a3e37c..f103957999 100644 --- a/sentry-android-replay/api/sentry-android-replay.api +++ b/sentry-android-replay/api/sentry-android-replay.api @@ -49,7 +49,7 @@ public final class io/sentry/android/replay/ReplayCache$Companion { public final fun makeReplayCacheDir (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;)Ljava/io/File; } -public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable { +public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/gestures/TouchRecorderCallback, java/io/Closeable { public fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V public fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;)V public synthetic fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;ILkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -103,7 +103,18 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion { public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig; } -public abstract interface class io/sentry/android/replay/TouchRecorderCallback { +public final class io/sentry/android/replay/gestures/GestureRecorder : io/sentry/android/replay/OnRootViewsChangedListener { + public fun (Lio/sentry/SentryOptions;Lio/sentry/android/replay/gestures/TouchRecorderCallback;)V + public fun onRootViewsChanged (Landroid/view/View;Z)V + public final fun stop ()V +} + +public final class io/sentry/android/replay/gestures/ReplayGestureConverter { + public fun (Lio/sentry/transport/ICurrentDateProvider;)V + public final fun convert (Landroid/view/MotionEvent;Lio/sentry/android/replay/ScreenshotRecorderConfig;)Ljava/util/List; +} + +public abstract interface class io/sentry/android/replay/gestures/TouchRecorderCallback { public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index 6e6d9ea052..996f7d2ba4 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -20,6 +20,8 @@ import io.sentry.android.replay.capture.BufferCaptureStrategy import io.sentry.android.replay.capture.CaptureStrategy import io.sentry.android.replay.capture.CaptureStrategy.ReplaySegment import io.sentry.android.replay.capture.SessionCaptureStrategy +import io.sentry.android.replay.gestures.GestureRecorder +import io.sentry.android.replay.gestures.TouchRecorderCallback import io.sentry.android.replay.util.MainLooperHandler import io.sentry.android.replay.util.sample import io.sentry.android.replay.util.submitSafely @@ -37,6 +39,7 @@ import java.io.File import java.security.SecureRandom import java.util.LinkedList import java.util.concurrent.atomic.AtomicBoolean +import kotlin.LazyThreadSafetyMode.NONE public class ReplayIntegration( private val context: Context, @@ -62,16 +65,20 @@ public class ReplayIntegration( recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)?, replayCacheProvider: ((replayId: SentryId, recorderConfig: ScreenshotRecorderConfig) -> ReplayCache)?, replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null, - mainLooperHandler: MainLooperHandler? = null + mainLooperHandler: MainLooperHandler? = null, + gestureRecorderProvider: (() -> GestureRecorder)? = null ) : this(context, dateProvider, recorderProvider, recorderConfigProvider, replayCacheProvider) { this.replayCaptureStrategyProvider = replayCaptureStrategyProvider this.mainLooperHandler = mainLooperHandler ?: MainLooperHandler() + this.gestureRecorderProvider = gestureRecorderProvider } private lateinit var options: SentryOptions private var hub: IHub? = null private var recorder: Recorder? = null + private var gestureRecorder: GestureRecorder? = null private val random by lazy { SecureRandom() } + private val rootViewsSpy by lazy(NONE) { RootViewsSpy.install() } // TODO: probably not everything has to be thread-safe here internal val isEnabled = AtomicBoolean(false) @@ -81,6 +88,7 @@ public class ReplayIntegration( private var replayBreadcrumbConverter: ReplayBreadcrumbConverter = NoOpReplayBreadcrumbConverter.getInstance() private var replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null private var mainLooperHandler: MainLooperHandler = MainLooperHandler() + private var gestureRecorderProvider: (() -> GestureRecorder)? = null private lateinit var recorderConfig: ScreenshotRecorderConfig @@ -100,7 +108,8 @@ public class ReplayIntegration( } this.hub = hub - recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this, mainLooperHandler) + recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, mainLooperHandler) + gestureRecorder = gestureRecorderProvider?.invoke() ?: GestureRecorder(options, this) isEnabled.set(true) try { @@ -147,6 +156,7 @@ public class ReplayIntegration( captureStrategy?.start(recorderConfig) recorder?.start(recorderConfig) + registerRootViewListeners() } override fun resume() { @@ -197,7 +207,9 @@ public class ReplayIntegration( return } + unregisterRootViewListeners() recorder?.stop() + gestureRecorder?.stop() captureStrategy?.stop() isRecording.set(false) captureStrategy?.close() @@ -252,6 +264,20 @@ public class ReplayIntegration( captureStrategy?.onTouchEvent(event) } + private fun registerRootViewListeners() { + if (recorder is OnRootViewsChangedListener) { + rootViewsSpy.listeners += (recorder as OnRootViewsChangedListener) + } + rootViewsSpy.listeners += gestureRecorder + } + + private fun unregisterRootViewListeners() { + if (recorder is OnRootViewsChangedListener) { + rootViewsSpy.listeners -= (recorder as OnRootViewsChangedListener) + } + rootViewsSpy.listeners -= gestureRecorder + } + private fun cleanupReplays(unfinishedReplayId: String = "") { // clean up old replays options.cacheDirPath?.let { cacheDir -> diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 40fb6ef931..9680c2a3ad 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -288,6 +288,18 @@ public data class ScreenshotRecorderConfig( val frameRate: Int, val bitRate: Int ) { + internal constructor( + scaleFactorX: Float, + scaleFactorY: Float + ) : this( + recordingWidth = 0, + recordingHeight = 0, + scaleFactorX = scaleFactorX, + scaleFactorY = scaleFactorY, + frameRate = 0, + bitRate = 0 + ) + companion object { /** * Since codec block size is 16, so we have to adjust the width and height to it, otherwise diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt index 01147e3a7f..9e846dfcf0 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt @@ -1,13 +1,8 @@ package io.sentry.android.replay import android.annotation.TargetApi -import android.view.MotionEvent import android.view.View -import android.view.Window -import io.sentry.SentryLevel.DEBUG -import io.sentry.SentryLevel.ERROR import io.sentry.SentryOptions -import io.sentry.android.replay.util.FixedWindowCallback import io.sentry.android.replay.util.MainLooperHandler import io.sentry.android.replay.util.gracefullyShutdown import io.sentry.android.replay.util.scheduleAtFixedRateSafely @@ -17,24 +12,18 @@ import java.util.concurrent.ScheduledFuture import java.util.concurrent.ThreadFactory import java.util.concurrent.TimeUnit.MILLISECONDS import java.util.concurrent.atomic.AtomicBoolean -import kotlin.LazyThreadSafetyMode.NONE @TargetApi(26) internal class WindowRecorder( private val options: SentryOptions, private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null, - private val touchRecorderCallback: TouchRecorderCallback? = null, private val mainLooperHandler: MainLooperHandler -) : Recorder { +) : Recorder, OnRootViewsChangedListener { internal companion object { private const val TAG = "WindowRecorder" } - private val rootViewsSpy by lazy(NONE) { - RootViewsSpy.install() - } - private val isRecording = AtomicBoolean(false) private val rootViews = ArrayList>() private var recorder: ScreenshotRecorder? = null @@ -43,15 +32,11 @@ internal class WindowRecorder( Executors.newSingleThreadScheduledExecutor(RecorderExecutorServiceThreadFactory()) } - private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added -> + override fun onRootViewsChanged(root: View, added: Boolean) { if (added) { rootViews.add(WeakReference(root)) recorder?.bind(root) - - root.startGestureTracking() } else { - root.stopGestureTracking() - recorder?.unbind(root) rootViews.removeAll { it.get() == root } @@ -68,11 +53,10 @@ internal class WindowRecorder( } recorder = ScreenshotRecorder(recorderConfig, options, mainLooperHandler, screenshotRecorderCallback) - rootViewsSpy.listeners += onRootViewsChangedListener capturingTask = capturer.scheduleAtFixedRateSafely( options, "$TAG.capture", - 0L, + 100L, // delay the first run by a bit, to allow root view listener to register 1000L / recorderConfig.frameRate, MILLISECONDS ) { @@ -88,7 +72,6 @@ internal class WindowRecorder( } override fun stop() { - rootViewsSpy.listeners -= onRootViewsChangedListener rootViews.forEach { recorder?.unbind(it.get()) } recorder?.close() rootViews.clear() @@ -103,55 +86,6 @@ internal class WindowRecorder( capturer.gracefullyShutdown(options) } - private fun View.startGestureTracking() { - val window = phoneWindow - if (window == null) { - options.logger.log(DEBUG, "Window is invalid, not tracking gestures") - return - } - - if (touchRecorderCallback == null) { - options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures") - return - } - - val delegate = window.callback - window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate) - } - - private fun View.stopGestureTracking() { - val window = phoneWindow - if (window == null) { - options.logger.log(DEBUG, "Window was null in stopGestureTracking") - return - } - - if (window.callback is SentryReplayGestureRecorder) { - val delegate = (window.callback as SentryReplayGestureRecorder).delegate - window.callback = delegate - } - } - - private class SentryReplayGestureRecorder( - private val options: SentryOptions, - private val touchRecorderCallback: TouchRecorderCallback?, - delegate: Window.Callback? - ) : FixedWindowCallback(delegate) { - override fun dispatchTouchEvent(event: MotionEvent?): Boolean { - if (event != null) { - val copy: MotionEvent = MotionEvent.obtainNoHistory(event) - try { - touchRecorderCallback?.onTouchEvent(copy) - } catch (e: Throwable) { - options.logger.log(ERROR, "Error dispatching touch event", e) - } finally { - copy.recycle() - } - } - return super.dispatchTouchEvent(event) - } - } - private class RecorderExecutorServiceThreadFactory : ThreadFactory { private var cnt = 0 override fun newThread(r: Runnable): Thread { @@ -161,7 +95,3 @@ internal class WindowRecorder( } } } - -public interface TouchRecorderCallback { - fun onTouchEvent(event: MotionEvent) -} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt index d888c2e33d..97cb386109 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt @@ -23,16 +23,12 @@ import io.sentry.android.replay.ScreenshotRecorderConfig import io.sentry.android.replay.capture.CaptureStrategy.Companion.createSegment import io.sentry.android.replay.capture.CaptureStrategy.Companion.currentEventsLock import io.sentry.android.replay.capture.CaptureStrategy.ReplaySegment +import io.sentry.android.replay.gestures.ReplayGestureConverter import io.sentry.android.replay.util.PersistableLinkedList import io.sentry.android.replay.util.gracefullyShutdown import io.sentry.android.replay.util.submitSafely import io.sentry.protocol.SentryId import io.sentry.rrweb.RRWebEvent -import io.sentry.rrweb.RRWebIncrementalSnapshotEvent -import io.sentry.rrweb.RRWebInteractionEvent -import io.sentry.rrweb.RRWebInteractionEvent.InteractionType -import io.sentry.rrweb.RRWebInteractionMoveEvent -import io.sentry.rrweb.RRWebInteractionMoveEvent.Position import io.sentry.transport.ICurrentDateProvider import java.io.File import java.util.Date @@ -56,15 +52,12 @@ internal abstract class BaseCaptureStrategy( internal companion object { private const val TAG = "CaptureStrategy" - - // rrweb values - private const val TOUCH_MOVE_DEBOUNCE_THRESHOLD = 50 - private const val CAPTURE_MOVE_EVENT_THRESHOLD = 500 } private val persistingExecutor: ScheduledExecutorService by lazy { Executors.newSingleThreadScheduledExecutor(ReplayPersistingExecutorServiceThreadFactory()) } + private val gestureConverter = ReplayGestureConverter(dateProvider) protected val isTerminating = AtomicBoolean(false) protected var cache: ReplayCache? = null @@ -94,9 +87,6 @@ internal abstract class BaseCaptureStrategy( persistingExecutor, cacheProvider = { cache } ) - private val currentPositions = LinkedHashMap>(10) - private var touchMoveBaseline = 0L - private var lastCapturedMoveEvent = 0L protected val replayExecutor: ScheduledExecutorService by lazy { executor ?: Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory()) @@ -169,7 +159,7 @@ internal abstract class BaseCaptureStrategy( } override fun onTouchEvent(event: MotionEvent) { - val rrwebEvents = event.toRRWebIncrementalSnapshotEvent() + val rrwebEvents = gestureConverter.convert(event, recorderConfig) if (rrwebEvents != null) { synchronized(currentEventsLock) { currentEvents += rrwebEvents @@ -199,126 +189,6 @@ internal abstract class BaseCaptureStrategy( } } - private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): List? { - val event = this - return when (event.actionMasked) { - MotionEvent.ACTION_MOVE -> { - // we only throttle move events as those can be overwhelming - val now = dateProvider.currentTimeMillis - if (lastCapturedMoveEvent != 0L && lastCapturedMoveEvent + TOUCH_MOVE_DEBOUNCE_THRESHOLD > now) { - return null - } - lastCapturedMoveEvent = now - - currentPositions.keys.forEach { pId -> - val pIndex = event.findPointerIndex(pId) - - if (pIndex == -1) { - // no data for this pointer - return@forEach - } - - // idk why but rrweb does it like dis - if (touchMoveBaseline == 0L) { - touchMoveBaseline = now - } - - currentPositions[pId]!! += Position().apply { - x = event.getX(pIndex) * recorderConfig.scaleFactorX - y = event.getY(pIndex) * recorderConfig.scaleFactorY - id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE - timeOffset = now - touchMoveBaseline - } - } - - val totalOffset = now - touchMoveBaseline - return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) { - val moveEvents = mutableListOf() - for ((pointerId, positions) in currentPositions) { - if (positions.isNotEmpty()) { - moveEvents += RRWebInteractionMoveEvent().apply { - this.timestamp = now - this.positions = positions.map { pos -> - pos.timeOffset -= totalOffset - pos - } - this.pointerId = pointerId - } - currentPositions[pointerId]!!.clear() - } - } - touchMoveBaseline = 0L - moveEvents - } else { - null - } - } - - MotionEvent.ACTION_DOWN, - MotionEvent.ACTION_POINTER_DOWN -> { - val pId = event.getPointerId(event.actionIndex) - val pIndex = event.findPointerIndex(pId) - - if (pIndex == -1) { - // no data for this pointer - return null - } - - // new finger down - add a new pointer for tracking movement - currentPositions[pId] = ArrayList() - listOf( - RRWebInteractionEvent().apply { - timestamp = dateProvider.currentTimeMillis - x = event.getX(pIndex) * recorderConfig.scaleFactorX - y = event.getY(pIndex) * recorderConfig.scaleFactorY - id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE - pointerId = pId - interactionType = InteractionType.TouchStart - } - ) - } - MotionEvent.ACTION_UP, - MotionEvent.ACTION_POINTER_UP -> { - val pId = event.getPointerId(event.actionIndex) - val pIndex = event.findPointerIndex(pId) - - if (pIndex == -1) { - // no data for this pointer - return null - } - - // finger lift up - remove the pointer from tracking - currentPositions.remove(pId) - listOf( - RRWebInteractionEvent().apply { - timestamp = dateProvider.currentTimeMillis - x = event.getX(pIndex) * recorderConfig.scaleFactorX - y = event.getY(pIndex) * recorderConfig.scaleFactorY - id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE - pointerId = pId - interactionType = InteractionType.TouchEnd - } - ) - } - MotionEvent.ACTION_CANCEL -> { - // gesture cancelled - remove all pointers from tracking - currentPositions.clear() - listOf( - RRWebInteractionEvent().apply { - timestamp = dateProvider.currentTimeMillis - x = event.x * recorderConfig.scaleFactorX - y = event.y * recorderConfig.scaleFactorY - id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE - pointerId = 0 // the pointerId is not used for TouchCancel, so just set it to 0 - interactionType = InteractionType.TouchCancel - } - ) - } - - else -> null - } - } - private inline fun persistableAtomicNullable( initialValue: T? = null, propertyName: String, diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/GestureRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/GestureRecorder.kt new file mode 100644 index 0000000000..57302aaac1 --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/GestureRecorder.kt @@ -0,0 +1,85 @@ +package io.sentry.android.replay.gestures + +import android.view.MotionEvent +import android.view.View +import android.view.Window +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.ERROR +import io.sentry.SentryOptions +import io.sentry.android.replay.OnRootViewsChangedListener +import io.sentry.android.replay.phoneWindow +import io.sentry.android.replay.util.FixedWindowCallback +import java.lang.ref.WeakReference + +class GestureRecorder( + private val options: SentryOptions, + private val touchRecorderCallback: TouchRecorderCallback +) : OnRootViewsChangedListener { + + private val rootViews = ArrayList>() + + override fun onRootViewsChanged(root: View, added: Boolean) { + if (added) { + rootViews.add(WeakReference(root)) + root.startGestureTracking() + } else { + root.stopGestureTracking() + rootViews.removeAll { it.get() == root } + } + } + + fun stop() { + rootViews.forEach { it.get()?.stopGestureTracking() } + rootViews.clear() + } + + private fun View.startGestureTracking() { + val window = phoneWindow + if (window == null) { + options.logger.log(DEBUG, "Window is invalid, not tracking gestures") + return + } + + val delegate = window.callback + if (delegate !is SentryReplayGestureRecorder) { + window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate) + } + } + + private fun View.stopGestureTracking() { + val window = phoneWindow + if (window == null) { + options.logger.log(DEBUG, "Window was null in stopGestureTracking") + return + } + + if (window.callback is SentryReplayGestureRecorder) { + val delegate = (window.callback as SentryReplayGestureRecorder).delegate + window.callback = delegate + } + } + + internal class SentryReplayGestureRecorder( + private val options: SentryOptions, + private val touchRecorderCallback: TouchRecorderCallback?, + delegate: Window.Callback? + ) : FixedWindowCallback(delegate) { + override fun dispatchTouchEvent(event: MotionEvent?): Boolean { + if (event != null) { + val copy: MotionEvent = MotionEvent.obtainNoHistory(event) + try { + touchRecorderCallback?.onTouchEvent(copy) + } catch (e: Throwable) { + options.logger.log(ERROR, "Error dispatching touch event", e) + } finally { + copy.recycle() + } + } + return super.dispatchTouchEvent(event) + } + } +} + +public interface TouchRecorderCallback { + fun onTouchEvent(event: MotionEvent) +} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/ReplayGestureConverter.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/ReplayGestureConverter.kt new file mode 100644 index 0000000000..59d6b30bce --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/gestures/ReplayGestureConverter.kt @@ -0,0 +1,144 @@ +package io.sentry.android.replay.gestures + +import android.view.MotionEvent +import io.sentry.android.replay.ScreenshotRecorderConfig +import io.sentry.rrweb.RRWebIncrementalSnapshotEvent +import io.sentry.rrweb.RRWebInteractionEvent +import io.sentry.rrweb.RRWebInteractionEvent.InteractionType +import io.sentry.rrweb.RRWebInteractionMoveEvent +import io.sentry.rrweb.RRWebInteractionMoveEvent.Position +import io.sentry.transport.ICurrentDateProvider + +class ReplayGestureConverter( + private val dateProvider: ICurrentDateProvider +) { + + internal companion object { + // rrweb values + private const val TOUCH_MOVE_DEBOUNCE_THRESHOLD = 50 + private const val CAPTURE_MOVE_EVENT_THRESHOLD = 500 + } + + private val currentPositions = LinkedHashMap>(10) + private var touchMoveBaseline = 0L + private var lastCapturedMoveEvent = 0L + + fun convert(event: MotionEvent, recorderConfig: ScreenshotRecorderConfig): List? { + return when (event.actionMasked) { + MotionEvent.ACTION_MOVE -> { + // we only throttle move events as those can be overwhelming + val now = dateProvider.currentTimeMillis + if (lastCapturedMoveEvent != 0L && lastCapturedMoveEvent + TOUCH_MOVE_DEBOUNCE_THRESHOLD > now) { + return null + } + lastCapturedMoveEvent = now + + currentPositions.keys.forEach { pId -> + val pIndex = event.findPointerIndex(pId) + + if (pIndex == -1) { + // no data for this pointer + return@forEach + } + + // idk why but rrweb does it like dis + if (touchMoveBaseline == 0L) { + touchMoveBaseline = now + } + + currentPositions[pId]!! += Position().apply { + x = event.getX(pIndex) * recorderConfig.scaleFactorX + y = event.getY(pIndex) * recorderConfig.scaleFactorY + id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE + timeOffset = now - touchMoveBaseline + } + } + + val totalOffset = now - touchMoveBaseline + return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) { + val moveEvents = mutableListOf() + for ((pointerId, positions) in currentPositions) { + if (positions.isNotEmpty()) { + moveEvents += RRWebInteractionMoveEvent().apply { + this.timestamp = now + this.positions = positions.map { pos -> + pos.timeOffset -= totalOffset + pos + } + this.pointerId = pointerId + } + currentPositions[pointerId]!!.clear() + } + } + touchMoveBaseline = 0L + moveEvents + } else { + null + } + } + + MotionEvent.ACTION_DOWN, + MotionEvent.ACTION_POINTER_DOWN -> { + val pId = event.getPointerId(event.actionIndex) + val pIndex = event.findPointerIndex(pId) + + if (pIndex == -1) { + // no data for this pointer + return null + } + + // new finger down - add a new pointer for tracking movement + currentPositions[pId] = ArrayList() + listOf( + RRWebInteractionEvent().apply { + timestamp = dateProvider.currentTimeMillis + x = event.getX(pIndex) * recorderConfig.scaleFactorX + y = event.getY(pIndex) * recorderConfig.scaleFactorY + id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE + pointerId = pId + interactionType = InteractionType.TouchStart + } + ) + } + MotionEvent.ACTION_UP, + MotionEvent.ACTION_POINTER_UP -> { + val pId = event.getPointerId(event.actionIndex) + val pIndex = event.findPointerIndex(pId) + + if (pIndex == -1) { + // no data for this pointer + return null + } + + // finger lift up - remove the pointer from tracking + currentPositions.remove(pId) + listOf( + RRWebInteractionEvent().apply { + timestamp = dateProvider.currentTimeMillis + x = event.getX(pIndex) * recorderConfig.scaleFactorX + y = event.getY(pIndex) * recorderConfig.scaleFactorY + id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE + pointerId = pId + interactionType = InteractionType.TouchEnd + } + ) + } + MotionEvent.ACTION_CANCEL -> { + // gesture cancelled - remove all pointers from tracking + currentPositions.clear() + listOf( + RRWebInteractionEvent().apply { + timestamp = dateProvider.currentTimeMillis + x = event.x * recorderConfig.scaleFactorX + y = event.y * recorderConfig.scaleFactorY + id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE + pointerId = 0 // the pointerId is not used for TouchCancel, so just set it to 0 + interactionType = InteractionType.TouchCancel + } + ) + } + + else -> null + } + } +} diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt index e96375dfa6..1518b41a81 100644 --- a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt @@ -27,6 +27,7 @@ import io.sentry.android.replay.ReplayCache.Companion.SEGMENT_KEY_TIMESTAMP import io.sentry.android.replay.ReplayCache.Companion.SEGMENT_KEY_WIDTH import io.sentry.android.replay.capture.CaptureStrategy import io.sentry.android.replay.capture.SessionCaptureStrategyTest.Fixture.Companion.VIDEO_DURATION +import io.sentry.android.replay.gestures.GestureRecorder import io.sentry.cache.PersistingScopeObserver import io.sentry.protocol.SentryException import io.sentry.protocol.SentryId @@ -100,6 +101,7 @@ class ReplayIntegrationTest { recorderProvider: (() -> Recorder)? = null, replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null, recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)? = null, + gestureRecorderProvider: (() -> GestureRecorder)? = null, dateProvider: ICurrentDateProvider = CurrentDateProvider.getInstance() ): ReplayIntegration { options.run { @@ -112,7 +114,8 @@ class ReplayIntegrationTest { recorderProvider, recorderConfigProvider = recorderConfigProvider, replayCacheProvider = { _, _ -> replayCache }, - replayCaptureStrategyProvider = replayCaptureStrategyProvider + replayCaptureStrategyProvider = replayCaptureStrategyProvider, + gestureRecorderProvider = gestureRecorderProvider ) } } @@ -360,10 +363,16 @@ class ReplayIntegrationTest { } @Test - fun `stop calls stop for recorder and strategy and sets recording to false`() { + fun `stop calls stop for recorders and strategy and sets recording to false`() { val captureStrategy = mock() val recorder = mock() - val replay = fixture.getSut(context, recorderProvider = { recorder }, replayCaptureStrategyProvider = { captureStrategy }) + val gestureRecorder = mock() + val replay = fixture.getSut( + context, + recorderProvider = { recorder }, + replayCaptureStrategyProvider = { captureStrategy }, + gestureRecorderProvider = { gestureRecorder } + ) replay.register(fixture.hub, fixture.options) replay.start() @@ -371,6 +380,7 @@ class ReplayIntegrationTest { verify(captureStrategy).stop() verify(recorder).stop() + verify(gestureRecorder).stop() assertFalse(replay.isRecording) } diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/GestureRecorderTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/GestureRecorderTest.kt new file mode 100644 index 0000000000..bb2de2b7c8 --- /dev/null +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/GestureRecorderTest.kt @@ -0,0 +1,131 @@ +package io.sentry.android.replay.gestures + +import android.R +import android.app.Activity +import android.os.Bundle +import android.view.MotionEvent +import android.view.View +import android.widget.LinearLayout +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.SentryOptions +import io.sentry.android.core.internal.gestures.NoOpWindowCallback +import io.sentry.android.replay.gestures.GestureRecorder.SentryReplayGestureRecorder +import io.sentry.android.replay.phoneWindow +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.annotation.Config +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +@Config(sdk = [30]) +class GestureRecorderTest { + internal class Fixture { + + val options = SentryOptions() + + fun getSut( + touchRecorderCallback: TouchRecorderCallback = NoOpTouchRecorderCallback() + ): GestureRecorder { + return GestureRecorder(options, touchRecorderCallback) + } + } + + private val fixture = Fixture() + private class NoOpTouchRecorderCallback : TouchRecorderCallback { + override fun onTouchEvent(event: MotionEvent) = Unit + } + + @Test + fun `when new window added and window callback is already wrapped, does not wrap it again`() { + val activity = Robolectric.buildActivity(TestActivity::class.java).setup().get() + val gestureRecorder = fixture.getSut() + + activity.root.phoneWindow?.callback = SentryReplayGestureRecorder(fixture.options, null, null) + gestureRecorder.onRootViewsChanged(activity.root, true) + + assertFalse((activity.root.phoneWindow?.callback as SentryReplayGestureRecorder).delegate is SentryReplayGestureRecorder) + } + + @Test + fun `when new window added tracks touch events`() { + var called = false + val activity = Robolectric.buildActivity(TestActivity::class.java).setup().get() + val motionEvent = MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, 0f, 0f, 0) + val gestureRecorder = fixture.getSut( + touchRecorderCallback = object : TouchRecorderCallback { + override fun onTouchEvent(event: MotionEvent) { + assertEquals(MotionEvent.ACTION_DOWN, event.action) + called = true + } + } + ) + + gestureRecorder.onRootViewsChanged(activity.root, true) + + activity.root.phoneWindow?.callback?.dispatchTouchEvent(motionEvent) + assertTrue(called) + } + + @Test + fun `when window removed and window is not sentry recorder does nothing`() { + val activity = Robolectric.buildActivity(TestActivity::class.java).setup().get() + val gestureRecorder = fixture.getSut() + + activity.root.phoneWindow?.callback = NoOpWindowCallback() + gestureRecorder.onRootViewsChanged(activity.root, false) + + assertTrue(activity.root.phoneWindow?.callback is NoOpWindowCallback) + } + + @Test + fun `when window removed stops tracking touch events`() { + val activity = Robolectric.buildActivity(TestActivity::class.java).setup().get() + val gestureRecorder = fixture.getSut() + + gestureRecorder.onRootViewsChanged(activity.root, true) + gestureRecorder.onRootViewsChanged(activity.root, false) + + assertFalse(activity.root.phoneWindow?.callback is SentryReplayGestureRecorder) + } + + @Test + fun `when stopped stops tracking all windows`() { + val activity1 = Robolectric.buildActivity(TestActivity::class.java).setup().get() + val activity2 = Robolectric.buildActivity(TestActivity2::class.java).setup().get() + val gestureRecorder = fixture.getSut() + + gestureRecorder.onRootViewsChanged(activity1.root, true) + gestureRecorder.onRootViewsChanged(activity2.root, true) + gestureRecorder.stop() + + assertFalse(activity1.root.phoneWindow?.callback is SentryReplayGestureRecorder) + assertFalse(activity2.root.phoneWindow?.callback is SentryReplayGestureRecorder) + } +} + +private class TestActivity : Activity() { + lateinit var root: View + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setTheme(R.style.Theme_Holo_Light) + root = LinearLayout(this) + setContentView(root) + actionBar!!.setIcon(R.drawable.ic_lock_power_off) + } +} + +private class TestActivity2 : Activity() { + lateinit var root: View + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setTheme(R.style.Theme_Holo_Light) + root = LinearLayout(this) + setContentView(root) + actionBar!!.setIcon(R.drawable.ic_lock_power_off) + } +} diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/ReplayGestureConverterTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/ReplayGestureConverterTest.kt new file mode 100644 index 0000000000..00ae93af4a --- /dev/null +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/gestures/ReplayGestureConverterTest.kt @@ -0,0 +1,240 @@ +package io.sentry.android.replay.gestures + +import android.view.MotionEvent +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.android.replay.ScreenshotRecorderConfig +import io.sentry.rrweb.RRWebInteractionEvent +import io.sentry.rrweb.RRWebInteractionMoveEvent +import io.sentry.transport.ICurrentDateProvider +import org.junit.runner.RunWith +import org.robolectric.annotation.Config +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +@Config(sdk = [30]) +class ReplayGestureConverterTest { + internal class Fixture { + var now: Long = 1000L + + fun getSut( + dateProvider: ICurrentDateProvider = ICurrentDateProvider { now } + ): ReplayGestureConverter { + return ReplayGestureConverter(dateProvider) + } + } + + private val fixture = Fixture() + + @Test + fun `convert ACTION_DOWN event`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + val event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, 100f, 200f, 0) + + val result = sut.convert(event, recorderConfig) + + assertNotNull(result) + assertEquals(1, result.size) + assertTrue(result[0] is RRWebInteractionEvent) + with(result[0] as RRWebInteractionEvent) { + assertEquals(1000L, timestamp) + assertEquals(100f, x) + assertEquals(200f, y) + assertEquals(0, id) + assertEquals(0, pointerId) + assertEquals(RRWebInteractionEvent.InteractionType.TouchStart, interactionType) + } + + event.recycle() + } + + @Test + fun `convert ACTION_MOVE event with debounce`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + val event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_MOVE, 100f, 200f, 0) + + // First call should pass + var result = sut.convert(event, recorderConfig) + assertNotNull(result) + + // Second call within debounce threshold should be null + fixture.now += 40 // Increase time by 40ms + result = sut.convert(event, recorderConfig) + assertNull(result) + + event.recycle() + } + + @Test + fun `convert ACTION_MOVE event with capture threshold`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + val downEvent = MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, 100f, 200f, 0) + val moveEvent = MotionEvent.obtain(0, 0, MotionEvent.ACTION_MOVE, 110f, 210f, 0) + + // Add a pointer to currentPositions + sut.convert(downEvent, recorderConfig) + + // First call should not trigger capture + var result = sut.convert(moveEvent, recorderConfig) + assertNull(result) + + // Second call should trigger capture + fixture.now += 600 // Increase time by 600ms + result = sut.convert(moveEvent, recorderConfig) + assertNotNull(result) + with(result[0] as RRWebInteractionMoveEvent) { + assertEquals(1600L, timestamp) + assertEquals(2, positions!!.size) + assertEquals(110f, positions!![0].x) + assertEquals(210f, positions!![0].y) + assertEquals(0, positions!![0].id) + assertEquals(0, pointerId) + } + + downEvent.recycle() + moveEvent.recycle() + } + + @Test + fun `convert ACTION_UP event`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + val event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_UP, 100f, 200f, 0) + + val result = sut.convert(event, recorderConfig) + + assertNotNull(result) + assertEquals(1, result.size) + assertTrue(result[0] is RRWebInteractionEvent) + with(result[0] as RRWebInteractionEvent) { + assertEquals(1000L, timestamp) + assertEquals(100f, x) + assertEquals(200f, y) + assertEquals(0, id) + assertEquals(0, pointerId) + assertEquals(RRWebInteractionEvent.InteractionType.TouchEnd, interactionType) + } + + event.recycle() + } + + @Test + fun `convert ACTION_CANCEL event`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + val event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_CANCEL, 100f, 200f, 0) + + val result = sut.convert(event, recorderConfig) + + assertNotNull(result) + assertEquals(1, result.size) + assertTrue(result[0] is RRWebInteractionEvent) + with(result[0] as RRWebInteractionEvent) { + assertEquals(1000L, timestamp) + assertEquals(100f, x) + assertEquals(200f, y) + assertEquals(0, id) + assertEquals(0, pointerId) + assertEquals(RRWebInteractionEvent.InteractionType.TouchCancel, interactionType) + } + + event.recycle() + } + + @Test + fun `convert event with different scale factors`() { + val sut = fixture.getSut() + val customRecorderConfig = ScreenshotRecorderConfig(scaleFactorX = 0.5f, scaleFactorY = 1.5f) + val event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, 100f, 200f, 0) + + val result = sut.convert(event, customRecorderConfig) + + assertNotNull(result) + assertEquals(1, result.size) + assertTrue(result[0] is RRWebInteractionEvent) + with(result[0] as RRWebInteractionEvent) { + assertEquals(1000L, timestamp) + assertEquals(50f, x) // 100 * 0.5 + assertEquals(300f, y) // 200 * 1.5 + assertEquals(0, id) + assertEquals(0, pointerId) + assertEquals(RRWebInteractionEvent.InteractionType.TouchStart, interactionType) + } + + event.recycle() + } + + @Test + fun `convert multi-pointer events`() { + val sut = fixture.getSut() + val recorderConfig = ScreenshotRecorderConfig(scaleFactorX = 1f, scaleFactorY = 1f) + + // Simulate first finger down + var event = MotionEvent.obtain(0, 0, MotionEvent.ACTION_DOWN, 100f, 100f, 0) + var result = sut.convert(event, recorderConfig) + assertNotNull(result) + assertTrue(result[0] is RRWebInteractionEvent) + assertEquals(RRWebInteractionEvent.InteractionType.TouchStart, (result[0] as RRWebInteractionEvent).interactionType) + event.recycle() + + // Simulate second finger down + val properties = MotionEvent.PointerProperties() + properties.id = 1 + properties.toolType = MotionEvent.TOOL_TYPE_FINGER + val pointerProperties = arrayOf(MotionEvent.PointerProperties(), properties) + val pointerCoords = arrayOf( + MotionEvent.PointerCoords().apply { x = 100f; y = 100f }, + MotionEvent.PointerCoords().apply { x = 200f; y = 200f } + ) + event = MotionEvent.obtain(0, 1, MotionEvent.ACTION_POINTER_DOWN or (1 shl MotionEvent.ACTION_POINTER_INDEX_SHIFT), 2, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0) + fixture.now += 100 // Increase time by 100ms + result = sut.convert(event, recorderConfig) + assertNotNull(result) + assertTrue(result[0] is RRWebInteractionEvent) + assertEquals(RRWebInteractionEvent.InteractionType.TouchStart, (result[0] as RRWebInteractionEvent).interactionType) + assertEquals(1, (result[0] as RRWebInteractionEvent).pointerId) + event.recycle() + + // Simulate move event + pointerCoords[0].x = 90f + pointerCoords[0].y = 90f + pointerCoords[1].x = 210f + pointerCoords[1].y = 210f + event = MotionEvent.obtain(0, 2, MotionEvent.ACTION_MOVE, 2, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0) + // First call should not trigger capture + result = sut.convert(event, recorderConfig) + assertNull(result) + + fixture.now += 600 // Increase time by 600ms to trigger move capture + result = sut.convert(event, recorderConfig) + assertNotNull(result) + assertTrue((result[0] as RRWebInteractionMoveEvent).positions!!.size == 2) + event.recycle() + + // Simulate second finger up + event = MotionEvent.obtain(0, 3, MotionEvent.ACTION_POINTER_UP or (1 shl MotionEvent.ACTION_POINTER_INDEX_SHIFT), 2, pointerProperties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0) + fixture.now += 100 // Increase time by 100ms + result = sut.convert(event, recorderConfig) + assertNotNull(result) + assertTrue(result[0] is RRWebInteractionEvent) + assertEquals(RRWebInteractionEvent.InteractionType.TouchEnd, (result[0] as RRWebInteractionEvent).interactionType) + assertEquals(1, (result[0] as RRWebInteractionEvent).pointerId) + event.recycle() + + // Simulate first finger up + event = MotionEvent.obtain(0, 4, MotionEvent.ACTION_UP, 90f, 90f, 0) + fixture.now += 100 // Increase time by 100ms + result = sut.convert(event, recorderConfig) + assertNotNull(result) + assertTrue(result[0] is RRWebInteractionEvent) + assertEquals(RRWebInteractionEvent.InteractionType.TouchEnd, (result[0] as RRWebInteractionEvent).interactionType) + assertEquals(0, (result[0] as RRWebInteractionEvent).pointerId) + event.recycle() + } +}