Skip to content

Commit

Permalink
feat: ensure repaint on replay screenshot capture (#2527)
Browse files Browse the repository at this point in the history 
* feat: ensure repaint on replay screenshot capture

* chore: update changelog

* linter issue
  • Loading branch information
@vaind
vaind authored Dec 20, 2024
1 parent 19a9adb commit c689845
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 112 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

- Check `SentryTracer` type in TTFD tracker ([#2508](https://github.com/getsentry/sentry-dart/pull/2508))
- Warning (in a debug build) if a potentially sensitive widget is not masked or unmasked explicitly ([#2375](https://github.com/getsentry/sentry-dart/pull/2375))
- Replay: ensure visual update before capturing screenshots ([#2527](https://github.com/getsentry/sentry-dart/pull/2527))

### Dependencies

Expand Down
43 changes: 28 additions & 15 deletions flutter/lib/src/native/cocoa/sentry_native_cocoa.dart
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import 'dart:async';
import 'dart:ffi';
import 'dart:typed_data';
import 'dart:ui';

import 'package:meta/meta.dart';
Expand Down Expand Up @@ -45,22 +46,34 @@ class SentryNativeCocoa extends SentryNativeChannel {
});
}

return _replayRecorder?.capture((screenshot) async {
final image = screenshot.image;
final imageData =
await image.toByteData(format: ImageByteFormat.png);
if (imageData != null) {
options.logger(
SentryLevel.debug,
'Replay: captured screenshot ('
'${image.width}x${image.height} pixels, '
'${imageData.lengthInBytes} bytes)');
return imageData.buffer.asUint8List();
} else {
options.logger(SentryLevel.warning,
'Replay: failed to convert screenshot to PNG');
}
final widgetsBinding = options.bindingUtils.instance;
if (widgetsBinding == null) {
options.logger(SentryLevel.warning,
'Replay: failed to capture screenshot, WidgetsBinding.instance is null');
return null;
}

final completer = Completer<Uint8List?>();
widgetsBinding.ensureVisualUpdate();
widgetsBinding.addPostFrameCallback((_) {
_replayRecorder?.capture((screenshot) async {
final image = screenshot.image;
final imageData =
await image.toByteData(format: ImageByteFormat.png);
if (imageData != null) {
options.logger(
SentryLevel.debug,
'Replay: captured screenshot ('
'${image.width}x${image.height} pixels, '
'${imageData.lengthInBytes} bytes)');
return imageData.buffer.asUint8List();
} else {
options.logger(SentryLevel.warning,
'Replay: failed to convert screenshot to PNG');
}
}).then(completer.complete, onError: completer.completeError);
});
return completer.future;
default:
throw UnimplementedError('Method ${call.method} not implemented');
}
Expand Down
150 changes: 79 additions & 71 deletions flutter/lib/src/replay/scheduled_recorder.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
late final ScheduledScreenshotRecorderCallback _callback;
var _status = _Status.running;
late final Duration _frameDuration;
late final _idleFrameFiller = _IdleFrameFiller(_frameDuration, _onScreenshot);
// late final _idleFrameFiller = _IdleFrameFiller(_frameDuration, _onScreenshot);

@override
@protected
Expand All @@ -35,8 +35,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
_frameDuration = Duration(milliseconds: 1000 ~/ config.frameRate);
assert(_frameDuration.inMicroseconds > 0);

_scheduler = Scheduler(_frameDuration, _capture,
options.bindingUtils.instance!.addPostFrameCallback);
_scheduler = Scheduler(_frameDuration, _capture, _addPostFrameCallback);

if (callback != null) {
_callback = callback;
Expand All @@ -47,6 +46,12 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
_callback = callback;
}

void _addPostFrameCallback(FrameCallback callback) {
options.bindingUtils.instance!
..ensureVisualUpdate()
..addPostFrameCallback(callback);
}

void start() {
assert(() {
// The following fails if callback hasn't been provided
Expand Down Expand Up @@ -74,14 +79,15 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
Future<void> stop() async {
options.logger(SentryLevel.debug, "$logName: stopping capture.");
_status = _Status.stopped;
await Future.wait([_scheduler.stop(), _idleFrameFiller.stop()]);
await _scheduler.stop();
// await Future.wait([_scheduler.stop(), _idleFrameFiller.stop()]);
options.logger(SentryLevel.debug, "$logName: capture stopped.");
}

Future<void> pause() async {
if (_status == _Status.running) {
_status = _Status.paused;
_idleFrameFiller.pause();
// _idleFrameFiller.pause();
await _scheduler.stop();
}
}
Expand All @@ -90,7 +96,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
if (_status == _Status.paused) {
_status = _Status.running;
_startScheduler();
_idleFrameFiller.resume();
// _idleFrameFiller.resume();
}
}

Expand All @@ -104,7 +110,7 @@ class ScheduledScreenshotRecorder extends ReplayScreenshotRecorder {
final screenshot = ScreenshotPng(
image.width, image.height, imageData, capturedScreenshot.timestamp);
await _onScreenshot(screenshot, true);
_idleFrameFiller.actualFrameReceived(screenshot);
// _idleFrameFiller.actualFrameReceived(screenshot);
} else {
options.logger(
SentryLevel.debug,
Expand Down Expand Up @@ -140,69 +146,71 @@ class ScreenshotPng {

const ScreenshotPng(this.width, this.height, this.data, this.timestamp);
}

// Workaround for https://github.com/getsentry/sentry-java/issues/3677
// In short: when there are no postFrameCallbacks issued by Flutter (because
// there are no animations or user interactions), the replay recorder will
// need to get screenshots at a fixed frame rate. This class is responsible for
// filling the gaps between actual frames with the most recent frame.
class _IdleFrameFiller {
final Duration _interval;
final ScheduledScreenshotRecorderCallback _callback;
var _status = _Status.running;
Future<void>? _scheduled;
ScreenshotPng? _mostRecent;

_IdleFrameFiller(this._interval, this._callback);

void actualFrameReceived(ScreenshotPng screenshot) {
// We store the most recent frame but only repost it when the most recent
// one is the same instance (unchanged).
_mostRecent = screenshot;
// Also, the initial reposted frame will be delayed to allow actual frames
// to cancel the reposting.
repostLater(_interval * 1.5, screenshot);
}

Future<void> stop() async {
_status = _Status.stopped;
final scheduled = _scheduled;
_scheduled = null;
_mostRecent = null;
await scheduled;
}

void pause() {
if (_status == _Status.running) {
_status = _Status.paused;
}
}

void resume() {
if (_status == _Status.paused) {
_status = _Status.running;
}
}

void repostLater(Duration delay, ScreenshotPng screenshot) {
_scheduled = Future.delayed(delay, () async {
if (_status == _Status.stopped) {
return;
}

// Only repost if the screenshot haven't changed.
if (screenshot == _mostRecent) {
if (_status == _Status.running) {
// We don't strictly need to await here but it helps to reduce load.
// If the callback takes a long time, we still wait between calls,
// based on the configured rate.
await _callback(screenshot, false);
}
// On subsequent frames, we stick to the actual frame rate.
repostLater(_interval, screenshot);
}
});
}
}
// TODO this is currently unused because we've decided to capture on every
// frame. Consider removing if we don't reverse the decision in the future.

// /// Workaround for https://github.com/getsentry/sentry-java/issues/3677
// /// In short: when there are no postFrameCallbacks issued by Flutter (because
// /// there are no animations or user interactions), the replay recorder will
// /// need to get screenshots at a fixed frame rate. This class is responsible for
// /// filling the gaps between actual frames with the most recent frame.
// class _IdleFrameFiller {
// final Duration _interval;
// final ScheduledScreenshotRecorderCallback _callback;
// var _status = _Status.running;
// Future<void>? _scheduled;
// ScreenshotPng? _mostRecent;

// _IdleFrameFiller(this._interval, this._callback);

// void actualFrameReceived(ScreenshotPng screenshot) {
// // We store the most recent frame but only repost it when the most recent
// // one is the same instance (unchanged).
// _mostRecent = screenshot;
// // Also, the initial reposted frame will be delayed to allow actual frames
// // to cancel the reposting.
// repostLater(_interval * 1.5, screenshot);
// }

// Future<void> stop() async {
// _status = _Status.stopped;
// final scheduled = _scheduled;
// _scheduled = null;
// _mostRecent = null;
// await scheduled;
// }

// void pause() {
// if (_status == _Status.running) {
// _status = _Status.paused;
// }
// }

// void resume() {
// if (_status == _Status.paused) {
// _status = _Status.running;
// }
// }

// void repostLater(Duration delay, ScreenshotPng screenshot) {
// _scheduled = Future.delayed(delay, () async {
// if (_status == _Status.stopped) {
// return;
// }

// // Only repost if the screenshot haven't changed.
// if (screenshot == _mostRecent) {
// if (_status == _Status.running) {
// // We don't strictly need to await here but it helps to reduce load.
// // If the callback takes a long time, we still wait between calls,
// // based on the configured rate.
// await _callback(screenshot, false);
// }
// // On subsequent frames, we stick to the actual frame rate.
// repostLater(_interval, screenshot);
// }
// });
// }
// }

enum _Status { stopped, running, paused }
41 changes: 15 additions & 26 deletions flutter/test/replay/replay_native_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ void main() {
MockPlatform.android(),
MockPlatform.iOs(),
]) {
group('$SentryNativeBinding ($mockPlatform)', () {
group('$SentryNativeBinding (${mockPlatform.operatingSystem})', () {
late SentryNativeBinding sut;
late NativeChannelFixture native;
late SentryFlutterOptions options;
Expand All @@ -45,7 +45,7 @@ void main() {
'directory': 'dir',
'width': 800,
'height': 600,
'frameRate': 10,
'frameRate': 1000,
};
}

Expand Down Expand Up @@ -116,14 +116,15 @@ void main() {
testWidgets('captures images', (tester) async {
await tester.runAsync(() async {
when(hub.configureScope(captureAny)).thenReturn(null);
await pumpTestElement(tester);
pumpAndSettle() => tester.pumpAndSettle(const Duration(seconds: 1));

if (mockPlatform.isAndroid) {
var callbackFinished = Completer<void>();

nextFrame({bool wait = true}) async {
final future = callbackFinished.future;
tester.binding.scheduleFrame();
await tester.pumpAndSettle(const Duration(seconds: 1));
await pumpAndSettle();
await future.timeout(Duration(milliseconds: wait ? 1000 : 100),
onTimeout: () {
if (wait) {
Expand All @@ -132,29 +133,24 @@ void main() {
});
}

imageInfo(File file) => file.readAsBytesSync().length;

fileToImageMap(Iterable<File> files) =>
{for (var file in files) file.path: imageInfo(file)};
imageSizeBytes(File file) => file.readAsBytesSync().length;

final capturedImages = <String, int>{};
when(native.handler('addReplayScreenshot', any))
.thenAnswer((invocation) async {
.thenAnswer((invocation) {
final path =
invocation.positionalArguments[1]["path"] as String;
capturedImages[path] = imageInfo(fs.file(path));
capturedImages[path] = imageSizeBytes(fs.file(path));
callbackFinished.complete();
callbackFinished = Completer<void>();
return null;
});

fsImages() {
final files = replayDir.listSync().map((f) => f as File);
return fileToImageMap(files);
return {for (var f in files) f.path: imageSizeBytes(f)};
}

await pumpTestElement(tester);

await nextFrame(wait: false);
expect(fsImages(), isEmpty);
verifyNever(native.handler('addReplayScreenshot', any));
Expand Down Expand Up @@ -202,24 +198,17 @@ void main() {
expect(capturedImages, equals(fsImages()));
expect(capturedImages.length, count);
} else if (mockPlatform.isIOS) {
nextFrame() async {
tester.binding.scheduleFrame();
await Future<void>.delayed(const Duration(milliseconds: 100));
await tester.pumpAndSettle(const Duration(seconds: 1));
}

await pumpTestElement(tester);
await nextFrame();

final imagaData = await native.invokeFromNative(
var imagaData = native.invokeFromNative(
'captureReplayScreenshot', replayConfig);
expect(imagaData?.lengthInBytes, greaterThan(3000));
await pumpAndSettle();
expect((await imagaData)?.lengthInBytes, greaterThan(3000));

// Happens if the session-replay rate is 0.
replayConfig['replayId'] = null;
final imagaData2 = await native.invokeFromNative(
imagaData = native.invokeFromNative(
'captureReplayScreenshot', replayConfig);
expect(imagaData2?.lengthInBytes, greaterThan(3000));
await pumpAndSettle();
expect((await imagaData)?.lengthInBytes, greaterThan(3000));
} else {
fail('unsupported platform');
}
Expand Down

0 comments on commit c689845

Please sign in to comment.