diff --git a/packages/integration_test/CHANGELOG.md b/packages/integration_test/CHANGELOG.md index d57819e331ea..3bed2b8b73b4 100644 --- a/packages/integration_test/CHANGELOG.md +++ b/packages/integration_test/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.9.0 + +* Add screenshot capability to web tests. + ## 0.8.2 * Add support to get timeline. diff --git a/packages/integration_test/example/test_driver/example_integration_extended.dart b/packages/integration_test/example/test_driver/example_integration_extended.dart new file mode 100644 index 000000000000..79ed2762165e --- /dev/null +++ b/packages/integration_test/example/test_driver/example_integration_extended.dart @@ -0,0 +1,18 @@ +// This is a Flutter widget test can take a screenshot. +// +// NOTE: Screenshots are only supported on Web for now. +// +// To perform an interaction with a widget in your test, use the WidgetTester +// utility that Flutter provides. For example, you can send tap and scroll +// gestures. You can also use WidgetTester to find child widgets in the widget +// tree, read text, and verify that the values of widget properties are correct. + +import 'package:integration_test/integration_test.dart'; + +import 'example_integration_io_extended.dart' + if (dart.library.html) 'example_integration_web_extended.dart' as tests; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + tests.main(); +} diff --git a/packages/integration_test/example/test_driver/example_integration_extended_test.dart b/packages/integration_test/example/test_driver/example_integration_extended_test.dart new file mode 100644 index 000000000000..1428a5092a78 --- /dev/null +++ b/packages/integration_test/example/test_driver/example_integration_extended_test.dart @@ -0,0 +1,14 @@ +import 'dart:async'; + +import 'package:flutter_driver/flutter_driver.dart'; +import 'package:integration_test/integration_test_driver_extended.dart'; + +Future main() async { + final FlutterDriver driver = await FlutterDriver.connect(); + await integrationDriver( + driver: driver, + onScreenshot: (String screenshotName, List screenshotBytes) async { + return true; + }, + ); +} diff --git a/packages/integration_test/example/test_driver/example_integration_io_extended.dart b/packages/integration_test/example/test_driver/example_integration_io_extended.dart new file mode 100644 index 000000000000..56fee6f7179c --- /dev/null +++ b/packages/integration_test/example/test_driver/example_integration_io_extended.dart @@ -0,0 +1,38 @@ +// This is a basic Flutter widget test. +// +// To perform an interaction with a widget in your test, use the WidgetTester +// utility that Flutter provides. For example, you can send tap and scroll +// gestures. You can also use WidgetTester to find child widgets in the widget +// tree, read text, and verify that the values of widget properties are correct. + +import 'dart:io' show Platform; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +import 'package:integration_test_example/main.dart' as app; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('verify text', (WidgetTester tester) async { + // Build our app and trigger a frame. + app.main(); + + // Trigger a frame. + await tester.pumpAndSettle(); + + // TODO: https://github.com/flutter/flutter/issues/51890 + // Add screenshot capability for mobile platforms. + + // Verify that platform version is retrieved. + expect( + find.byWidgetPredicate( + (Widget widget) => + widget is Text && + widget.data.startsWith('Platform: ${Platform.operatingSystem}'), + ), + findsOneWidget, + ); + }); +} diff --git a/packages/integration_test/example/test_driver/example_integration_web_extended.dart b/packages/integration_test/example/test_driver/example_integration_web_extended.dart new file mode 100644 index 000000000000..210c2dac75ba --- /dev/null +++ b/packages/integration_test/example/test_driver/example_integration_web_extended.dart @@ -0,0 +1,52 @@ +// This is a basic Flutter widget test. +// +// To perform an interaction with a widget in your test, use the WidgetTester +// utility that Flutter provides. For example, you can send tap and scroll +// gestures. You can also use WidgetTester to find child widgets in the widget +// tree, read text, and verify that the values of widget properties are correct. + +import 'dart:html' as html; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +import 'package:integration_test_example/main.dart' as app; + +void main() { + final IntegrationTestWidgetsFlutterBinding binding = + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('verify text', (WidgetTester tester) async { + // Build our app and trigger a frame. + app.main(); + + // Trigger a frame. + await tester.pumpAndSettle(); + + // Take a screenshot. + await binding.takeScreenshot('platform_name'); + + // Verify that platform is retrieved. + expect( + find.byWidgetPredicate( + (Widget widget) => + widget is Text && + widget.data + .startsWith('Platform: ${html.window.navigator.platform}\n'), + ), + findsOneWidget, + ); + }); + + testWidgets('verify screenshot', (WidgetTester tester) async { + // Build our app and trigger a frame. + app.main(); + + // Trigger a frame. + await tester.pumpAndSettle(); + + // Multiple methods can take screenshots. Screenshots are taken with the + // same order the methods run. + await binding.takeScreenshot('platform_name_2'); + }); +} diff --git a/packages/integration_test/lib/_callback_io.dart b/packages/integration_test/lib/_callback_io.dart new file mode 100644 index 000000000000..c1a447e27cab --- /dev/null +++ b/packages/integration_test/lib/_callback_io.dart @@ -0,0 +1,62 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'common.dart'; + +/// The dart:io implementation of [CallbackManager]. +/// +/// See also: +/// +/// * [_callback_web.dart], which has the dart:html implementation +CallbackManager get callbackManager => _singletonCallbackManager; + +/// IOCallbackManager singleton. +final IOCallbackManager _singletonCallbackManager = IOCallbackManager(); + +/// Manages communication between `integration_tests` and the `driver_tests`. +/// +/// This is the dart:io implementation. +class IOCallbackManager implements CallbackManager { + @override + Future> callback( + Map params, IntegrationTestResults testRunner) async { + final String command = params['command']; + Map response; + switch (command) { + case 'request_data': + final bool allTestsPassed = await testRunner.allTestsPassed.future; + response = { + 'message': allTestsPassed + ? Response.allTestsPassed(data: testRunner.reportData).toJson() + : Response.someTestsFailed( + testRunner.failureMethodsDetails, + data: testRunner.reportData, + ).toJson(), + }; + break; + case 'get_health': + response = {'status': 'ok'}; + break; + default: + throw UnimplementedError('$command is not implemented'); + } + return { + 'isError': false, + 'response': response, + }; + } + + @override + void cleanup() { + // no-op. + // Add any IO platform specific Completer/Future cleanups to here if any + // comes up in the future. For example: `WebCallbackManager.cleanup`. + } + + @override + Future takeScreenshot(String screenshot) { + throw UnimplementedError( + 'Screenshots are not implemented on this platform'); + } +} diff --git a/packages/integration_test/lib/_callback_web.dart b/packages/integration_test/lib/_callback_web.dart new file mode 100644 index 000000000000..036098148d99 --- /dev/null +++ b/packages/integration_test/lib/_callback_web.dart @@ -0,0 +1,170 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'common.dart'; + +/// The dart:html implementation of [CallbackManager]. +/// +/// See also: +/// +/// * [_callback_io.dart], which has the dart:io implementation +CallbackManager get callbackManager => _singletonWebDriverCommandManager; + +/// WebDriverCommandManager singleton. +final WebCallbackManager _singletonWebDriverCommandManager = + WebCallbackManager(); + +/// Manages communication between `integration_tests` and the `driver_tests`. +/// +/// Along with responding to callbacks from the driver side this calls enables +/// usage of Web Driver commands by sending [WebDriverCommand]s to driver side. +/// +/// Tests can execute an Web Driver commands such as `screenshot` using browsers' +/// WebDriver APIs. +/// +/// See: https://www.w3.org/TR/webdriver/ +class WebCallbackManager implements CallbackManager { + /// App side tests will put the command requests from WebDriver to this pipe. + Completer _webDriverCommandPipe = + Completer(); + + /// Updated when WebDriver completes the request by the test method. + /// + /// For example, a test method will ask for a screenshot by calling + /// `takeScreenshot`. When this screenshot is taken [_driverCommandComplete] + /// will complete. + Completer _driverCommandComplete = Completer(); + + /// Takes screenshot using WebDriver screenshot command. + /// + /// Only works on Web when tests are run via `flutter driver` command. + /// + /// See: https://www.w3.org/TR/webdriver/#screen-capture. + @override + Future takeScreenshot(String screenshotName) async { + await _sendWebDriverCommand(WebDriverCommand.screenshot(screenshotName)); + } + + Future _sendWebDriverCommand(WebDriverCommand command) async { + try { + _webDriverCommandPipe.complete(Future.value(command)); + final bool awaitCommand = await _driverCommandComplete.future; + if (!awaitCommand) { + throw Exception( + 'Web Driver Command ${command.type} failed while waiting for ' + 'driver side'); + } + } catch (exception) { + throw Exception('Web Driver Command failed: ${command.type} with ' + 'exception $exception'); + } finally { + // Reset the completer. + _driverCommandComplete = Completer(); + } + } + + /// The callback function to response the driver side input. + /// + /// Provides a handshake mechanism for executing [WebDriverCommand]s on the + /// driver side. + @override + Future> callback( + Map params, IntegrationTestResults testRunner) async { + final String command = params['command']; + Map response; + switch (command) { + case 'request_data': + return params['message'] == null + ? _requestData(testRunner) + : _requestDataWithMessage(params['message'], testRunner); + break; + case 'get_health': + response = {'status': 'ok'}; + break; + default: + throw UnimplementedError('$command is not implemented'); + } + return { + 'isError': false, + 'response': response, + }; + } + + Future> _requestDataWithMessage( + String extraMessage, IntegrationTestResults testRunner) async { + Map response; + // Driver side tests' status is added as an extra message. + final DriverTestMessage message = + DriverTestMessage.fromString(extraMessage); + // If driver side tests are pending send the first command in the + // `commandPipe` to the tests. + if (message.isPending) { + final WebDriverCommand command = await _webDriverCommandPipe.future; + switch (command.type) { + case WebDriverCommandType.screenshot: + final Map data = Map.from(command.values); + data.addAll( + WebDriverCommand.typeToMap(WebDriverCommandType.screenshot)); + response = { + 'message': Response.webDriverCommand(data: data).toJson(), + }; + break; + case WebDriverCommandType.noop: + final Map data = Map(); + data.addAll(WebDriverCommand.typeToMap(WebDriverCommandType.noop)); + response = { + 'message': Response.webDriverCommand(data: data).toJson(), + }; + break; + default: + throw UnimplementedError('${command.type} is not implemented'); + } + } else { + final Map data = Map(); + data.addAll(WebDriverCommand.typeToMap(WebDriverCommandType.ack)); + response = { + 'message': Response.webDriverCommand(data: data).toJson(), + }; + _driverCommandComplete.complete(Future.value(message.isSuccess)); + _webDriverCommandPipe = Completer(); + } + return { + 'isError': false, + 'response': response, + }; + } + + Future> _requestData( + IntegrationTestResults testRunner) async { + final bool allTestsPassed = await testRunner.allTestsPassed.future; + final Map response = { + 'message': allTestsPassed + ? Response.allTestsPassed(data: testRunner.reportData).toJson() + : Response.someTestsFailed( + testRunner.failureMethodsDetails, + data: testRunner.reportData, + ).toJson(), + }; + return { + 'isError': false, + 'response': response, + }; + } + + @override + void cleanup() { + if (!_webDriverCommandPipe.isCompleted) { + _webDriverCommandPipe + .complete(Future.value(WebDriverCommand.noop())); + } + + if (!_driverCommandComplete.isCompleted) { + _driverCommandComplete.complete(Future.value(false)); + } + } +} diff --git a/packages/integration_test/lib/common.dart b/packages/integration_test/lib/common.dart index 789b1fa54948..53714a8e97ee 100644 --- a/packages/integration_test/lib/common.dart +++ b/packages/integration_test/lib/common.dart @@ -2,8 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; import 'dart:convert'; +/// Classes shared between `integration_test.dart` and `flutter drive` based +/// adoptor (ex: `integration_test_driver.dart`). + /// An object sent from integration_test back to the Flutter Driver in response to /// `request_data` command. class Response { @@ -23,6 +27,16 @@ class Response { Response.someTestsFailed(this._failureDetails, {this.data}) : this._allTestsPassed = false; + /// Constructor for failure response. + Response.toolException({String ex}) + : this._allTestsPassed = false, + this._failureDetails = [Failure('ToolException', ex)]; + + /// Constructor for web driver commands response. + Response.webDriverCommand({this.data}) + : this._allTestsPassed = false, + this._failureDetails = null; + /// Whether the test ran successfully or not. bool get allTestsPassed => _allTestsPassed; @@ -123,3 +137,165 @@ class Failure { return Failure(failure['methodName'], failure['details']); } } + +/// Message used to communicate between app side tests and driver tests. +/// +/// Not all `integration_tests` use this message. They are only used when app +/// side tests are sending [WebDriverCommand]s to the driver side. +/// +/// These messages are used for the handshake since they carry information on +/// the driver side test such as: status pending or tests failed. +class DriverTestMessage { + final bool _isSuccess; + final bool _isPending; + + /// When tests are failed on the driver side. + DriverTestMessage.error() + : _isSuccess = false, + _isPending = false; + + /// When driver side is waiting on [WebDriverCommand]s to be sent from the + /// app side. + DriverTestMessage.pending() + : _isSuccess = false, + _isPending = true; + + /// When driver side successfully completed executing the [WebDriverCommand]. + DriverTestMessage.complete() + : _isSuccess = true, + _isPending = false; + + // /// Status of this message. + // /// + // /// The status will be use to notify `integration_test` of driver side's + // /// state. + // String get status => _status; + + /// Has the command completed successfully by the driver. + bool get isSuccess => _isSuccess; + + /// Is the driver waiting for a command. + bool get isPending => _isPending; + + /// Depending on the values of [isPending] and [isSuccess], returns a string + /// to represent the [DriverTestMessage]. + /// + /// Used as an alternative method to converting the object to json since + /// [RequestData] is only accepting string as `message`. + @override + String toString() { + if (isPending) { + return 'pending'; + } else if (isSuccess) { + return 'complete'; + } else { + return 'error'; + } + } + + /// Return a DriverTestMessage depending on `status`. + static DriverTestMessage fromString(String status) { + switch (status) { + case 'error': + return DriverTestMessage.error(); + case 'pending': + return DriverTestMessage.pending(); + case 'complete': + return DriverTestMessage.complete(); + default: + throw StateError('This type of status does not exist: $status'); + } + } +} + +/// Types of different WebDriver commands that can be used in web integration +/// tests. +/// +/// These commands are either commands that WebDriver can execute or used +/// for the communication between `integration_test` and the driver test. +enum WebDriverCommandType { + /// Acknowlegement for the previously sent message. + ack, + + /// No further WebDriver commands is requested by the app-side tests. + noop, + + /// Asking WebDriver to take a screenshot of the Web page. + screenshot, +} + +/// Command for WebDriver to execute. +/// +/// Only works on Web when tests are run via `flutter driver` command. +/// +/// See: https://www.w3.org/TR/webdriver/ +class WebDriverCommand { + /// Type of the [WebDriverCommand]. + /// + /// Currently the only command that triggers a WebDriver API is `screenshot`. + /// + /// There are also `ack` and `noop` commands defined to manage the handshake + /// during the communication. + final WebDriverCommandType type; + + /// Used for adding extra values to the commands such as file name for + /// `screenshot`. + final Map values; + + /// Constructor for [WebDriverCommandType.noop] command. + WebDriverCommand.noop() + : this.type = WebDriverCommandType.noop, + this.values = Map(); + + /// Constructor for [WebDriverCommandType.noop] screenshot. + WebDriverCommand.screenshot(String screenshot_name) + : this.type = WebDriverCommandType.screenshot, + this.values = {'screenshot_name': screenshot_name}; + + /// Util method for converting [WebDriverCommandType] to a map entry. + /// + /// Used for converting messages to json format. + static Map typeToMap(WebDriverCommandType type) => { + 'web_driver_command': '${type}', + }; +} + +/// Template methods each class that responses the driver side inputs must +/// implement. +/// +/// Depending on the platform the communication between `integration_tests` and +/// the `driver_tests` can be different. +/// +/// For the web implementation [WebCallbackManager]. +/// For the io implementation [IOCallbackManager]. +abstract class CallbackManager { + /// The callback function to response the driver side input. + Future> callback( + Map params, IntegrationTestResults testRunner); + + /// Request to take a screenshot of the application. + Future takeScreenshot(String screenshot); + + /// Cleanup and completers or locks used during the communication. + void cleanup(); +} + +/// Interface that surfaces test results of integration tests. +/// +/// Implemented by [IntegrationTestWidgetsFlutterBinding]s. +/// +/// Any class which needs to access the test results but do not want to create +/// a cyclic dependency [IntegrationTestWidgetsFlutterBinding]s can use this +/// interface. Example [CallbackManager]. +abstract class IntegrationTestResults { + /// Stores failure details. + /// + /// Failed test method's names used as key. + List get failureMethodsDetails; + + /// The extra data for the reported result. + Map get reportData; + + /// Whether all the test methods completed succesfully. + Completer get allTestsPassed; +} diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index f6980bc9d6d1..4176c1c2c5e1 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -15,13 +15,15 @@ import 'package:vm_service/vm_service_io.dart' as vm_io; import 'common.dart'; import '_extension_io.dart' if (dart.library.html) '_extension_web.dart'; +import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' + as driver_actions; const String _success = 'success'; /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results /// on a channel to adapt them to native instrumentation test format. -class IntegrationTestWidgetsFlutterBinding - extends LiveTestWidgetsFlutterBinding { +class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding + implements IntegrationTestResults { /// Sets up a listener to report that the tests are finished when everything is /// torn down. IntegrationTestWidgetsFlutterBinding() { @@ -36,6 +38,7 @@ class IntegrationTestWidgetsFlutterBinding _allTestsPassed.complete(true); } } + callbackManager.cleanup(); await _channel.invokeMethod( 'allTestsFinished', { @@ -104,8 +107,13 @@ class IntegrationTestWidgetsFlutterBinding ); } + @override + Completer get allTestsPassed => _allTestsPassed; final Completer _allTestsPassed = Completer(); + @override + List get failureMethodsDetails => _failures; + /// Similar to [WidgetsFlutterBinding.ensureInitialized]. /// /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and @@ -136,35 +144,27 @@ class IntegrationTestWidgetsFlutterBinding /// If it's `null`, no extra data is attached to the result. /// /// The default value is `null`. - Map reportData; + @override + Map get reportData => _reportData; + Map _reportData; + set reportData(Map data) => this._reportData = data; + + /// Manages callbacks received from driver side and commands send to driver + /// side. + final CallbackManager callbackManager = driver_actions.callbackManager; - /// the callback function to response the driver side input. + /// Taking a screenshot. + /// + /// Called by test methods. Implementation differs for each platform. + Future takeScreenshot(String screenshotName) async { + await callbackManager.takeScreenshot(screenshotName); + } + + /// The callback function to response the driver side input. @visibleForTesting Future> callback(Map params) async { - final String command = params['command']; - Map response; - switch (command) { - case 'request_data': - final bool allTestsPassed = await _allTestsPassed.future; - response = { - 'message': allTestsPassed - ? Response.allTestsPassed(data: reportData).toJson() - : Response.someTestsFailed( - _failures, - data: reportData, - ).toJson(), - }; - break; - case 'get_health': - response = {'status': 'ok'}; - break; - default: - throw UnimplementedError('$command is not implemented'); - } - return { - 'isError': false, - 'response': response, - }; + return await callbackManager.callback( + params, this /* as IntegrationTestResults */); } // Emulates the Flutter driver extension, returning 'pass' or 'fail'. diff --git a/packages/integration_test/lib/integration_test_driver_extended.dart b/packages/integration_test/lib/integration_test_driver_extended.dart new file mode 100644 index 000000000000..bc38bb71de50 --- /dev/null +++ b/packages/integration_test/lib/integration_test_driver_extended.dart @@ -0,0 +1,75 @@ +import 'dart:async'; +import 'dart:io'; + +import 'common.dart'; + +import 'package:flutter_driver/flutter_driver.dart'; + +/// Example Integration Test which can also run WebDriver command depending on +/// the requests coming from the test methods. +Future integrationDriver( + {FlutterDriver driver, Function onScreenshot}) async { + if (driver == null) { + driver = await FlutterDriver.connect(); + } + // Test states that it's waiting on web driver commands. + // [DriverTestMessage] is converted to string since json format causes an + // error if it's used as a message for requestData. + String jsonResponse = + await driver.requestData(DriverTestMessage.pending().toString()); + + Response response = Response.fromJson(jsonResponse); + + // Until `integration_test` returns a [WebDriverCommandType.noop], keep + // executing WebDriver commands. + while (response.data != null && + response.data['web_driver_command'] != null && + response.data['web_driver_command'] != '${WebDriverCommandType.noop}') { + final String webDriverCommand = response.data['web_driver_command']; + if (webDriverCommand == '${WebDriverCommandType.screenshot}') { + // Use `driver.screenshot()` method to get a screenshot of the web page. + final List screenshotImage = await driver.screenshot(); + final String screenshotName = response.data['screenshot_name']; + + final bool screenshotSuccess = + await onScreenshot(screenshotName, screenshotImage); + if (screenshotSuccess) { + jsonResponse = + await driver.requestData(DriverTestMessage.complete().toString()); + } else { + jsonResponse = + await driver.requestData(DriverTestMessage.error().toString()); + } + + response = Response.fromJson(jsonResponse); + } else if (webDriverCommand == '${WebDriverCommandType.ack}') { + // Previous command completed ask for a new one. + jsonResponse = + await driver.requestData(DriverTestMessage.pending().toString()); + + response = Response.fromJson(jsonResponse); + } else { + break; + } + } + + // If No-op command is sent, ask for the result of all tests. + if (response.data != null && + response.data['web_driver_command'] != null && + response.data['web_driver_command'] == '${WebDriverCommandType.noop}') { + jsonResponse = await driver.requestData(null); + + response = Response.fromJson(jsonResponse); + print('result $jsonResponse'); + } + + await driver.close(); + + if (response.allTestsPassed) { + print('All tests passed.'); + exit(0); + } else { + print('Failure Details:\n${response.formattedFailureDetails}'); + exit(1); + } +} diff --git a/packages/integration_test/pubspec.yaml b/packages/integration_test/pubspec.yaml index 9dd4ade6ce49..6e0412924748 100644 --- a/packages/integration_test/pubspec.yaml +++ b/packages/integration_test/pubspec.yaml @@ -1,6 +1,6 @@ name: integration_test description: Runs tests that use the flutter_test API as integration tests. -version: 0.8.2 +version: 0.9.0 homepage: https://github.com/flutter/plugins/tree/master/packages/integration_test environment: