Disable sandboxing for macOS apps and tests in CI (#149618)

macOS 14 added new requirements that un-codesigned sandbox apps must be granted access when changed. Waiting for this UI caused macOS tests to fail on macOS 14 because the test runner forced codesigning off. Additionally, adding codesigning is not sufficient, since it must still be approved before codesigning is enough to pass the check. As a workaround, this PR disables sandboxing for macOS apps/tests in CI.

![Screenshot 2024-05-30 at 2 41 33�PM](https://github.com/flutter/flutter/assets/682784/1bc32620-5edb-420a-866c-5cc529b2ac55)

https://developer.apple.com/documentation/updates/security#June-2023)
> App Sandbox now associates your macOS app with its sandbox container using its code signature. The operating system asks the person using your app to grant permission if it tries to access a sandbox container associated with a different app. For more information, see [Accessing files from the macOS App Sandbox](https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox).

And that link explains why this is happening on a macOS 14 update:

> In macOS 14 and later, the operating system uses your app�s code signature to associate it with its sandbox container. If your app tries to access the sandbox container owned by another app, the system asks the person using your app whether to grant access. If the person denies access and your app is already running, then it can�t read or write the files in the other app�s sandbox container. If the person denies access while your app is launching and trying to enter the other app�s sandbox container, your app fails to launch.
> 
> The operating system also tracks the association between an app�s code signing identity and its sandbox container for helper tools, including launch agents. If a person denies permission for a launch agent to enter its sandbox container and the app fails to start, launchd starts the launch agent again and the operating system re-requests access.

Fixes flutter/flutter#149268.
Fixes framework part of flutter/flutter#149264.
Might fix packages issue: flutter/flutter#149329.

Verified framework tests:
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac%20plugin_test_macos/9/overview
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac%20run_debug_test_macos/2/overview
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac%20tool_integration_tests_4_4/2/overview
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac%20integration_ui_test_test_macos/3/overview
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac%20flavors_test_macos/3/overview
https://ci.chromium.org/ui/p/flutter/builders/staging.shadow/Mac_benchmark%20complex_layout_scroll_perf_macos__timeline_summary/6/overview

Author: vashworth

dev/devicelab/lib/framework/ios.dart

@@ -191,6 +191,13 @@ Future<bool> runXcodeTests({
     codeSignStyle = environment['FLUTTER_XCODE_CODE_SIGN_STYLE'];
     provisioningProfile = environment['FLUTTER_XCODE_PROVISIONING_PROFILE_SPECIFIER'];
   }
+  File? disabledSandboxEntitlementFile;
+  if (platformDirectory.endsWith('macos')) {
+    disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
+      platformDirectory,
+      configuration,
+    );
+  }
   final String resultBundleTemp = Directory.systemTemp.createTempSync('flutter_xcresult.').path;
   final String resultBundlePath = path.join(resultBundleTemp, 'result');
   final int testResultExit = await exec(
@@ -214,6 +221,8 @@ Future<bool> runXcodeTests({
         'CODE_SIGN_STYLE=$codeSignStyle',
       if (provisioningProfile != null)
         'PROVISIONING_PROFILE_SPECIFIER=$provisioningProfile',
+      if (disabledSandboxEntitlementFile != null)
+        'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
     ],
     workingDirectory: platformDirectory,
     canFail: true,
@@ -247,3 +256,55 @@ Future<bool> runXcodeTests({
   }
   return true;
 }
+
+/// Finds and copies macOS entitlements file. In the copy, disables sandboxing.
+/// If entitlements file is not found, returns null.
+///
+/// As of macOS 14, testing a macOS sandbox app may prompt the user to grant
+/// access to the app. To workaround this in CI, we create and use a entitlements
+/// file with sandboxing disabled. See
+/// https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox.
+File? _createDisabledSandboxEntitlementFile(
+  String platformDirectory,
+  String configuration,
+) {
+  String entitlementDefaultFileName;
+  if (configuration == 'Release') {
+    entitlementDefaultFileName = 'Release';
+  } else {
+    entitlementDefaultFileName = 'DebugProfile';
+  }
+
+  final String entitlementFilePath = path.join(
+    platformDirectory,
+    'Runner',
+    '$entitlementDefaultFileName.entitlements',
+  );
+  final File entitlementFile = File(entitlementFilePath);
+
+  if (!entitlementFile.existsSync()) {
+    print('Unable to find entitlements file at ${entitlementFile.path}');
+    return null;
+  }
+
+  final String originalEntitlementFileContents =
+      entitlementFile.readAsStringSync();
+  final String tempEntitlementPath = Directory.systemTemp
+      .createTempSync('flutter_disable_sandbox_entitlement.')
+      .path;
+  final File disabledSandboxEntitlementFile = File(path.join(
+    tempEntitlementPath,
+    '${entitlementDefaultFileName}WithDisabledSandboxing.entitlements',
+  ));
+  disabledSandboxEntitlementFile.createSync(recursive: true);
+  disabledSandboxEntitlementFile.writeAsStringSync(
+    originalEntitlementFileContents.replaceAll(
+      RegExp(r'<key>com\.apple\.security\.app-sandbox<\/key>[\S\s]*?<true\/>'),
+      '''
+<key>com.apple.security.app-sandbox</key>
+	<false/>''',
+    ),
+  );
+
+  return disabledSandboxEntitlementFile;
+}

packages/flutter_tools/lib/src/commands/build_macos.dart

@@ -72,6 +72,7 @@ class BuildMacosCommand extends BuildSubCommand {
         flutterUsage: globals.flutterUsage,
         analytics: analytics,
       ),
+      usingCISystem: usingCISystem,
     );
     return FlutterCommandResult.success();
   }

packages/flutter_tools/lib/src/desktop_device.dart

@@ -16,6 +16,8 @@ import 'convert.dart';
 import 'devfs.dart';
 import 'device.dart';
 import 'device_port_forwarder.dart';
+import 'globals.dart' as globals;
+import 'macos/macos_device.dart';
 import 'protocol_discovery.dart';
 
 /// A partial implementation of Device for desktop-class devices to inherit
@@ -119,6 +121,7 @@ abstract class DesktopDevice extends Device {
       await buildForDevice(
         buildInfo: debuggingOptions.buildInfo,
         mainPath: mainPath,
+        usingCISystem: debuggingOptions.usingCISystem,
       );
     }
 
@@ -159,8 +162,39 @@ abstract class DesktopDevice extends Device {
       logger: _logger,
     );
     try {
+      Timer? timer;
+      if (this is MacOSDevice) {
+        if (await globals.isRunningOnBot) {
+          const int defaultTimeout = 5;
+          timer = Timer(const Duration(minutes: defaultTimeout), () {
+            // As of macOS 14, if sandboxing is enabled and the app is not codesigned,
+            // a dialog will prompt the user to allow the app to run. This will
+            // cause tests in CI to hang. In CI, we workaround this by setting
+            // the CODE_SIGN_ENTITLEMENTS build setting to a version with
+            // sandboxing disabled.
+            final String sandboxingMessage;
+            if (debuggingOptions.usingCISystem) {
+              sandboxingMessage = 'Ensure sandboxing is disabled by checking '
+                  'the set CODE_SIGN_ENTITLEMENTS.';
+            } else {
+              sandboxingMessage = 'Consider codesigning your app or disabling '
+                  'sandboxing. Flutter will attempt to disable sandboxing if '
+                  'the `--ci` flag is provided.';
+            }
+            _logger.printError(
+                'The Dart VM Service was not discovered after $defaultTimeout '
+                'minutes. If the app has sandboxing enabled and is not '
+                'codesigned or codesigning changed, this may be caused by a '
+                'system prompt asking for access. $sandboxingMessage\n'
+                'See https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox '
+                'for more information.');
+          });
+        }
+      }
+
       final Uri? vmServiceUri = await vmServiceDiscovery.uri;
       if (vmServiceUri != null) {
+        timer?.cancel();
         onAttached(package, buildInfo, process);
         return LaunchResult.succeeded(vmServiceUri: vmServiceUri);
       }
@@ -199,6 +233,7 @@ abstract class DesktopDevice extends Device {
   Future<void> buildForDevice({
     required BuildInfo buildInfo,
     String? mainPath,
+    bool usingCISystem = false,
   });
 
   /// Returns the path to the executable to run for [package] on this device for

packages/flutter_tools/lib/src/linux/linux_device.dart

@@ -62,6 +62,7 @@ class LinuxDevice extends DesktopDevice {
   Future<void> buildForDevice({
     String? mainPath,
     required BuildInfo buildInfo,
+    bool usingCISystem = false,
   }) async {
     await buildLinux(
       FlutterProject.current().linux,

packages/flutter_tools/lib/src/macos/build_macos.dart

@@ -65,6 +65,7 @@ Future<void> buildMacOS({
   required bool verboseLogging,
   bool configOnly = false,
   SizeAnalyzer? sizeAnalyzer,
+  bool usingCISystem = false,
 }) async {
   final Directory? xcodeWorkspace = flutterProject.macos.xcodeWorkspace;
   if (xcodeWorkspace == null) {
@@ -153,6 +154,19 @@ Future<void> buildMacOS({
     'Building macOS application...',
   );
   int result;
+
+  File? disabledSandboxEntitlementFile;
+  if (usingCISystem) {
+    disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
+      flutterProject.macos,
+      configuration,
+    );
+    if (disabledSandboxEntitlementFile != null) {
+      globals.logger.printStatus(
+        'Detected macOS app running in CI, turning off sandboxing.');
+    }
+  }
+
   try {
     result = await globals.processUtils.stream(<String>[
       '/usr/bin/env',
@@ -170,6 +184,8 @@ Future<void> buildMacOS({
       else
         '-quiet',
       'COMPILER_INDEX_STORE_ENABLE=NO',
+      if (disabledSandboxEntitlementFile != null)
+        'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
       ...environmentVariablesAsXcodeBuildSettings(globals.platform),
     ],
     trace: true,
@@ -271,3 +287,52 @@ Future<void> _writeCodeSizeAnalysis(BuildInfo buildInfo, SizeAnalyzer? sizeAnaly
     'dart devtools --appSizeBase=$relativeAppSizePath'
   );
 }
+
+/// Finds and copies macOS entitlements file. In the copy, disables sandboxing.
+/// If entitlements file is not found, returns null.
+///
+/// As of macOS 14, running a macOS sandbox app may prompt the user to grant
+/// access to the app. To workaround this in CI, we create and use a entitlements
+/// file with sandboxing disabled. See
+/// https://developer.apple.com/documentation/security/app_sandbox/accessing_files_from_the_macos_app_sandbox.
+File? _createDisabledSandboxEntitlementFile(
+  MacOSProject macos,
+  String configuration,
+) {
+  String entitlementDefaultFileName;
+  if (configuration == 'Release') {
+    entitlementDefaultFileName = 'Release';
+  } else {
+    entitlementDefaultFileName = 'DebugProfile';
+  }
+
+  // TODO(vashworth): Once https://github.com/flutter/flutter/issues/146204 is
+  // fixed, it would be better to get the path to the entitlement file from the
+  // project's build settings (CODE_SIGN_ENTITLEMENTS).
+  final File entitlementFile = macos.hostAppRoot
+      .childDirectory('Runner')
+      .childFile('$entitlementDefaultFileName.entitlements');
+
+  if (!entitlementFile.existsSync()) {
+    globals.logger.printTrace(
+        'Unable to find entitlements file at ${entitlementFile.path}');
+    return null;
+  }
+
+  final String entitlementFileContents = entitlementFile.readAsStringSync();
+  final File disabledSandboxEntitlementFile = globals.fs.systemTempDirectory
+      .createTempSync('flutter_disable_sandbox_entitlement.')
+      .childFile(
+        '${entitlementDefaultFileName}WithDisabledSandboxing.entitlements',
+      );
+  disabledSandboxEntitlementFile.createSync(recursive: true);
+  disabledSandboxEntitlementFile.writeAsStringSync(
+    entitlementFileContents.replaceAll(
+      RegExp(r'<key>com\.apple\.security\.app-sandbox<\/key>[\S\s]*?<true\/>'),
+      '''
+<key>com.apple.security.app-sandbox</key>
+	<false/>''',
+    ),
+  );
+  return disabledSandboxEntitlementFile;
+}

packages/flutter_tools/lib/src/macos/macos_device.dart

@@ -70,12 +70,14 @@ class MacOSDevice extends DesktopDevice {
   Future<void> buildForDevice({
     required BuildInfo buildInfo,
     String? mainPath,
+    bool usingCISystem = false,
   }) async {
     await buildMacOS(
       flutterProject: FlutterProject.current(),
       buildInfo: buildInfo,
       targetOverride: mainPath,
       verboseLogging: _logger.isVerbose,
+      usingCISystem: usingCISystem,
     );
   }
 

packages/flutter_tools/lib/src/macos/macos_ipad_device.dart

@@ -116,6 +116,7 @@ class MacOSDesignedForIPadDevice extends DesktopDevice {
   Future<void> buildForDevice({
     String? mainPath,
     required BuildInfo buildInfo,
+    bool usingCISystem = false,
   }) async {
     // Only attaching to a running app launched from Xcode is supported.
     throw UnimplementedError('Building for "$name" is not supported.');

packages/flutter_tools/lib/src/runner/flutter_command.dart

@@ -376,7 +376,16 @@ abstract class FlutterCommand extends Command<void> {
   String? get packagesPath => stringArg(FlutterGlobalOptions.kPackagesOption, global: true);
 
   /// Whether flutter is being run from our CI.
-  bool get usingCISystem => boolArg(FlutterGlobalOptions.kContinuousIntegrationFlag, global: true);
+  ///
+  /// This is true if `--ci` is passed to the command or if environment
+  /// variable `LUCI_CI` is `True`.
+  bool get usingCISystem {
+    return boolArg(
+          FlutterGlobalOptions.kContinuousIntegrationFlag,
+          global: true,
+        ) ||
+        globals.platform.environment['LUCI_CI'] == 'True';
+  }
 
   String? get debugLogsDirectoryPath => stringArg(FlutterGlobalOptions.kDebugLogsDirectoryFlag, global: true);
 

packages/flutter_tools/lib/src/windows/windows_device.dart

@@ -60,6 +60,7 @@ class WindowsDevice extends DesktopDevice {
   Future<void> buildForDevice({
     String? mainPath,
     required BuildInfo buildInfo,
+    bool usingCISystem = false,
   }) async {
     await buildWindows(
       FlutterProject.current().windows,