diff --git a/sky/tools/create_macos_framework.py b/sky/tools/create_macos_framework.py index 6be5780bfd2d3..b981a8264d3ff 100755 --- a/sky/tools/create_macos_framework.py +++ b/sky/tools/create_macos_framework.py @@ -5,7 +5,6 @@ # found in the LICENSE file. import argparse -import subprocess import shutil import sys import os @@ -38,57 +37,32 @@ def main(): if os.path.isabs(args.x64_out_dir) else sky_utils.buildroot_relative_path(args.x64_out_dir) ) - fat_framework_bundle = os.path.join(dst, 'FlutterMacOS.framework') arm64_framework = os.path.join(arm64_out_dir, 'FlutterMacOS.framework') - x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework') - - arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS') - x64_dylib = os.path.join(x64_framework, 'FlutterMacOS') - if not os.path.isdir(arm64_framework): print('Cannot find macOS arm64 Framework at %s' % arm64_framework) return 1 + x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework') if not os.path.isdir(x64_framework): print('Cannot find macOS x64 Framework at %s' % x64_framework) return 1 + arm64_dylib = sky_utils.get_framework_dylib_path(arm64_framework) if not os.path.isfile(arm64_dylib): print('Cannot find macOS arm64 dylib at %s' % arm64_dylib) return 1 + x64_dylib = sky_utils.get_framework_dylib_path(x64_framework) if not os.path.isfile(x64_dylib): print('Cannot find macOS x64 dylib at %s' % x64_dylib) return 1 - sky_utils.copy_tree(arm64_framework, fat_framework_bundle, symlinks=True) - - regenerate_symlinks(fat_framework_bundle) - - fat_framework_binary = os.path.join(fat_framework_bundle, 'Versions', 'A', 'FlutterMacOS') - - # Create the arm64/x64 fat framework. - sky_utils.lipo([arm64_dylib, x64_dylib], fat_framework_binary) - - # Make the framework readable and executable: u=rwx,go=rx. - subprocess.check_call(['chmod', '755', fat_framework_bundle]) - - # Add group and other readability to all files. - versions_path = os.path.join(fat_framework_bundle, 'Versions') - subprocess.check_call(['chmod', '-R', 'og+r', versions_path]) - # Find all the files below the target dir with owner execute permission - find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'], - stdout=subprocess.PIPE) - # Add execute permission for other and group for all files that had it for owner. - xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'], - stdin=find_subprocess.stdout) - find_subprocess.wait() - xargs_subprocess.wait() - - process_framework(dst, args, fat_framework_bundle, fat_framework_binary) + fat_framework = os.path.join(dst, 'FlutterMacOS.framework') + sky_utils.create_fat_macos_framework(fat_framework, arm64_framework, x64_framework) + process_framework(dst, args, fat_framework) # Create XCFramework from the arm64 and x64 fat framework. - xcframeworks = [fat_framework_bundle] + xcframeworks = [fat_framework] create_xcframework(location=dst, name='FlutterMacOS', frameworks=xcframeworks) if args.zip: @@ -97,56 +71,26 @@ def main(): return 0 -def regenerate_symlinks(fat_framework_bundle): - """Regenerates the symlinks structure. - - Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks. - This logic regenerates the symlinks in the expected structure. - """ - if os.path.islink(os.path.join(fat_framework_bundle, 'FlutterMacOS')): - return - os.remove(os.path.join(fat_framework_bundle, 'FlutterMacOS')) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Headers'), True) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Modules'), True) - shutil.rmtree(os.path.join(fat_framework_bundle, 'Resources'), True) - current_version_path = os.path.join(fat_framework_bundle, 'Versions', 'Current') - shutil.rmtree(current_version_path, True) - os.symlink('A', current_version_path) - os.symlink( - os.path.join('Versions', 'Current', 'FlutterMacOS'), - os.path.join(fat_framework_bundle, 'FlutterMacOS') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Headers'), os.path.join(fat_framework_bundle, 'Headers') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Modules'), os.path.join(fat_framework_bundle, 'Modules') - ) - os.symlink( - os.path.join('Versions', 'Current', 'Resources'), - os.path.join(fat_framework_bundle, 'Resources') - ) - +def process_framework(dst, args, framework_path): + framework_binary = sky_utils.get_framework_dylib_path(framework_path) -def process_framework(dst, args, fat_framework_bundle, fat_framework_binary): if args.dsym: - dsym_out = os.path.splitext(fat_framework_bundle)[0] + '.dSYM' - sky_utils.extract_dsym(fat_framework_binary, dsym_out) + dsym_out = os.path.join(dst, 'FlutterMacOS.dSYM') + sky_utils.extract_dsym(framework_binary, dsym_out) if args.zip: - dsym_dst = os.path.join(dst, 'FlutterMacOS.dSYM') - sky_utils.create_zip(dsym_dst, 'FlutterMacOS.dSYM.zip', ['.']) - # Double zip to make it consistent with legacy artifacts. + # Create a zip of just the contents of the dSYM, then create a zip of that zip. # TODO(fujino): remove this once https://github.com/flutter/flutter/issues/125067 is resolved - sky_utils.create_zip(dsym_dst, 'FlutterMacOS.dSYM_.zip', ['FlutterMacOS.dSYM.zip']) + sky_utils.create_zip(dsym_out, 'FlutterMacOS.dSYM.zip', ['.']) + sky_utils.create_zip(dsym_out, 'FlutterMacOS.dSYM_.zip', ['FlutterMacOS.dSYM.zip']) # Overwrite the FlutterMacOS.dSYM.zip with the double-zipped archive. - dsym_final_src_path = os.path.join(dsym_dst, 'FlutterMacOS.dSYM_.zip') + dsym_final_src_path = os.path.join(dsym_out, 'FlutterMacOS.dSYM_.zip') dsym_final_dst_path = os.path.join(dst, 'FlutterMacOS.dSYM.zip') shutil.move(dsym_final_src_path, dsym_final_dst_path) if args.strip: unstripped_out = os.path.join(dst, 'FlutterMacOS.unstripped') - sky_utils.strip_binary(fat_framework_binary, unstripped_out) + sky_utils.strip_binary(framework_binary, unstripped_out) def zip_framework(dst): diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index 5e6a762fa7b1c..a2053fb64eff5 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -54,6 +54,67 @@ def create_zip(cwd, zip_filename, paths): subprocess.check_call(['zip', '-r', '-y', zip_filename] + paths, cwd=cwd) +def create_fat_macos_framework(fat_framework, arm64_framework, x64_framework): + """Creates a fat framework from two arm64 and x64 frameworks.""" + # Clone the arm64 framework bundle as a starting point. + copy_tree(arm64_framework, fat_framework, symlinks=True) + _regenerate_symlinks(fat_framework) + lipo([get_framework_dylib_path(arm64_framework), + get_framework_dylib_path(x64_framework)], get_framework_dylib_path(fat_framework)) + _set_framework_permissions(fat_framework) + + +def _regenerate_symlinks(framework_path): + """Regenerates the framework symlink structure. + + When building on the bots, the framework is produced in one shard, uploaded + to LUCI's content-addressable storage cache (CAS), then pulled down in + another shard. When that happens, symlinks are dereferenced, resulting a + corrupted framework. This regenerates the expected symlink farm. + """ + # If the dylib is symlinked, assume symlinks are all fine and bail out. + # The shutil.rmtree calls below only work on directories, and fail on symlinks. + framework_name = get_framework_name(framework_path) + framework_binary = get_framework_dylib_path(framework_path) + if os.path.islink(os.path.join(framework_path, framework_name)): + return + + # Delete any existing files/directories. + os.remove(framework_binary) + shutil.rmtree(os.path.join(framework_path, 'Headers'), True) + shutil.rmtree(os.path.join(framework_path, 'Modules'), True) + shutil.rmtree(os.path.join(framework_path, 'Resources'), True) + current_version_path = os.path.join(framework_path, 'Versions', 'Current') + shutil.rmtree(current_version_path, True) + + # Recreate the expected framework symlinks. + os.symlink('A', current_version_path) + os.symlink(os.path.join(current_version_path, framework_name), framework_binary) + os.symlink(os.path.join(current_version_path, 'Headers'), os.path.join(framework_path, 'Headers')) + os.symlink(os.path.join(current_version_path, 'Modules'), os.path.join(framework_path, 'Modules')) + os.symlink( + os.path.join(current_version_path, 'Resources'), os.path.join(framework_path, 'Resources') + ) + + +def _set_framework_permissions(framework_dir): + """Sets framework contents to be world readable, and world executable if user-executable.""" + # Make the framework readable and executable: u=rwx,go=rx. + subprocess.check_call(['chmod', '755', framework_dir]) + + # Add group and other readability to all files. + subprocess.check_call(['chmod', '-R', 'og+r', framework_dir]) + + # Find all the files below the target dir with owner execute permission and + # set og+x where it had the execute permission set for the owner. + find_subprocess = subprocess.Popen(['find', framework_dir, '-perm', '-100', '-print0'], + stdout=subprocess.PIPE) + xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'], + stdin=find_subprocess.stdout) + find_subprocess.wait() + xargs_subprocess.wait() + + def _dsymutil_path(): """Returns the path to dsymutil within Flutter's clang toolchain.""" arch_subpath = 'mac-arm64' if platform.processor() == 'arm' else 'mac-x64' @@ -61,6 +122,16 @@ def _dsymutil_path(): return buildroot_relative_path(dsymutil_path) +def get_framework_name(framework_dir): + """Returns Foo given /path/to/Foo.framework.""" + return os.path.splitext(os.path.basename(framework_dir))[0] + + +def get_framework_dylib_path(framework_dir): + """Returns /path/to/Foo.framework/Versions/A/Foo given /path/to/Foo.framework.""" + return os.path.join(framework_dir, 'Versions', 'A', get_framework_name(framework_dir)) + + def extract_dsym(binary_path, dsym_out_path): """Extracts a dSYM bundle from the specified Mach-O binary.""" arch_dir = 'mac-arm64' if platform.processor() == 'arm' else 'mac-x64'