Custom dyld linker for iOS mach-o executable files (apache#7875)

* [IOS-RPC] Fix compilation iOS_PRC app

Signed-off-by: Alexander Peskov <peskovnn@gmail.com>

CMakeLists.txt

@@ -76,6 +76,7 @@ tvm_option(USE_TENSORRT "Build with TensorRT, must have CUDA and CUDNN enabled"
 tvm_option(USE_RANDOM "Build with random support" ON)
 tvm_option(USE_MICRO_STANDALONE_RUNTIME "Build with micro.standalone_runtime support" OFF)
 tvm_option(USE_CPP_RPC "Build CPP RPC" OFF)
+tvm_option(USE_IOS_RPC "Build iOS RPC" OFF)
 tvm_option(USE_TFLITE "Build with tflite support" OFF)
 tvm_option(USE_TENSORFLOW_PATH "TensorFlow root path when use TFLite" none)
 tvm_option(USE_COREML "Build with coreml support" OFF)
@@ -467,6 +468,10 @@ if(USE_CPP_RPC)
   add_subdirectory("apps/cpp_rpc")
 endif()
 
+if(USE_IOS_RPC)
+  add_subdirectory("apps/ios_rpc")
+endif()
+
 if(USE_RELAY_DEBUG)
   message(STATUS "Building Relay in debug mode...")
   target_compile_definitions(tvm_objs PRIVATE "USE_RELAY_DEBUG")

apps/ios_rpc/CMakeLists.txt

@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include(ExternalProject)
+
+# Check if xcodebuild tool is available and configured.
+# Otherwise will skip all iOS specific targets.
+execute_process(COMMAND xcodebuild -version
+        RESULT_VARIABLE XCBUILD_AVAILABLE
+        OUTPUT_QUIET
+        ERROR_QUIET
+        )
+
+if (NOT XCBUILD_AVAILABLE EQUAL 0)
+    message(WARNING
+            "The build tool xcodebuild is not properly configured. Please install Xcode app and specify "
+            "path to it via DEVELOPER_DIR env var or \"sudo xcode-select -switch <path-to-xcode-dev-dir>\".\n"
+            "iOS RPC application target is switched off."
+            )
+    return()
+endif()
+
+
+# External project with custom mach-o dynamic loader
+# It is required to load unsigned shared modules on real iOS devices
+ExternalProject_Add(custom_dso_loader
+        GIT_REPOSITORY https://github.com/octoml/macho-dyld.git
+        GIT_TAG 48d1e8b5c40c7f5b744cb089634af17dd86125b2
+        PREFIX custom_dso_loader
+        LOG_DOWNLOAD TRUE
+        LOG_CONFIGURE TRUE
+        CMAKE_ARGS
+            -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>  # to install into local build dir
+            -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+            -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}
+            -DCMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION}
+            -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}
+            -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}
+            -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=${CMAKE_BUILD_WITH_INSTALL_NAME_DIR}
+        )
+
+# iOS RPC Xcode project wrapper to integrate into Cmake
+ExternalProject_Add(ios_rpc
+        PREFIX ios_rpc
+        DEPENDS custom_dso_loader
+        SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+        CONFIGURE_COMMAND ""
+        INSTALL_COMMAND ""
+        BUILD_COMMAND xcodebuild
+            -scheme tvmrpc
+            -configuration ${CMAKE_BUILD_TYPE}
+            -project <SOURCE_DIR>/tvmrpc.xcodeproj
+            -derivedDataPath <BINARY_DIR>
+            -sdk ${CMAKE_OSX_SYSROOT}
+            -arch ${CMAKE_OSX_ARCHITECTURES}
+            -hideShellScriptEnvironment
+            build
+                IPHONEOS_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}
+                DEVELOPMENT_TEAM=${CMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM}
+                TVM_BUILD_DIR=${CMAKE_BINARY_DIR}
+                USE_CUSTOM_DSO_LOADER=YES
+        )

apps/ios_rpc/README.md

@@ -52,14 +52,18 @@ Now App can be closed by pressing the home button (or even removed from a device
 ## Workflow
 Due to security restriction of iOS10. We cannot upload dynamic libraries to the App and load it from sandbox.
 Instead, we need to build a list of libraries, pack them into the app bundle, launch the RPC server and
-connect to test the bundled libraries. We use ```xcodebuild test``` to automate this process.
+connect to test the bundled libraries. We use ```xcodebuild test``` to automate this process. There is also
+one more approach to workaround this limitation, for more details please take a look into section
+[Custom DSO loader integration](#custom-dso-loader-plugin).
 
 The test script [tests/ios_rpc_test.py](tests/ios_rpc_test.py) is a good template for the workflow. With this
-script, we don't need to manually operate the iOS App, this script will build the app, run it and collect the results automatically.
+script, we don't need to manually operate the iOS App, this script will build the app, run it and collect the results 
+automatically.
 
  To run the script,  you need to configure the following environment variables
 
 - ```TVM_IOS_CODESIGN``` The signature you use to codesign the app and libraries (e.g. ```iPhone Developer: Name (XXXX)```)
+- ```TVM_IOS_TEAM_ID``` The developer Team ID available at https://developer.apple.com/account/#/membership     
 - ```TVM_IOS_RPC_ROOT``` The root directory of the iOS rpc project
 - ```TVM_IOS_RPC_PROXY_HOST``` The RPC proxy address (see above)
 - ```TVM_IOS_RPC_DESTINATION``` The Xcode target device (e.g. ```platform=iOS,id=xxxx```)
@@ -89,3 +93,40 @@ Then connect to the proxy via the python script.
 
 We can also use the RPC App directly, by typing in the address and press connect to connect to the proxy.
 However, the restriction is we can only load the modules that are bundled to the App.
+
+## Custom DSO loader plugin
+While iOS platform itself doesn't allow us to run an unsigned binary, where is a partial ability to run JIT code
+on real iOS devices. While application is running under debug session, system allows allocating memory with write
+and execute permissions (requirements of debugger). So we can use this feature to load binary on RPC side. For this
+purpose we use custom version of `dlopen` function which doesn't check signature and permissions for module loading.
+This custom `dlopen` mechanic is integrated into TVM RPC as plugin and registered to execution only inside iOS RPC
+application.
+
+The custom implementation of `dlopen` and other functions from `dlfcn.h` header are placed in separate repository,
+and will be downloaded automatically during cmake build for iOS. To run cmake build you may use next flags:
+```shell
+export DEVELOPER_DIR=/Applications/Xcode.app  # iOS SDK is part of Xcode bundle. Have to set it as default Dev Env
+cmake ..
+  -DCMAKE_BUILD_TYPE=Debug
+  -DCMAKE_SYSTEM_NAME=iOS
+  -DCMAKE_SYSTEM_VERSION=14.0
+  -DCMAKE_OSX_SYSROOT=iphoneos
+  -DCMAKE_OSX_ARCHITECTURES=arm64
+  -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0
+  -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON
+  -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=XXXXXXXXXX  # insert your Team ID
+  -DUSE_IOS_RPC=ON  # to enable build iOS RPC application from TVM project tree
+cmake --build . --target custom_dso_loader ios_rpc  # Will use custom DSO loader by default
+# Resulting iOS RPC app bundle will be placed in:
+# apps/ios_rpc/ios_rpc/src/ios_rpc-build/Build/Products/[CONFIG]-iphoneos/tvmrpc.app
+```
+
+To enable using of Custom DSO Plugin during xcode build outsde of Cmake you should specify two additional variables.
+You can do it manually inside Xcode IDE or via command line args for `xcodebuild`. Make sure that `custom_dso_loader`
+target from previous step is already built.
+* TVM_BUILD_DIR=path-to-tvm-ios-build-dir
+* USE_CUSTOM_DSO_LOADER=1
+
+iOS RPC application with enabled custom DSO loader is able to process modules passed via regular
+`remote.upload("my_module.dylib")` mechanics. For example take a look inside `test_rpc_module_with_upload` test case
+of file [ios_rpc_test.py](tests/ios_rpc_test.py).

apps/ios_rpc/tests/ios_rpc_test.py

@@ -31,7 +31,7 @@
 
 # Set to be address of tvm proxy.
 proxy_host = os.environ["TVM_IOS_RPC_PROXY_HOST"]
-# Set your desination via env variable.
+# Set your destination via env variable.
 # Should in format "platform=iOS,id=<the test device uuid>"
 destination = os.environ["TVM_IOS_RPC_DESTINATION"]
 
@@ -103,10 +103,48 @@ def test_rpc_module():
     a_np = np.random.uniform(size=1024).astype(A.dtype)
     a = tvm.nd.array(a_np, dev)
     b = tvm.nd.array(np.zeros(1024, dtype=A.dtype), dev)
-    time_f = f2.time_evaluator(f1.entry_name, dev, number=10)
+    time_f = f2.time_evaluator(f2.entry_name, dev, number=10)
     cost = time_f(a, b).mean
     print("%g secs/op" % cost)
     np.testing.assert_equal(b.asnumpy(), a.asnumpy() + 1)
 
 
-test_rpc_module()
+def test_rpc_module_with_upload():
+    server = xcode.popen_test_rpc(proxy_host, proxy_port, key, destination=destination)
+
+    remote = rpc.connect(proxy_host, proxy_port, key=key)
+    try:
+        remote.get_function("runtime.module.loadfile_dylib_custom")
+    except AttributeError as e:
+        print(e)
+        print("Skip test. You are using iOS RPC without custom DSO loader enabled.")
+        return
+
+    n = tvm.runtime.convert(1024)
+    A = te.placeholder((n,), name="A")
+    B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
+    temp = utils.tempdir()
+    s = te.create_schedule(B.op)
+    xo, xi = s[B].split(B.op.axis[0], factor=64)
+    s[B].parallel(xi)
+    s[B].pragma(xo, "parallel_launch_point")
+    s[B].pragma(xi, "parallel_barrier_when_finish")
+    f = tvm.build(s, [A, B], target, name="myadd_cpu")
+    path_dso = temp.relpath("cpu_lib.dylib")
+    f.export_library(path_dso, xcode.create_dylib, arch=arch, sdk=sdk)
+
+    dev = remote.cpu(0)
+    remote.upload(path_dso)
+    f = remote.load_module("cpu_lib.dylib")
+    a_np = np.random.uniform(size=1024).astype(A.dtype)
+    a = tvm.nd.array(a_np, dev)
+    b = tvm.nd.array(np.zeros(1024, dtype=A.dtype), dev)
+    time_f = f.time_evaluator(f.entry_name, dev, number=10)
+    cost = time_f(a, b).mean
+    print("%g secs/op" % cost)
+    np.testing.assert_equal(b.asnumpy(), a.asnumpy() + 1)
+
+
+if __name__ == "__main__":
+    test_rpc_module()
+    test_rpc_module_with_upload()

apps/ios_rpc/tvmrpc.xcodeproj/project.pbxproj

@@ -419,17 +419,33 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_OBJC_ARC = NO;
 				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
 				DEVELOPMENT_TEAM = 3FR42MXLK9;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"USE_CUSTOM_DSO_LOADER=${USE_CUSTOM_DSO_LOADER}",
+				);
 				GCC_SYMBOLS_PRIVATE_EXTERN = NO;
 				HEADER_SEARCH_PATHS = (
 					../../include,
 					../../3rdparty/dlpack/include,
 					"../../3rdparty/dmlc-core/include",
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/include",
 				);
 				INFOPLIST_FILE = tvmrpc/Info.plist;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
+				LIBRARY_SEARCH_PATHS = (
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/lib",
+					"${TVM_BUILD_DIR}",
+				);
+				OTHER_LDFLAGS = "${_DSO_LOADER_NAME_${USE_CUSTOM_DSO_LOADER}}";
 				PRODUCT_BUNDLE_IDENTIFIER = org.apache.tvmrpc;
 				PRODUCT_NAME = "$(TARGET_NAME)";
+				TVM_BUILD_DIR = "path-to-tvm-ios-build-folder";
+				USE_CUSTOM_DSO_LOADER = 0;
+				WARNING_CFLAGS = "-Wno-shorten-64-to-32";
+				_DSO_LOADER_NAME_0 = "";
+				_DSO_LOADER_NAME_1 = "-lmacho_dyld";
 			};
 			name = Debug;
 		};
@@ -439,17 +455,33 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_OBJC_ARC = NO;
 				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
 				DEVELOPMENT_TEAM = 3FR42MXLK9;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"USE_CUSTOM_DSO_LOADER=${USE_CUSTOM_DSO_LOADER}",
+				);
 				GCC_SYMBOLS_PRIVATE_EXTERN = NO;
 				HEADER_SEARCH_PATHS = (
 					../../include,
 					../../3rdparty/dlpack/include,
 					"../../3rdparty/dmlc-core/include",
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/include",
 				);
 				INFOPLIST_FILE = tvmrpc/Info.plist;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
+				LIBRARY_SEARCH_PATHS = (
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/lib",
+					"${TVM_BUILD_DIR}",
+				);
+				OTHER_LDFLAGS = "${_DSO_LOADER_NAME_${USE_CUSTOM_DSO_LOADER}}";
 				PRODUCT_BUNDLE_IDENTIFIER = org.apache.tvmrpc;
 				PRODUCT_NAME = "$(TARGET_NAME)";
+				TVM_BUILD_DIR = "path-to-tvm-ios-build-folder";
+				USE_CUSTOM_DSO_LOADER = 0;
+				WARNING_CFLAGS = "-Wno-shorten-64-to-32";
+				_DSO_LOADER_NAME_0 = "";
+				_DSO_LOADER_NAME_1 = "-lmacho_dyld";
 			};
 			name = Release;
 		};

apps/ios_rpc/tvmrpc/TVMRuntime.mm

@@ -27,10 +27,12 @@
 #include "../../../src/runtime/dso_library.cc"
 #include "../../../src/runtime/file_utils.cc"
 #include "../../../src/runtime/library_module.cc"
+#include "../../../src/runtime/logging.cc"
 #include "../../../src/runtime/metadata_module.cc"
 #include "../../../src/runtime/module.cc"
 #include "../../../src/runtime/ndarray.cc"
 #include "../../../src/runtime/object.cc"
+#include "../../../src/runtime/profiling.cc"
 #include "../../../src/runtime/registry.cc"
 #include "../../../src/runtime/system_library.cc"
 #include "../../../src/runtime/thread_pool.cc"
@@ -53,6 +55,10 @@
 // CoreML
 #include "../../../src/runtime/contrib/coreml/coreml_runtime.mm"
 
+#if defined(USE_CUSTOM_DSO_LOADER) && USE_CUSTOM_DSO_LOADER == 1
+#include <custom_dlfcn.h>
+#endif
+
 namespace tvm {
 namespace runtime {
 namespace detail {
@@ -145,6 +151,12 @@ void LaunchSyncServer() {
     // only load dylib from frameworks.
     NSBundle* bundle = [NSBundle mainBundle];
     base = [[bundle privateFrameworksPath] stringByAppendingPathComponent:@"tvm"];
+
+    if (Registry::Get("runtime.module.loadfile_dylib_custom")) {
+      // Custom dso laoder is present. Will use it.
+      base = NSTemporaryDirectory();
+      fmt = "dylib_custom";
+    }
   } else {
     // Load other modules in tempdir.
     base = NSTemporaryDirectory();
@@ -155,6 +167,41 @@ void LaunchSyncServer() {
   *rv = Module::LoadFromFile(name, fmt);
   LOG(INFO) << "Load module from " << name << " ...";
 });
+
+#if defined(USE_CUSTOM_DSO_LOADER) && USE_CUSTOM_DSO_LOADER == 1
+
+// Custom dynamic library loader. Supports unsigned binary
+class UnsignedDSOLoader final : public Library {
+ public:
+  ~UnsignedDSOLoader() {
+    if (lib_handle_) {
+      custom_dlclose(lib_handle_);
+      lib_handle_ = nullptr;
+    };
+  }
+  void Init(const std::string& name) {
+    lib_handle_ = custom_dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL);
+    ICHECK(lib_handle_ != nullptr)
+        << "Failed to load dynamic shared library " << name << " " << custom_dlerror();
+  }
+
+  void* GetSymbol(const char* name) final { return custom_dlsym(lib_handle_, name); }
+
+ private:
+  // Library handle
+  void* lib_handle_{nullptr};
+};
+
+// Add UnsignedDSOLoader plugin in global registry
+TVM_REGISTER_GLOBAL("runtime.module.loadfile_dylib_custom")
+    .set_body([](TVMArgs args, TVMRetValue* rv) {
+      auto n = make_object<UnsignedDSOLoader>();
+      n->Init(args[0]);
+      *rv = CreateModuleFromLibrary(n);
+    });
+
+#endif
+
 }  // namespace runtime
 }  // namespace tvm
 

apps/ios_rpc/tvmrpc/ViewController.mm

@@ -149,8 +149,8 @@ - (void)open {
   CFWriteStreamRef writeStream;
   CFStreamCreatePairWithSocketToHost(NULL, (__bridge CFStringRef)self.proxyURL.text,
                                      [self.proxyPort.text intValue], &readStream, &writeStream);
-  inputStream_ = (__bridge_transfer NSInputStream*)readStream;
-  outputStream_ = (__bridge_transfer NSOutputStream*)writeStream;
+  inputStream_ = (NSInputStream*)readStream;
+  outputStream_ = (NSOutputStream*)writeStream;
   [inputStream_ setDelegate:self];
   [outputStream_ setDelegate:self];
   [inputStream_ scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];

cmake/config.cmake

@@ -93,6 +93,9 @@ set(USE_RPC ON)
 # Whether to build the C++ RPC server binary
 set(USE_CPP_RPC OFF)
 
+# Whether to build the iOS RPC server application
+set(USE_IOS_RPC OFF)
+
 # Whether embed stackvm into the runtime
 set(USE_STACKVM_RUNTIME OFF)