From ed915b5c1e0d9eef7a7e15248e402a056228f43b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 12 Oct 2023 18:04:30 +0200 Subject: [PATCH] [backport] feat(buildtool): build zlib, openssl, libevent, and tor for iOS This diff backports #1370 to the release/3.19 branch. This diff extends buildtool to builds zlib, openssl, libevent, and tor for iOS. We're only targeting 64 bit architectures, which is what ooni/probe-ios needs. We're targeting iOS >= 12.0, which is what ooni/probe-ios needs. A subsequent diff will introduce unit tests to make sure we don't break the iOS build. Reference issue: https://github.com/ooni/probe/issues/2564. This diff was extracted from https://github.com/ooni/probe-cli/pull/1366. --- .github/workflows/ios.yml | 3 + Makefile | 1 + internal/cmd/buildtool/android.go | 2 +- internal/cmd/buildtool/builddeps.go | 5 + .../internal/buildtoolmodel/buildtoolmodel.go | 4 + .../internal/buildtooltest/buildtooltest.go | 19 +++ internal/cmd/buildtool/ios.go | 157 ++++++++++++++++++ internal/libtor/ios/amd64/.gitignore | 2 + internal/libtor/ios/arm64/.gitignore | 2 + 9 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 internal/libtor/ios/amd64/.gitignore create mode 100644 internal/libtor/ios/arm64/.gitignore diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index fbcffd885e..6ab92595e2 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -35,6 +35,9 @@ jobs: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + # ./internal/cmd/buildtool needs coreutils for sha256 plus GNU build tools + - run: brew install autoconf automake coreutils libtool + - run: make EXPECTED_XCODE_VERSION=14.2 MOBILE/ios - uses: actions/upload-artifact@v3 diff --git a/Makefile b/Makefile index a1dcd72886..43ddb7b580 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,7 @@ android: search/for/java #help: The `make MOBILE/ios` command builds the oonimkall library for iOS. .PHONY: MOBILE/ios MOBILE/ios: search/for/zip search/for/xcode + go run ./internal/cmd/buildtool ios cdeps zlib openssl libevent tor go run ./internal/cmd/buildtool ios gomobile ./MOBILE/ios/zipframework ./MOBILE/ios/createpodspec diff --git a/internal/cmd/buildtool/android.go b/internal/cmd/buildtool/android.go index e2f6286b98..b8a4d88d39 100644 --- a/internal/cmd/buildtool/android.go +++ b/internal/cmd/buildtool/android.go @@ -44,7 +44,7 @@ func androidSubcommand() *cobra.Command { }) cmd.AddCommand(&cobra.Command{ - Use: "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]", + Use: "cdeps [zlib|openssl|libevent|tor...]", Short: "Cross compiles C dependencies for Android", Run: func(cmd *cobra.Command, args []string) { for _, arg := range args { diff --git a/internal/cmd/buildtool/builddeps.go b/internal/cmd/buildtool/builddeps.go index b499b45a48..cc4bececfb 100644 --- a/internal/cmd/buildtool/builddeps.go +++ b/internal/cmd/buildtool/builddeps.go @@ -82,3 +82,8 @@ func (*buildDeps) GOOS() string { func (*buildDeps) VerifySHA256(expectedSHA256 string, tarball string) { cdepsMustVerifySHA256(expectedSHA256, tarball) } + +// XCRun implements buildtoolmodel.Dependencies +func (*buildDeps) XCRun(args ...string) string { + return iosXCRun(args...) +} diff --git a/internal/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go b/internal/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go index 2ac8f89ec6..bac8c6856a 100644 --- a/internal/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go +++ b/internal/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go @@ -52,4 +52,8 @@ type Dependencies interface { // WindowsMingwCheck makes sure we're using the // expected version of mingw-w64. WindowsMingwCheck() + + // XCRun executes Xcode's xcrun tool with the given arguments and returns + // the first line of text emitted by xcrun or PANICS on failure. + XCRun(args ...string) string } diff --git a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go index 17bffeeca3..4ebcbe7272 100644 --- a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go +++ b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go @@ -245,3 +245,22 @@ func (cc *DependenciesCallCounter) increment(name string) { } cc.Counter[name]++ } + +// XCRun implements buildtoolmodel.Dependencies. +func (*DependenciesCallCounter) XCRun(args ...string) string { + runtimex.Assert(len(args) >= 1, "expected at least one argument") + switch args[0] { + case "-sdk": + runtimex.Assert(len(args) == 3, "expected three arguments") + runtimex.Assert(args[2] == "--show-sdk-path", "the third argument must be --show-sdk-path") + return filepath.Join("Developer", "SDKs", args[1]) + + case "-find": + runtimex.Assert(len(args) == 4, "expected four arguments") + runtimex.Assert(args[1] == "-sdk", "the second argument must be -sdk") + return filepath.Join("Developer", "SDKs", args[2], "bin", args[3]) + + default: + panic(errors.New("the first argument must be -sdk or -find")) + } +} diff --git a/internal/cmd/buildtool/ios.go b/internal/cmd/buildtool/ios.go index 9208b8c00d..7fa94bc5a9 100644 --- a/internal/cmd/buildtool/ios.go +++ b/internal/cmd/buildtool/ios.go @@ -5,10 +5,15 @@ package main // import ( + "errors" + "fmt" "path/filepath" + "runtime" "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/shellx" "github.com/spf13/cobra" ) @@ -19,6 +24,7 @@ func iosSubcommand() *cobra.Command { Use: "ios", Short: "Builds oonimkall and its dependencies for iOS", } + cmd.AddCommand(&cobra.Command{ Use: "gomobile", Short: "Builds oonimkall for iOS using gomobile", @@ -26,6 +32,18 @@ func iosSubcommand() *cobra.Command { iosBuildGomobile(&buildDeps{}) }, }) + + cmd.AddCommand(&cobra.Command{ + Use: "cdeps [zlib|openssl|libevent|tor...]", + Short: "Cross compiles C dependencies for iOS", + Run: func(cmd *cobra.Command, args []string) { + for _, arg := range args { + iosCdepsBuildMain(arg, &buildDeps{}) + } + }, + Args: cobra.MinimumNArgs(1), + }) + return cmd } @@ -41,6 +59,145 @@ func iosBuildGomobile(deps buildtoolmodel.Dependencies) { output: filepath.Join("MOBILE", "ios", "oonimkall.xcframework"), target: "ios", } + log.Info("building the mobile library using gomobile") gomobileBuild(config) } + +// iosCdepsBuildMain builds C dependencies for ios. +func iosCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) { + runtimex.Assert(runtime.GOOS == "darwin", "this command requires darwin") + + // The ooni/probe-ios app explicitly only targets amd64 and arm64. It also targets + // as the minimum version iOS 12, while one cannot target a version of iOS > 10 when + // building for 32-bit targets. Hence, using only 64 bit archs here is fine. + archs := []string{"arm64", "amd64"} + for _, arch := range archs { + iosCdepsBuildArch(deps, arch, name) + } +} + +// iosPlatformForOONIArch maps the ooniArch to the iOS platform +var iosPlatformForOONIArch = map[string]string{ + "amd64": "iphonesimulator", + "arm64": "iphoneos", +} + +// iosAppleArchForOONIArch maps the ooniArch to the corresponding apple arch +var iosAppleArchForOONIArch = map[string]string{ + "amd64": "x86_64", + "arm64": "arm64", +} + +// iosMinVersionFlagForOONIArch maps the ooniArch to the corresponding compiler flag +// to set the minimum version of either iphoneos or iphonesimulator. +// +// Note: the documentation of clang fetched on 2023-10-12 explicitly mentions that +// ios-version-min is an alias for iphoneos-version-min. Likewise, ios-simulator-version-min +// aliaes iphonesimulator-version-min. +// +// See https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-mios-simulator-version-min +var iosMinVersionFlagForOONIArch = map[string]string{ + "amd64": "-miphonesimulator-version-min=", + "arm64": "-miphoneos-version-min=", +} + +// iosCdepsBuildArch builds the given dependency for the given arch +func iosCdepsBuildArch(deps buildtoolmodel.Dependencies, ooniArch string, name string) { + cdenv := iosNewCBuildEnv(deps, ooniArch) + switch name { + case "libevent": + cdepsLibeventBuildMain(cdenv, deps) + case "openssl": + cdepsOpenSSLBuildMain(cdenv, deps) + case "tor": + cdepsTorBuildMain(cdenv, deps) + case "zlib": + cdepsZlibBuildMain(cdenv, deps) + default: + panic(fmt.Errorf("unknown dependency: %s", name)) + } +} + +// iosMinVersion is the minimum version that we support. We're using the +// same value used by the ooni/probe-ios app as of 2023-10.12. +const iosMinVersion = "12.0" + +// iosNewCBuildEnv creates a new [cBuildEnv] for the given ooniArch ("arm64" or "amd64"). +func iosNewCBuildEnv(deps buildtoolmodel.Dependencies, ooniArch string) *cBuildEnv { + destdir := runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute + "internal", "libtor", "ios", ooniArch, + ))) + + var ( + appleArch = iosAppleArchForOONIArch[ooniArch] + minVersionFlag = iosMinVersionFlagForOONIArch[ooniArch] + platform = iosPlatformForOONIArch[ooniArch] + ) + runtimex.Assert(appleArch != "", "empty appleArch") + runtimex.Assert(minVersionFlag != "", "empty minVersionFlag") + runtimex.Assert(platform != "", "empty platform") + + isysroot := deps.XCRun("-sdk", platform, "--show-sdk-path") + + out := &cBuildEnv{ + ANDROID_HOME: "", // not needed + ANDROID_NDK_ROOT: "", // not needed + AS: deps.XCRun("-find", "-sdk", platform, "as"), + AR: deps.XCRun("-find", "-sdk", platform, "ar"), + BINPATH: "", // not needed + CC: deps.XCRun("-find", "-sdk", platform, "cc"), + CFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-O2", + "-arch", appleArch, + "-fembed-bitcode", + }, + CONFIGURE_HOST: "", // later + DESTDIR: destdir, + CXX: deps.XCRun("-find", "-sdk", platform, "c++"), + CXXFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-arch", appleArch, + "-fembed-bitcode", + "-O2", + }, + GOARCH: ooniArch, + GOARM: "", // not needed + LD: deps.XCRun("-find", "-sdk", platform, "ld"), + LDFLAGS: []string{ + "-isysroot", isysroot, + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-arch", appleArch, + "-fembed-bitcode", + }, + OPENSSL_COMPILER: "", // later + OPENSSL_POST_COMPILER_FLAGS: []string{ + minVersionFlag + iosMinVersion, // tricky: they must be concatenated + "-fembed-bitcode", + }, + RANLIB: deps.XCRun("-find", "-sdk", platform, "ranlib"), + STRIP: deps.XCRun("-find", "-sdk", platform, "strip"), + } + + switch ooniArch { + case "arm64": + out.CONFIGURE_HOST = "arm-apple-darwin" + out.OPENSSL_COMPILER = "ios64-xcrun" + case "amd64": + out.CONFIGURE_HOST = "x86_64-apple-darwin" + out.OPENSSL_COMPILER = "iossimulator-xcrun" + default: + panic(errors.New("unsupported ooniArch")) + } + + return out +} + +// iosXCRun invokes `xcrun [args]` and returns its result of panics. This function +// is called indirectly by the iOS build through [buildtoolmodel.Dependencies]. +func iosXCRun(args ...string) string { + return string(must.FirstLineBytes(must.RunOutput(log.Log, "xcrun", args...))) +} diff --git a/internal/libtor/ios/amd64/.gitignore b/internal/libtor/ios/amd64/.gitignore new file mode 100644 index 0000000000..f32e7dc720 --- /dev/null +++ b/internal/libtor/ios/amd64/.gitignore @@ -0,0 +1,2 @@ +/include +/lib diff --git a/internal/libtor/ios/arm64/.gitignore b/internal/libtor/ios/arm64/.gitignore new file mode 100644 index 0000000000..f32e7dc720 --- /dev/null +++ b/internal/libtor/ios/arm64/.gitignore @@ -0,0 +1,2 @@ +/include +/lib