[backport] feat(buildtool): build zlib, openssl, libevent, and tor fo…

…r iOS

This diff backports #1370 to the release/3.19 branch.

This diff extends buildtool to builds zlib, openssl, libevent, and tor
for iOS.

We're only targeting 64 bit architectures, which is what ooni/probe-ios
needs.

We're targeting iOS >= 12.0, which is what ooni/probe-ios needs.

A subsequent diff will introduce unit tests to make sure we don't break
the iOS build.

Reference issue: ooni/probe#2564.

This diff was extracted from
#1366.

.github/workflows/ios.yml

@@ -35,6 +35,9 @@ jobs:
           PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}
           PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}
 
+      # ./internal/cmd/buildtool needs coreutils for sha256 plus GNU build tools
+      - run: brew install autoconf automake coreutils libtool
+
       - run: make EXPECTED_XCODE_VERSION=14.2 MOBILE/ios
 
       - uses: actions/upload-artifact@v3

Makefile

@@ -98,6 +98,7 @@ android: search/for/java
 #help: The `make MOBILE/ios` command builds the oonimkall library for iOS.
 .PHONY: MOBILE/ios
 MOBILE/ios: search/for/zip search/for/xcode
+	go run ./internal/cmd/buildtool ios cdeps zlib openssl libevent tor
 	go run ./internal/cmd/buildtool ios gomobile
 	./MOBILE/ios/zipframework
 	./MOBILE/ios/createpodspec

internal/cmd/buildtool/android.go

@@ -44,7 +44,7 @@ func androidSubcommand() *cobra.Command {
 	})
 
 	cmd.AddCommand(&cobra.Command{
-		Use:   "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]",
+		Use:   "cdeps [zlib|openssl|libevent|tor...]",
 		Short: "Cross compiles C dependencies for Android",
 		Run: func(cmd *cobra.Command, args []string) {
 			for _, arg := range args {

internal/cmd/buildtool/builddeps.go

@@ -82,3 +82,8 @@ func (*buildDeps) GOOS() string {
 func (*buildDeps) VerifySHA256(expectedSHA256 string, tarball string) {
 	cdepsMustVerifySHA256(expectedSHA256, tarball)
 }
+
+// XCRun implements buildtoolmodel.Dependencies
+func (*buildDeps) XCRun(args ...string) string {
+	return iosXCRun(args...)
+}

internal/cmd/buildtool/internal/buildtoolmodel/buildtoolmodel.go

@@ -52,4 +52,8 @@ type Dependencies interface {
 	// WindowsMingwCheck makes sure we're using the
 	// expected version of mingw-w64.
 	WindowsMingwCheck()
+
+	// XCRun executes Xcode's xcrun tool with the given arguments and returns
+	// the first line of text emitted by xcrun or PANICS on failure.
+	XCRun(args ...string) string
 }

internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go

@@ -245,3 +245,22 @@ func (cc *DependenciesCallCounter) increment(name string) {
 	}
 	cc.Counter[name]++
 }
+
+// XCRun implements buildtoolmodel.Dependencies.
+func (*DependenciesCallCounter) XCRun(args ...string) string {
+	runtimex.Assert(len(args) >= 1, "expected at least one argument")
+	switch args[0] {
+	case "-sdk":
+		runtimex.Assert(len(args) == 3, "expected three arguments")
+		runtimex.Assert(args[2] == "--show-sdk-path", "the third argument must be --show-sdk-path")
+		return filepath.Join("Developer", "SDKs", args[1])
+
+	case "-find":
+		runtimex.Assert(len(args) == 4, "expected four arguments")
+		runtimex.Assert(args[1] == "-sdk", "the second argument must be -sdk")
+		return filepath.Join("Developer", "SDKs", args[2], "bin", args[3])
+
+	default:
+		panic(errors.New("the first argument must be -sdk or -find"))
+	}
+}

internal/cmd/buildtool/ios.go

@@ -5,10 +5,15 @@ package main
 //
 
 import (
+	"errors"
+	"fmt"
 	"path/filepath"
+	"runtime"
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel"
+	"github.com/ooni/probe-cli/v3/internal/must"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/shellx"
 	"github.com/spf13/cobra"
 )
@@ -19,13 +24,26 @@ func iosSubcommand() *cobra.Command {
 		Use:   "ios",
 		Short: "Builds oonimkall and its dependencies for iOS",
 	}
+
 	cmd.AddCommand(&cobra.Command{
 		Use:   "gomobile",
 		Short: "Builds oonimkall for iOS using gomobile",
 		Run: func(cmd *cobra.Command, args []string) {
 			iosBuildGomobile(&buildDeps{})
 		},
 	})
+
+	cmd.AddCommand(&cobra.Command{
+		Use:   "cdeps [zlib|openssl|libevent|tor...]",
+		Short: "Cross compiles C dependencies for iOS",
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, arg := range args {
+				iosCdepsBuildMain(arg, &buildDeps{})
+			}
+		},
+		Args: cobra.MinimumNArgs(1),
+	})
+
 	return cmd
 }
 
@@ -41,6 +59,145 @@ func iosBuildGomobile(deps buildtoolmodel.Dependencies) {
 		output:     filepath.Join("MOBILE", "ios", "oonimkall.xcframework"),
 		target:     "ios",
 	}
+
 	log.Info("building the mobile library using gomobile")
 	gomobileBuild(config)
 }
+
+// iosCdepsBuildMain builds C dependencies for ios.
+func iosCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) {
+	runtimex.Assert(runtime.GOOS == "darwin", "this command requires darwin")
+
+	// The ooni/probe-ios app explicitly only targets amd64 and arm64. It also targets
+	// as the minimum version iOS 12, while one cannot target a version of iOS > 10 when
+	// building for 32-bit targets. Hence, using only 64 bit archs here is fine.
+	archs := []string{"arm64", "amd64"}
+	for _, arch := range archs {
+		iosCdepsBuildArch(deps, arch, name)
+	}
+}
+
+// iosPlatformForOONIArch maps the ooniArch to the iOS platform
+var iosPlatformForOONIArch = map[string]string{
+	"amd64": "iphonesimulator",
+	"arm64": "iphoneos",
+}
+
+// iosAppleArchForOONIArch maps the ooniArch to the corresponding apple arch
+var iosAppleArchForOONIArch = map[string]string{
+	"amd64": "x86_64",
+	"arm64": "arm64",
+}
+
+// iosMinVersionFlagForOONIArch maps the ooniArch to the corresponding compiler flag
+// to set the minimum version of either iphoneos or iphonesimulator.
+//
+// Note: the documentation of clang fetched on 2023-10-12 explicitly mentions that
+// ios-version-min is an alias for iphoneos-version-min. Likewise, ios-simulator-version-min
+// aliaes iphonesimulator-version-min.
+//
+// See https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-mios-simulator-version-min
+var iosMinVersionFlagForOONIArch = map[string]string{
+	"amd64": "-miphonesimulator-version-min=",
+	"arm64": "-miphoneos-version-min=",
+}
+
+// iosCdepsBuildArch builds the given dependency for the given arch
+func iosCdepsBuildArch(deps buildtoolmodel.Dependencies, ooniArch string, name string) {
+	cdenv := iosNewCBuildEnv(deps, ooniArch)
+	switch name {
+	case "libevent":
+		cdepsLibeventBuildMain(cdenv, deps)
+	case "openssl":
+		cdepsOpenSSLBuildMain(cdenv, deps)
+	case "tor":
+		cdepsTorBuildMain(cdenv, deps)
+	case "zlib":
+		cdepsZlibBuildMain(cdenv, deps)
+	default:
+		panic(fmt.Errorf("unknown dependency: %s", name))
+	}
+}
+
+// iosMinVersion is the minimum version that we support. We're using the
+// same value used by the ooni/probe-ios app as of 2023-10.12.
+const iosMinVersion = "12.0"
+
+// iosNewCBuildEnv creates a new [cBuildEnv] for the given ooniArch ("arm64" or "amd64").
+func iosNewCBuildEnv(deps buildtoolmodel.Dependencies, ooniArch string) *cBuildEnv {
+	destdir := runtimex.Try1(filepath.Abs(filepath.Join( // must be absolute
+		"internal", "libtor", "ios", ooniArch,
+	)))
+
+	var (
+		appleArch      = iosAppleArchForOONIArch[ooniArch]
+		minVersionFlag = iosMinVersionFlagForOONIArch[ooniArch]
+		platform       = iosPlatformForOONIArch[ooniArch]
+	)
+	runtimex.Assert(appleArch != "", "empty appleArch")
+	runtimex.Assert(minVersionFlag != "", "empty minVersionFlag")
+	runtimex.Assert(platform != "", "empty platform")
+
+	isysroot := deps.XCRun("-sdk", platform, "--show-sdk-path")
+
+	out := &cBuildEnv{
+		ANDROID_HOME:     "", // not needed
+		ANDROID_NDK_ROOT: "", // not needed
+		AS:               deps.XCRun("-find", "-sdk", platform, "as"),
+		AR:               deps.XCRun("-find", "-sdk", platform, "ar"),
+		BINPATH:          "", // not needed
+		CC:               deps.XCRun("-find", "-sdk", platform, "cc"),
+		CFLAGS: []string{
+			"-isysroot", isysroot,
+			minVersionFlag + iosMinVersion, // tricky: they must be concatenated
+			"-O2",
+			"-arch", appleArch,
+			"-fembed-bitcode",
+		},
+		CONFIGURE_HOST: "", // later
+		DESTDIR:        destdir,
+		CXX:            deps.XCRun("-find", "-sdk", platform, "c++"),
+		CXXFLAGS: []string{
+			"-isysroot", isysroot,
+			minVersionFlag + iosMinVersion, // tricky: they must be concatenated
+			"-arch", appleArch,
+			"-fembed-bitcode",
+			"-O2",
+		},
+		GOARCH: ooniArch,
+		GOARM:  "", // not needed
+		LD:     deps.XCRun("-find", "-sdk", platform, "ld"),
+		LDFLAGS: []string{
+			"-isysroot", isysroot,
+			minVersionFlag + iosMinVersion, // tricky: they must be concatenated
+			"-arch", appleArch,
+			"-fembed-bitcode",
+		},
+		OPENSSL_COMPILER: "", // later
+		OPENSSL_POST_COMPILER_FLAGS: []string{
+			minVersionFlag + iosMinVersion, // tricky: they must be concatenated
+			"-fembed-bitcode",
+		},
+		RANLIB: deps.XCRun("-find", "-sdk", platform, "ranlib"),
+		STRIP:  deps.XCRun("-find", "-sdk", platform, "strip"),
+	}
+
+	switch ooniArch {
+	case "arm64":
+		out.CONFIGURE_HOST = "arm-apple-darwin"
+		out.OPENSSL_COMPILER = "ios64-xcrun"
+	case "amd64":
+		out.CONFIGURE_HOST = "x86_64-apple-darwin"
+		out.OPENSSL_COMPILER = "iossimulator-xcrun"
+	default:
+		panic(errors.New("unsupported ooniArch"))
+	}
+
+	return out
+}
+
+// iosXCRun invokes `xcrun [args]` and returns its result of panics. This function
+// is called indirectly by the iOS build through [buildtoolmodel.Dependencies].
+func iosXCRun(args ...string) string {
+	return string(must.FirstLineBytes(must.RunOutput(log.Log, "xcrun", args...)))
+}

internal/libtor/ios/amd64/.gitignore

@@ -0,0 +1,2 @@
+/include
+/lib

internal/libtor/ios/arm64/.gitignore

@@ -0,0 +1,2 @@
+/include
+/lib