feat: mechanism compile and embed tor (#1052)

This diff introduces a mechanism to compile and embed tor. We plan on using this functionality on Android and iOS but we have not finished working on updating the build yet. In the meanwhile, here's an interim result. We're committing the code to run unit and integration testing of this new functionality on linux/amd64.

The reference issue is ooni/probe#2365. 

The Go based build was sketched out in ooni/probe#2401.

.github/workflows/libtorlinux.yml

@@ -0,0 +1,29 @@
+# Runs tests for internal/libtor with -tags=ooni_libtor
+name: libtorlinux
+on:
+ pull_request:
+ push:
+ branches:
+ - "master"
+ - "release/**"
+ - "fullbuild"
+
+jobs:
+ test_ooni_libtor:
+ runs-on: ubuntu-20.04
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Get GOVERSION content
+ id: goversion
+ run: echo ::set-output name=version::$(cat GOVERSION)
+
+ - uses: magnetikonline/action-golang-cache@v2
+ with:
+ go-version: "${{ steps.goversion.outputs.version }}"
+ cache-key-suffix: "-libtorlinux-${{ steps.goversion.outputs.version }}"
+
+ - run: go run ./internal/cmd/buildtool linux cdeps zlib openssl libevent tor
+
+ - run: go test -count 1 -v -cover -tags ooni_libtor -race ./internal/libtor/...

CDEPS/README.md

@@ -0,0 +1,3 @@
+# Code to build C dependencies
+
+Directory used to compile C dependencies.

CDEPS/libevent/000.patch

@@ -0,0 +1,16 @@
+diff --git a/bufferevent_openssl.c b/bufferevent_openssl.c
+index b51b834..06b219e 100644
+--- a/bufferevent_openssl.c
++++ b/bufferevent_openssl.c
+@@ -67,6 +67,11 @@
+ #include <openssl/err.h>
+ #include "openssl-compat.h"
+
++#include <openssl/opensslv.h>
++#ifndef OPENSSL_OONI
++#error "We're not including the correct openssl/opensslv.h file"
++#endif
++
+ /*
+ * Define an OpenSSL bio that targets a bufferevent.
+ */

CDEPS/libevent/001.patch

@@ -0,0 +1,15 @@
+diff --git a/test/regress_zlib.c b/test/regress_zlib.c
+index 5fe7749..558155c 100644
+--- a/test/regress_zlib.c
++++ b/test/regress_zlib.c
+@@ -80,6 +80,10 @@
+
+ #include <zlib.h>
+
++#ifndef ZLIB_OONI
++#error "We're not including the correct zlib.h file"
++#endif
++
+ static int infilter_calls;
+ static int outfilter_calls;
+ static int readcb_finished;

CDEPS/libevent/002.patch

@@ -0,0 +1,15 @@
+diff --git a/include/event2/event.h b/include/event2/event.h
+index a6b6144..6abb474 100644
+--- a/include/event2/event.h
++++ b/include/event2/event.h
+@@ -1665,6 +1665,10 @@ int event_base_update_cache_time(struct event_base *base);
+ EVENT2_EXPORT_SYMBOL
+ void libevent_global_shutdown(void);
+
++/* EVENT_OONI is used by dependencies to ensure they are using the
++ correct event.h header and not some other header. */
++#define EVENT_OONI 1
++
+ #ifdef __cplusplus
+ }
+ #endif

CDEPS/openssl/000.patch

@@ -0,0 +1,15 @@
+diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c
+index b819337..e479bd6 100644
+--- a/crypto/comp/c_zlib.c
++++ b/crypto/comp/c_zlib.c
+@@ -34,6 +34,10 @@ static COMP_METHOD zlib_method_nozlib = {
+
+ # include <zlib.h>
+
++#ifndef ZLIB_OONI
++# error "We're not including the correct zlib.h file"
++#endif
++
+ static int zlib_stateful_init(COMP_CTX *ctx);
+ static void zlib_stateful_finish(COMP_CTX *ctx);
+ static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out,

CDEPS/openssl/001.patch

@@ -0,0 +1,14 @@
+diff --git a/include/openssl/opensslv.h b/include/openssl/opensslv.h
+index fd9400a..0d2c9dc 100644
+--- a/include/openssl/opensslv.h
++++ b/include/openssl/opensslv.h
+@@ -94,6 +94,9 @@ extern "C" {
+ # define SHLIB_VERSION_HISTORY ""
+ # define SHLIB_VERSION_NUMBER "1.1"
+
++/* OPENSSL_OONI is used by dependencies to ensure they are using the
++ correct OpenSSL headers and not some other headers. */
++#define OPENSSL_OONI 1
+
+ #ifdef __cplusplus
+ }

CDEPS/tor/000.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
+index 77de2d6..ce46554 100644
+--- a/src/lib/tls/tortls_openssl.c
++++ b/src/lib/tls/tortls_openssl.c
+@@ -45,6 +45,10 @@ DISABLE_GCC_WARNING("-Wredundant-decls")
+ #error "We require OpenSSL with ECC support"
+ #endif
+
++#ifndef OPENSSL_OONI
++#error "We're not including the correct openssl/opensslv.h file"
++#endif
++
+ #include <openssl/ssl.h>
+ #include <openssl/ssl3.h>
+ #include <openssl/err.h>

CDEPS/tor/001.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c
+index 52f9509..fb7e39e 100644
+--- a/src/lib/compress/compress_zlib.c
++++ b/src/lib/compress/compress_zlib.c
+@@ -45,6 +45,10 @@
+ #error "We require zlib version 1.2 or later."
+ #endif
+
++#ifndef ZLIB_OONI
++#error "We're not including the correct zlib.h file"
++#endif
++
+ static size_t tor_zlib_state_size_precalc(int inflate,
+ int windowbits, int memlevel);
+

CDEPS/tor/002.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c
+index fd840f8..2ec37e7 100644
+--- a/src/lib/evloop/compat_libevent.c
++++ b/src/lib/evloop/compat_libevent.c
+@@ -19,6 +19,10 @@
+ #include <event2/thread.h>
+ #include <string.h>
+
++#ifndef EVENT_OONI
++#error "We're not including the correct event2/event.h file"
++#endif
++
+ /** A string which, if it appears in a libevent log, should be ignored. */
+ static const char *suppress_msg = NULL;
+ /** Callback function passed to event_set_log() so we can intercept

CDEPS/zlib/000.patch

@@ -0,0 +1,15 @@
+diff --git a/zlib.h b/zlib.h
+index 953cb50..ec2e2f8 100644
+--- a/zlib.h
++++ b/zlib.h
+@@ -1928,6 +1928,10 @@ ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file,
+ # endif
+ #endif
+
++/* ZLIB_OONI is used by dependencies to ensure they are using the
++ correct zlib.h header and not some other header. */
++#define ZLIB_OONI 1
++
+ #ifdef __cplusplus
+ }
+ #endif

MONOREPO/w/build-android-with-cli.bash

@@ -8,8 +8,7 @@ source $reporoot/MONOREPO/tools/libcore.bash
 
 ./MOBILE/android/newkeystore
 
-./MOBILE/gomobile android ./pkg/oonimkall
-
+make ./MOBILE/android
 run cp -v MOBILE/android/oonimkall.aar ./MONOREPO/repo/probe-android/engine-experimental/
 
 (

internal/cmd/buildtool/cdeps.go

@@ -0,0 +1,155 @@
+package main
+
+//
+// Building C dependencies: common code
+//
+
+import (
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/apex/log"
+ "github.com/ooni/probe-cli/v3/internal/must"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+ "github.com/ooni/probe-cli/v3/internal/shellx"
+)
+
+// cdepsEnv contains the environment for compiling a C dependency.
+type cdepsEnv struct {
+ // cflags contains the CFLAGS to use when compiling.
+ cflags []string
+
+ // configureHost is the value to pass to ./configure's --host option.
+ configureHost string
+
+ // destdir is the directory where to install.
+ destdir string
+
+ // lfdlags contains the LDFLAGS to use when compiling.
+ ldflags []string
+
+ // openSSLAPIDefine is an extra define we need to add on Android.
+ openSSLAPIDefine string
+
+ // openSSLCompiler is the compiler name for OpenSSL.
+ openSSLCompiler string
+}
+
+// cdepsDependencies groups dependencies used when building cdeps.
+type cdepsDependencies interface {
+ // mustChdir changes the current working directory and returns the
+ // function to return to the original working directory.
+ mustChdir(dirname string) func()
+
+ // absoluteCurDir returns the absolute current directory.
+ absoluteCurDir() string
+
+ // verifySHA256 verifies that the tarball has the given checksum.
+ verifySHA256(expectedSHA256, tarball string)
+}
+
+// cdepsDependenciesStdlib are the [cdepsDependencies] used by default.
+type cdepsDependenciesStdlib struct{}
+
+var _ cdepsDependencies = &cdepsDependenciesStdlib{}
+
+// absoluteCurDir implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) absoluteCurDir() string {
+ return cdepsMustAbsoluteCurdir()
+}
+
+// verifySHA256 implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) verifySHA256(expectedSHA256, tarball string) {
+ cdepsMustVerifySHA256(expectedSHA256, tarball)
+}
+
+// mustChdir implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) mustChdir(dirname string) func() {
+ return cdepsMustChdir(dirname)
+}
+
+// cdepsAddCflags merges this struct's cflags with the extra cflags and
+// then stores the merged cflags into the given envp.
+func cdepsAddCflags(envp *shellx.Envp, c *cdepsEnv, extraCflags ...string) {
+ mergedCflags := append([]string{}, c.cflags...)
+ mergedCflags = append(mergedCflags, extraCflags...)
+ envp.Append("CFLAGS", strings.Join(mergedCflags, " "))
+}
+
+// cdepsAddLdflags merges this struct's ldflags with the extra ldflags and
+// then stores the merged ldflags into the given envp.
+func cdepsAddLdflags(envp *shellx.Envp, c *cdepsEnv, extraLdflags ...string) {
+ mergedLdflags := append([]string{}, c.ldflags...)
+ mergedLdflags = append(mergedLdflags, extraLdflags...)
+ envp.Append("LDFLAGS", strings.Join(mergedLdflags, " "))
+}
+
+// cdepsMustMkdirTemp creates a temporary directory.
+func cdepsMustMkdirTemp() string {
+ return runtimex.Try1(os.MkdirTemp("", ""))
+}
+
+// cdepsMustChdir changes the current directory to the given dir and
+// returns a function to return to the original working dir.
+func cdepsMustChdir(work string) func() {
+ prevdir := runtimex.Try1(os.Getwd())
+ log.Infof("cd %s", work)
+ runtimex.Try0(os.Chdir(work))
+ return func() {
+ runtimex.Try0(os.Chdir(prevdir))
+ log.Infof("cd %s", prevdir)
+ }
+}
+
+// cdepsMustFetch fetches the given URL using curl.
+func cdepsMustFetch(URL string) {
+ must.Run(log.Log, "curl", "-fsSLO", URL)
+}
+
+// cdepsMustVerifySHA256 verifies the SHA256 of the given tarball.
+func cdepsMustVerifySHA256(expectedSHA256, tarball string) {
+ firstline := string(must.FirstLineBytes(must.RunOutput(
+ log.Log, "sha256sum", tarball,
+ )))
+ sha256, _, good := strings.Cut(firstline, " ")
+ runtimex.Assert(good, "cannot obtain the first token")
+ runtimex.Assert(expectedSHA256 == sha256, "SHA256 mismatch")
+}
+
+// cdepsMustAbsoluteCurdir returns the absolute path of the current dir.
+func cdepsMustAbsoluteCurdir() string {
+ return runtimex.Try1(filepath.Abs("."))
+}
+
+// cdepsMustListPatches returns all the patches inside a dir.
+func cdepsMustListPatches(dir string) (out []string) {
+ entries := runtimex.Try1(os.ReadDir(dir))
+ for _, entry := range entries {
+ if !entry.Type().IsRegular() {
+ continue
+ }
+ if !strings.HasSuffix(entry.Name(), ".patch") {
+ continue
+ }
+ out = append(out, filepath.Join(dir, entry.Name()))
+ }
+ sort.Strings(out)
+ return
+}
+
+// cdepsDefaultShellxConfig returns the default config used when calling shellx.RunEx.
+func cdepsDefaultShellxConfig() *shellx.Config {
+ return &shellx.Config{
+ Logger: log.Log,
+ Flags: shellx.FlagShowStdoutStderr,
+ }
+}
+
+// cdepsMustRunWithDefaultConfig is a convenience wrapper
+// around calling [shellx.RunEx] and checking the return value.
+func cdepsMustRunWithDefaultConfig(envp *shellx.Envp, command string, args ...string) {
+ argv := runtimex.Try1(shellx.NewArgv(command, args...))
+ runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp))
+}