feat: mechanism compile and embed tor

.github/workflows/libtorlinux.yml

@@ -0,0 +1,29 @@
+# Runs tests for internal/libtor with -tags=ooni_libtor
+name: libtorlinux
+on:
+ pull_request:
+ push:
+ branches:
+ - "master"
+ - "release/**"
+ - "fullbuild"
+
+jobs:
+ test_ooni_libtor:
+ runs-on: ubuntu-20.04
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Get GOVERSION content
+ id: goversion
+ run: echo ::set-output name=version::$(cat GOVERSION)
+
+ - uses: magnetikonline/action-golang-cache@v2
+ with:
+ go-version: "${{ steps.goversion.outputs.version }}"
+ cache-key-suffix: "-libtorlinux-${{ steps.goversion.outputs.version }}"
+
+ - run: go run ./internal/cmd/buildtool linux cdeps zlib openssl libevent tor
+
+ - run: go test -count 1 -v -cover -tags ooni_libtor -race ./internal/libtor/...

CDEPS/README.md

@@ -0,0 +1,3 @@
+# Code to build C dependencies
+
+Directory used to compile C dependencies.

CDEPS/libevent/000.patch

@@ -0,0 +1,16 @@
+diff --git a/bufferevent_openssl.c b/bufferevent_openssl.c
+index b51b834..06b219e 100644
+--- a/bufferevent_openssl.c
++++ b/bufferevent_openssl.c
+@@ -67,6 +67,11 @@
+ #include <openssl/err.h>
+ #include "openssl-compat.h"
+
++#include <openssl/opensslv.h>
++#ifndef OPENSSL_OONI
++#error "We're not including the correct openssl/opensslv.h file"
++#endif
++
+ /*
+ * Define an OpenSSL bio that targets a bufferevent.
+ */

CDEPS/libevent/001.patch

@@ -0,0 +1,15 @@
+diff --git a/test/regress_zlib.c b/test/regress_zlib.c
+index 5fe7749..558155c 100644
+--- a/test/regress_zlib.c
++++ b/test/regress_zlib.c
+@@ -80,6 +80,10 @@
+
+ #include <zlib.h>
+
++#ifndef ZLIB_OONI
++#error "We're not including the correct zlib.h file"
++#endif
++
+ static int infilter_calls;
+ static int outfilter_calls;
+ static int readcb_finished;

CDEPS/libevent/002.patch

@@ -0,0 +1,15 @@
+diff --git a/include/event2/event.h b/include/event2/event.h
+index a6b6144..6abb474 100644
+--- a/include/event2/event.h
++++ b/include/event2/event.h
+@@ -1665,6 +1665,10 @@ int event_base_update_cache_time(struct event_base *base);
+ EVENT2_EXPORT_SYMBOL
+ void libevent_global_shutdown(void);
+
++/* EVENT_OONI is used by dependencies to ensure they are using the
++ correct event.h header and not some other header. */
++#define EVENT_OONI 1
++
+ #ifdef __cplusplus
+ }
+ #endif

CDEPS/openssl/000.patch

@@ -0,0 +1,15 @@
+diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c
+index b819337..e479bd6 100644
+--- a/crypto/comp/c_zlib.c
++++ b/crypto/comp/c_zlib.c
+@@ -34,6 +34,10 @@ static COMP_METHOD zlib_method_nozlib = {
+
+ # include <zlib.h>
+
++#ifndef ZLIB_OONI
++# error "We're not including the correct zlib.h file"
++#endif
++
+ static int zlib_stateful_init(COMP_CTX *ctx);
+ static void zlib_stateful_finish(COMP_CTX *ctx);
+ static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out,

CDEPS/openssl/001.patch

@@ -0,0 +1,14 @@
+diff --git a/include/openssl/opensslv.h b/include/openssl/opensslv.h
+index fd9400a..0d2c9dc 100644
+--- a/include/openssl/opensslv.h
++++ b/include/openssl/opensslv.h
+@@ -94,6 +94,9 @@ extern "C" {
+ # define SHLIB_VERSION_HISTORY ""
+ # define SHLIB_VERSION_NUMBER "1.1"
+
++/* OPENSSL_OONI is used by dependencies to ensure they are using the
++ correct OpenSSL headers and not some other headers. */
++#define OPENSSL_OONI 1
+
+ #ifdef __cplusplus
+ }

CDEPS/tor/000.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
+index 77de2d6..ce46554 100644
+--- a/src/lib/tls/tortls_openssl.c
++++ b/src/lib/tls/tortls_openssl.c
+@@ -45,6 +45,10 @@ DISABLE_GCC_WARNING("-Wredundant-decls")
+ #error "We require OpenSSL with ECC support"
+ #endif
+
++#ifndef OPENSSL_OONI
++#error "We're not including the correct openssl/opensslv.h file"
++#endif
++
+ #include <openssl/ssl.h>
+ #include <openssl/ssl3.h>
+ #include <openssl/err.h>

CDEPS/tor/001.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c
+index 52f9509..fb7e39e 100644
+--- a/src/lib/compress/compress_zlib.c
++++ b/src/lib/compress/compress_zlib.c
+@@ -45,6 +45,10 @@
+ #error "We require zlib version 1.2 or later."
+ #endif
+
++#ifndef ZLIB_OONI
++#error "We're not including the correct zlib.h file"
++#endif
++
+ static size_t tor_zlib_state_size_precalc(int inflate,
+ int windowbits, int memlevel);
+

CDEPS/tor/002.patch

@@ -0,0 +1,15 @@
+diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c
+index fd840f8..2ec37e7 100644
+--- a/src/lib/evloop/compat_libevent.c
++++ b/src/lib/evloop/compat_libevent.c
+@@ -19,6 +19,10 @@
+ #include <event2/thread.h>
+ #include <string.h>
+
++#ifndef EVENT_OONI
++#error "We're not including the correct event2/event.h file"
++#endif
++
+ /** A string which, if it appears in a libevent log, should be ignored. */
+ static const char *suppress_msg = NULL;
+ /** Callback function passed to event_set_log() so we can intercept

CDEPS/zlib/000.patch

@@ -0,0 +1,15 @@
+diff --git a/zlib.h b/zlib.h
+index 953cb50..ec2e2f8 100644
+--- a/zlib.h
++++ b/zlib.h
+@@ -1928,6 +1928,10 @@ ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file,
+ # endif
+ #endif
+
++/* ZLIB_OONI is used by dependencies to ensure they are using the
++ correct zlib.h header and not some other header. */
++#define ZLIB_OONI 1
++
+ #ifdef __cplusplus
+ }
+ #endif

MONOREPO/w/build-android-with-cli.bash

@@ -8,8 +8,7 @@ source $reporoot/MONOREPO/tools/libcore.bash
 
 ./MOBILE/android/newkeystore
 
-./MOBILE/gomobile android ./pkg/oonimkall
-
+make ./MOBILE/android
 run cp -v MOBILE/android/oonimkall.aar ./MONOREPO/repo/probe-android/engine-experimental/
 
 (

internal/cmd/buildtool/cdeps.go

@@ -0,0 +1,155 @@
+package main
+
+//
+// Building C dependencies: common code
+//
+
+import (
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/apex/log"
+ "github.com/ooni/probe-cli/v3/internal/must"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+ "github.com/ooni/probe-cli/v3/internal/shellx"
+)
+
+// cdepsEnv contains the environment for compiling a C dependency.
+type cdepsEnv struct {
+ // cflags contains the CFLAGS to use when compiling.
+ cflags []string
+
+ // configureHost is the value to pass to ./configure's --host option.
+ configureHost string
+
+ // destdir is the directory where to install.
+ destdir string
+
+ // lfdlags contains the LDFLAGS to use when compiling.
+ ldflags []string
+
+ // openSSLAPIDefine is an extra define we need to add on Android.
+ openSSLAPIDefine string
+
+ // openSSLCompiler is the compiler name for OpenSSL.
+ openSSLCompiler string
+}
+
+// cdepsDependencies groups dependencies used when building cdeps.
+type cdepsDependencies interface {
+ // mustChdir changes the current working directory and returns the
+ // function to return to the original working directory.
+ mustChdir(dirname string) func()
+
+ // absoluteCurDir returns the absolute current directory.
+ absoluteCurDir() string
+
+ // verifySHA256 verifies that the tarball has the given checksum.
+ verifySHA256(expectedSHA256, tarball string)
+}
+
+// cdepsDependenciesStdlib are the [cdepsDependencies] used by default.
+type cdepsDependenciesStdlib struct{}
+
+var _ cdepsDependencies = &cdepsDependenciesStdlib{}
+
+// absoluteCurDir implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) absoluteCurDir() string {
+ return cdepsMustAbsoluteCurdir()
+}
+
+// verifySHA256 implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) verifySHA256(expectedSHA256, tarball string) {
+ cdepsMustVerifySHA256(expectedSHA256, tarball)
+}
+
+// mustChdir implements cdepsDependencies
+func (c *cdepsDependenciesStdlib) mustChdir(dirname string) func() {
+ return cdepsMustChdir(dirname)
+}
+
+// cdepsAddCflags merges this struct's cflags with the extra cflags and
+// then stores the merged cflags into the given envp.
+func cdepsAddCflags(envp *shellx.Envp, c *cdepsEnv, extraCflags ...string) {
+ mergedCflags := append([]string{}, c.cflags...)
+ mergedCflags = append(mergedCflags, extraCflags...)
+ envp.Append("CFLAGS", strings.Join(mergedCflags, " "))
+}
+
+// cdepsAddLdflags merges this struct's ldflags with the extra ldflags and
+// then stores the merged ldflags into the given envp.
+func cdepsAddLdflags(envp *shellx.Envp, c *cdepsEnv, extraLdflags ...string) {
+ mergedLdflags := append([]string{}, c.ldflags...)
+ mergedLdflags = append(mergedLdflags, extraLdflags...)
+ envp.Append("LDFLAGS", strings.Join(mergedLdflags, " "))
+}
+
+// cdepsMustMkdirTemp creates a temporary directory.
+func cdepsMustMkdirTemp() string {
+ return runtimex.Try1(os.MkdirTemp("", ""))
+}
+
+// cdepsMustChdir changes the current directory to the given dir and
+// returns a function to return to the original working dir.
+func cdepsMustChdir(work string) func() {
+ prevdir := runtimex.Try1(os.Getwd())
+ log.Infof("cd %s", work)
+ runtimex.Try0(os.Chdir(work))
+ return func() {
+ runtimex.Try0(os.Chdir(prevdir))
+ log.Infof("cd %s", prevdir)
+ }
+}
+
+// cdepsMustFetch fetches the given URL using curl.
+func cdepsMustFetch(URL string) {
+ must.Run(log.Log, "curl", "-fsSLO", URL)
+}
+
+// cdepsMustVerifySHA256 verifies the SHA256 of the given tarball.
+func cdepsMustVerifySHA256(expectedSHA256, tarball string) {
+ firstline := string(must.FirstLineBytes(must.RunOutput(
+ log.Log, "sha256sum", tarball,
+ )))
+ sha256, _, good := strings.Cut(firstline, " ")
+ runtimex.Assert(good, "cannot obtain the first token")
+ runtimex.Assert(expectedSHA256 == sha256, "SHA256 mismatch")
+}
+
+// cdepsMustAbsoluteCurdir returns the absolute path of the current dir.
+func cdepsMustAbsoluteCurdir() string {
+ return runtimex.Try1(filepath.Abs("."))
+}
+
+// cdepsMustListPatches returns all the patches inside a dir.
+func cdepsMustListPatches(dir string) (out []string) {
+ entries := runtimex.Try1(os.ReadDir(dir))
+ for _, entry := range entries {
+ if !entry.Type().IsRegular() {
+ continue
+ }
+ if !strings.HasSuffix(entry.Name(), ".patch") {
+ continue
+ }
+ out = append(out, filepath.Join(dir, entry.Name()))
+ }
+ sort.Strings(out)
+ return
+}
+
+// cdepsDefaultShellxConfig returns the default config used when calling shellx.RunEx.
+func cdepsDefaultShellxConfig() *shellx.Config {
+ return &shellx.Config{
+ Logger: log.Log,
+ Flags: shellx.FlagShowStdoutStderr,
+ }
+}
+
+// cdepsMustRunWithDefaultConfig is a convenience wrapper
+// around calling [shellx.RunEx] and checking the return value.
+func cdepsMustRunWithDefaultConfig(envp *shellx.Envp, command string, args ...string) {
+ argv := runtimex.Try1(shellx.NewArgv(command, args...))
+ runtimex.Try0(shellx.RunEx(cdepsDefaultShellxConfig(), argv, envp))
+}