From f35b00259ee82f62a2b62ca0526865f7e3a8a24c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 12 Aug 2020 15:46:02 +0200 Subject: [PATCH] QA: make sure we can run on Linux without Docker (#836) * QA: make sure we can run from Linux Previous changes made it much more difficult to run on Linux. Yet, to do QA of Web Connectivity it's more practical to perform it on Linux using also OONI Probe v2.x and Measurement Kit. For this reason, this diff implements changes that should allow us to run both on Linux and when using Docker. While it's possible to create a Docker container where we deploy compiled versions of MK and OONI Probe v2.x, it's more practical to use a Linux system where they are already installed, like my Linux desktop. Part of https://github.com/ooni/probe-engine/issues/810 * QA: fix running on Linux 1. we cannot write on QA/HOME when running on Linux so just use `/tmp` as the HOME directory (slower but works) 2. the `/tmp` directory may be on another device than `/home`, so we cannot move, then just perform a copy Part of https://github.com/ooni/probe-engine/issues/810 --- QA/.dockerignore | 1 + {cmd/jafar => QA}/Dockerfile | 1 - QA/README.md | 5 ++--- QA/common.py | 9 +++++---- QA/rundocker.bash | 2 +- libminiooni/libminiooni.go | 22 +++++++++++++++------- 6 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 QA/.dockerignore rename {cmd/jafar => QA}/Dockerfile (77%) diff --git a/QA/.dockerignore b/QA/.dockerignore new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/QA/.dockerignore @@ -0,0 +1 @@ +* diff --git a/cmd/jafar/Dockerfile b/QA/Dockerfile similarity index 77% rename from cmd/jafar/Dockerfile rename to QA/Dockerfile index 3175c630..bd0586f4 100644 --- a/cmd/jafar/Dockerfile +++ b/QA/Dockerfile @@ -1,3 +1,2 @@ FROM alpine:edge RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3 -RUN adduser -D ooniprobe diff --git a/QA/README.md b/QA/README.md index 6ec773d6..93c85502 100644 --- a/QA/README.md +++ b/QA/README.md @@ -26,14 +26,13 @@ where `$nettest` is the nettest name (e.g. `telegram`) and `$ooni_exe` is the OONI Probe v2.x compatible binary to test. The Python script needs to run as root. Note however that sudo will also -be used to run `$ooni_exe` with the privileges of the `$SUDO_USER` that -called `sudo ./QA/$nettest.py $ooni_exe`. +be used to run `$ooni_exe` with the privileges of the `nobody` user. ## Run QA using a docker container Run test in a suitable Docker container using: -``` +```bash ./QA/rundocker.sh $nettest ``` diff --git a/QA/common.py b/QA/common.py index 6c5e5364..405d04b7 100644 --- a/QA/common.py +++ b/QA/common.py @@ -18,21 +18,22 @@ def execute(args): def execute_jafar_and_miniooni(ooni_exe, outfile, experiment, tag, args): """ Executes jafar and miniooni. Returns the test keys. """ + tmpoutfile = "/tmp/{}".format(outfile) with contextlib.suppress(FileNotFoundError): - os.remove(outfile) # just in case + os.remove(tmpoutfile) # just in case execute( [ "./jafar", "-main-command", - "%s -no '/home/ooniprobe/%s' %s" % (ooni_exe, outfile, experiment), + "{} -no '{}' --home /tmp {}".format(ooni_exe, tmpoutfile, experiment), "-main-user", - "ooniprobe", # created in cmd/jafar/Dockerfile + "nobody", # should be present on Unix "-tag", tag, ] + args ) - shutil.move("/home/ooniprobe/{}".format(outfile), outfile) + shutil.copy(tmpoutfile, outfile) result = read_result(outfile) assert isinstance(result, dict) assert isinstance(result["test_keys"], dict) diff --git a/QA/rundocker.bash b/QA/rundocker.bash index 33261afd..8ce7320c 100755 --- a/QA/rundocker.bash +++ b/QA/rundocker.bash @@ -1,5 +1,5 @@ #!/bin/sh set -ex DOCKER=${DOCKER:-docker} -$DOCKER build -t jafar-qa ./cmd/jafar/ +$DOCKER build -t jafar-qa ./QA/ $DOCKER run --privileged -v`pwd`:/jafar -w/jafar jafar-qa ./QA/pyrun.sh "$@" diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 222bb9d8..24a0dfc1 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -28,8 +28,9 @@ import ( // Options contains the options you can set from the CLI. type Options struct { Annotations []string - Inputs []string ExtraOptions []string + HomeDir string + Inputs []string NoBouncer bool NoGeoIP bool NoJSON bool @@ -68,14 +69,18 @@ func init() { &collectorURLUnused, "collector", 'c', "Unsupported option that used to set the collector base URL", "URL", ) - getopt.FlagLong( - &globalOptions.Inputs, "input", 'i', - "Add test-dependent input to the test input", "INPUT", - ) getopt.FlagLong( &globalOptions.ExtraOptions, "option", 'O', "Pass an option to the experiment", "KEY=VALUE", ) + getopt.FlagLong( + &globalOptions.HomeDir, "home", 0, + "Force specific home directory", "PATH", + ) + getopt.FlagLong( + &globalOptions.Inputs, "input", 'i', + "Add test-dependent input to the test input", "INPUT", + ) getopt.FlagLong( &globalOptions.NoBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", ) @@ -196,7 +201,10 @@ func (h *logHandler) HandleLog(e *log.Entry) (err error) { } // See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da -func gethomedir() string { +func gethomedir(optionsHome string) string { + if optionsHome != "" { + return optionsHome + } if runtime.GOOS == "windows" { home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") if home == "" { @@ -244,7 +252,7 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { } log.Log = logger - homeDir := gethomedir() + homeDir := gethomedir(currentOptions.HomeDir) fatalIfFalse(homeDir != "", "home directory is empty") miniooniDir := path.Join(homeDir, ".miniooni") assetsDir := path.Join(miniooniDir, "assets")