From 7ab2370102308ce2624f1e0251d0a595adb2925b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 25 Jan 2023 19:04:50 +0100 Subject: [PATCH] feat: rewrite linux build rules in Go (#1044) This diff rewrites Linux build rules in Go. See https://github.com/ooni/probe/issues/2401. --- CLI/go-build-alpine | 18 -- CLI/go-build-linux-static | 94 ---------- MONOREPO/tools/libgit.bash | 2 +- Makefile | 25 +-- internal/cmd/buildtool/linux.go | 18 ++ internal/cmd/buildtool/linuxdocker.go | 93 +++++++++ internal/cmd/buildtool/linuxdocker_test.go | 182 ++++++++++++++++++ internal/cmd/buildtool/linuxstatic.go | 112 +++++++++++ internal/cmd/buildtool/linuxstatic_test.go | 207 +++++++++++++++++++++ internal/cmd/buildtool/main.go | 3 +- 10 files changed, 625 insertions(+), 129 deletions(-) delete mode 100755 CLI/go-build-alpine delete mode 100755 CLI/go-build-linux-static create mode 100644 internal/cmd/buildtool/linux.go create mode 100644 internal/cmd/buildtool/linuxdocker.go create mode 100644 internal/cmd/buildtool/linuxdocker_test.go create mode 100644 internal/cmd/buildtool/linuxstatic.go create mode 100644 internal/cmd/buildtool/linuxstatic_test.go diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine deleted file mode 100755 index 1544867666..0000000000 --- a/CLI/go-build-alpine +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -euxo pipefail -# Some of the following exports are redundant but are however -# useful because they provide explicit logging -export CGO_ENABLED=1 -export GOARM=$GOARM -export GOCACHE=$GOCACHE -export GOMODCACHE=$GOMODCACHE -export GOOS=$GOOS -export GOARCH=$GOARCH -export OONI_PSIPHON_TAGS=${OONI_PSIPHON_TAGS:-} -for PACKAGE in $@; do - PRODUCT=$(basename $PACKAGE) - go build -o ./CLI/$PRODUCT-$GOOS-$OONIARCH \ - -tags=$OONI_PSIPHON_TAGS \ - -ldflags='-s -w -extldflags "-static"' \ - $GOLANG_EXTRA_FLAGS $PACKAGE -done diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static deleted file mode 100755 index 3ef177b28b..0000000000 --- a/CLI/go-build-linux-static +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [[ $# -lt 3 ]]; then - echo "" 1>&2 - echo "Docker-based compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2 - echo "" 1>&2 - echo "usage: $0 OOGOCACHEDIR OONIARCH PACKAGE..." 1>&2 - echo "" 1>&2 - echo "OOGOCACHEDIR is the directory under which to put GOCACHE and GOMODCACHE for" 1>&2 - echo "this build, which will be mounted and passed to Docker." 1>&2 - echo "" 1>&2 - echo "OONIARCH must be one of: 386, amd64, arm64, armv6, armv7." 1>&2 - echo "" 1>&2 - echo "Features:" 1>&2 - echo "" 1>&2 - echo "* automatically sets -tags=ooni_psiphon_config when possible;" 1>&2 - echo "" 1>&2 - echo "* if GOLANG_EXTRA_FLAGS is set, pass it to the Go compiler." 1>&2 - echo "" 1>&2 - echo "Example:" 1>&2 - echo "" 1>&2 - echo " ./CLI/go-build-linux-static arm64 ./internal/cmd/miniooni" 1>&2 - echo "" 1>&2 - exit 1 -fi - -GOLANG_DOCKER_IMAGE=golang:$(cat GOVERSION)-alpine -GOOS=linux -OOGOCACHEDIR=$1 -shift -OONIARCH=$1 -shift - -if [[ $OONIARCH == armv7 ]]; then - GOARCH=arm - GOARM=7 - DOCKER_ARCH=arm/v7 -elif [[ $OONIARCH == armv6 ]]; then - GOARCH=arm - GOARM=6 - DOCKER_ARCH=arm/v6 -else - GOARCH=$OONIARCH - GOARM= - DOCKER_ARCH=$OONIARCH -fi - -if [[ -f ./internal/engine/psiphon-config.json.age && - -f ./internal/engine/psiphon-config.key ]]; then - OONI_PSIPHON_TAGS=ooni_psiphon_config -else - OONI_PSIPHON_TAGS="" -fi - -# Implementation note: we must run docker as the user that invokes -# it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This -# constraint forces us to run all privileged operations early -# using a Dockerfile, so the build proper runs as $(id -u):$(id -g). - -GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache -GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache - -cat > CLI/Dockerfile << EOF -FROM --platform=linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE -RUN apk update -RUN apk upgrade -RUN apk add --no-progress gcc git linux-headers musl-dev -RUN adduser -D -h /home/oobuild -G nobody -u $(id -u) oobuild -ENV HOME=/home/oobuild -EOF - -TAGGED_IMAGE=oobuild-$OONIARCH-$(date +%Y%m%d%H) - -DOCKER_USER_OPTS="--user $(id -u):$(id -g)" - -set -x - -mkdir -p $GOCACHE $GOMODCACHE - -docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE - -if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then - docker build --platform linux/$DOCKER_ARCH -t $TAGGED_IMAGE CLI -fi - -docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \ - -e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \ - -v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \ - -e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \ - -e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \ - -e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \ - -v $(pwd):/ooni -w /ooni $TAGGED_IMAGE ./CLI/go-build-alpine "$@" diff --git a/MONOREPO/tools/libgit.bash b/MONOREPO/tools/libgit.bash index a9b92d5ea9..d3c1e332ee 100644 --- a/MONOREPO/tools/libgit.bash +++ b/MONOREPO/tools/libgit.bash @@ -174,7 +174,7 @@ clean_one_repo() { if [[ $(basename $dirname) == "probe-cli" ]]; then # Avoid completely removing all the cloned subrepos # as well as the important local.bash config file - extraflags="-e MONOREPO/repo/ -e MONOREPO/tools/local.bash" + extraflags="-e MONOREPO/repo/ -e MONOREPO/tools/local.bash -e GOCACHE" fi run git clean -dffx $extraflags ) diff --git a/Makefile b/Makefile index a66900d562..165afd9d8a 100644 --- a/Makefile +++ b/Makefile @@ -95,36 +95,36 @@ CLI/darwin: #help: The `make CLI/linux-static-386` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/386. .PHONY: CLI/linux-static-386 -CLI/linux-static-386: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) 386 ./cmd/ooniprobe ./internal/cmd/miniooni +CLI/linux-static-386: + go run ./internal/cmd/buildtool linux docker 386 #help: #help: The `make CLI/linux-static-amd64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/amd64. .PHONY: CLI/linux-static-amd64 -CLI/linux-static-amd64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) amd64 ./cmd/ooniprobe ./internal/cmd/miniooni +CLI/linux-static-amd64: + go run ./internal/cmd/buildtool linux docker amd64 #help: #help: The `make CLI/linux-static-armv6` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v6. .PHONY: CLI/linux-static-armv6 -CLI/linux-static-armv6: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv6 ./cmd/ooniprobe ./internal/cmd/miniooni +CLI/linux-static-armv6: + go run ./internal/cmd/buildtool linux docker armv6 #help: #help: The `make CLI/linux-static-armv7` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v7. .PHONY: CLI/linux-static-armv7 -CLI/linux-static-armv7: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv7 ./cmd/ooniprobe ./internal/cmd/miniooni +CLI/linux-static-armv7: + go run ./internal/cmd/buildtool linux docker armv7 #help: #help: The `make CLI/linux-static-arm64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm64. .PHONY: CLI/linux-static-arm64 -CLI/linux-static-arm64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni +CLI/linux-static-arm64: + go run ./internal/cmd/buildtool linux docker arm64 #help: #help: The `make CLI/miniooni` command creates a build of miniooni, for the current @@ -162,11 +162,6 @@ MOBILE/ios: search/for/go search/for/zip search/for/xcode maybe/copypsiphon ./MOBILE/ios/zipframework ./MOBILE/ios/createpodspec -.PHONY: search/for/docker -search/for/docker: - @printf "checking for docker... " - @command -v docker || { echo "not found"; exit 1; } - .PHONY: search/for/git search/for/git: @printf "checking for git... " diff --git a/internal/cmd/buildtool/linux.go b/internal/cmd/buildtool/linux.go new file mode 100644 index 0000000000..3e051a4bc3 --- /dev/null +++ b/internal/cmd/buildtool/linux.go @@ -0,0 +1,18 @@ +package main + +// +// Linux builds entry point +// + +import "github.com/spf13/cobra" + +// linuxSubcommand returns the linux [cobra.Command]. +func linuxSubcommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "linux", + Short: "Builds ooniprobe and miniooni for linux", + } + cmd.AddCommand(linuxDockerSubcommand()) + cmd.AddCommand(linuxStaticSubcommand()) + return cmd +} diff --git a/internal/cmd/buildtool/linuxdocker.go b/internal/cmd/buildtool/linuxdocker.go new file mode 100644 index 0000000000..6a47f888f0 --- /dev/null +++ b/internal/cmd/buildtool/linuxdocker.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// linuxDockerSubcommand returns the linuxDocker sucommand. +func linuxDockerSubcommand() *cobra.Command { + return &cobra.Command{ + Use: "docker {386|amd64|armv6|armv7|arm64}", + Short: "Builds ooniprobe and miniooni with static linking using docker", + Run: func(cmd *cobra.Command, args []string) { + linuxDockerBuildAll(&buildDeps{}, args[0]) + }, + Args: cobra.ExactArgs(1), + } +} + +// main is the main function of the linuxDocker subcommand. +func linuxDockerBuildAll(deps buildtoolmodel.Dependencies, ooniArch string) { + defer log.Infof("done") + deps.PsiphonMaybeCopyConfigFiles() + + golangVersion := string(must.FirstLineBytes(deps.LinuxReadGOVERSION("GOVERSION"))) + golangDockerImage := "golang:" + golangVersion + "-alpine" + + var ( + goarm string + dockerArch string + ) + switch ooniArch { + case "armv7": + goarm = "7" + dockerArch = "arm/v7" + case "armv6": + goarm = "6" + dockerArch = "arm/v6" + default: + goarm = "0" + dockerArch = ooniArch + } + + user := runtimex.Try1(user.Current()) + + // Implementation note: we must run docker as the user that invokes + // it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This + // constraint forces us to run all privileged operations early + // using a Dockerfile, so the build proper runs as $(id -u):$(id -g). + log.Infof("writing CLI/Dockerfile") + linuxDockerWriteDockerfile(deps, dockerArch, golangDockerImage, user.Uid) + + image := fmt.Sprintf("oobuild-%s-%s", ooniArch, time.Now().Format("20060102")) + + log.Infof("pull and build the correct docker image") + must.Run(log.Log, "docker", "pull", "--platform", "linux/"+dockerArch, golangDockerImage) + must.Run(log.Log, "docker", "build", "--platform", "linux/"+dockerArch, "-t", image, "CLI") + + log.Infof("run the build inside docker") + curdir := runtimex.Try1(os.Getwd()) + + must.Run( + log.Log, "docker", "run", + "--platform", "linux/"+dockerArch, + "--user", user.Uid, + "-v", curdir+":/ooni", + "-w", "/ooni", + image, + "go", "run", "./internal/cmd/buildtool", "linux", "static", "--goarm", goarm, + ) +} + +// linuxDockerWwriteDockerfile writes the CLI/Dockerfile file. +func linuxDockerWriteDockerfile(deps buildtoolmodel.Dependencies, dockerArch, golangDockerImage, uid string) { + content := []byte(fmt.Sprintf(` + FROM --platform=linux/%s %s + RUN apk update + RUN apk upgrade + RUN apk add --no-progress gcc git linux-headers musl-dev + RUN adduser -D -h /home/oobuild -G nobody -u %s oobuild + ENV HOME=/home/oobuild`, dockerArch, golangDockerImage, uid, + )) + deps.LinuxWriteDockerfile(filepath.Join("CLI", "Dockerfile"), content, 0600) +} diff --git a/internal/cmd/buildtool/linuxdocker_test.go b/internal/cmd/buildtool/linuxdocker_test.go new file mode 100644 index 0000000000..0bc88a426a --- /dev/null +++ b/internal/cmd/buildtool/linuxdocker_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "os" + "os/user" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtooltest" + "github.com/ooni/probe-cli/v3/internal/shellx/shellxtesting" +) + +func TestLinuxDockerBuildAll(t *testing.T) { + taggedImageSuffix := time.Now().Format("20060102") + + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + user, err := user.Current() + if err != nil { + t.Fatal(err) + } + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // ooniArch is the OONI arch value + ooniArch string + + // goarm is the GOARM value + goarm int64 + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "works as intended for arm64", + ooniArch: "arm64", + goarm: 0, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"docker", "pull", "--platform", "linux/arm64", "golang:1.14.17-alpine"}, + }, { + Env: []string{}, + Argv: []string{ + "docker", "build", "--platform", "linux/arm64", "-t", + "oobuild-arm64-" + taggedImageSuffix, "CLI", + }, + }, { + Env: []string{}, + Argv: []string{ + "docker", "run", "--platform", "linux/arm64", + "--user", user.Uid, "-v", cwd + ":/ooni", "-w", "/ooni", + "oobuild-arm64-" + taggedImageSuffix, "go", "run", "./internal/cmd/buildtool", + "linux", "static", "--goarm", "0", + }, + }}, + }, { + name: "works as intended for amd64", + ooniArch: "amd64", + goarm: 0, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"docker", "pull", "--platform", "linux/amd64", "golang:1.14.17-alpine"}, + }, { + Env: []string{}, + Argv: []string{ + "docker", "build", "--platform", "linux/amd64", "-t", + "oobuild-amd64-" + taggedImageSuffix, "CLI", + }, + }, { + Env: []string{}, + Argv: []string{ + "docker", "run", "--platform", "linux/amd64", + "--user", user.Uid, "-v", cwd + ":/ooni", "-w", "/ooni", + "oobuild-amd64-" + taggedImageSuffix, "go", "run", "./internal/cmd/buildtool", + "linux", "static", "--goarm", "0", + }, + }}, + }, { + name: "works as intended for 386", + ooniArch: "386", + goarm: 0, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"docker", "pull", "--platform", "linux/386", "golang:1.14.17-alpine"}, + }, { + Env: []string{}, + Argv: []string{ + "docker", "build", "--platform", "linux/386", "-t", + "oobuild-386-" + taggedImageSuffix, "CLI", + }, + }, { + Env: []string{}, + Argv: []string{ + "docker", "run", "--platform", "linux/386", + "--user", user.Uid, "-v", cwd + ":/ooni", "-w", "/ooni", + "oobuild-386-" + taggedImageSuffix, "go", "run", "./internal/cmd/buildtool", + "linux", "static", "--goarm", "0", + }, + }}, + }, { + name: "works as intended for armv7", + ooniArch: "armv7", + goarm: 0, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"docker", "pull", "--platform", "linux/arm/v7", "golang:1.14.17-alpine"}, + }, { + Env: []string{}, + Argv: []string{ + "docker", "build", "--platform", "linux/arm/v7", "-t", + "oobuild-armv7-" + taggedImageSuffix, "CLI", + }, + }, { + Env: []string{}, + Argv: []string{ + "docker", "run", "--platform", "linux/arm/v7", + "--user", user.Uid, "-v", cwd + ":/ooni", "-w", "/ooni", + "oobuild-armv7-" + taggedImageSuffix, "go", "run", "./internal/cmd/buildtool", + "linux", "static", "--goarm", "7", + }, + }}, + }, { + name: "works as intended for armv6", + ooniArch: "armv6", + goarm: 0, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"docker", "pull", "--platform", "linux/arm/v6", "golang:1.14.17-alpine"}, + }, { + Env: []string{}, + Argv: []string{ + "docker", "build", "--platform", "linux/arm/v6", "-t", + "oobuild-armv6-" + taggedImageSuffix, "CLI", + }, + }, { + Env: []string{}, + Argv: []string{ + "docker", "run", "--platform", "linux/arm/v6", + "--user", user.Uid, "-v", cwd + ":/ooni", "-w", "/ooni", + "oobuild-armv6-" + taggedImageSuffix, "go", "run", "./internal/cmd/buildtool", + "linux", "static", "--goarm", "6", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: false, + } + + shellxtesting.WithCustomLibrary(cc, func() { + linuxDockerBuildAll(deps, testcase.ooniArch) + }) + + expectCalls := map[string]int{ + buildtooltest.TagLinuxReadGOVERSION: 1, + buildtooltest.TagPsiphonMaybeCopyConfigFiles: 1, + buildtooltest.TagLinuxWriteDockerfile: 1, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/cmd/buildtool/linuxstatic.go b/internal/cmd/buildtool/linuxstatic.go new file mode 100644 index 0000000000..fbfed78af2 --- /dev/null +++ b/internal/cmd/buildtool/linuxstatic.go @@ -0,0 +1,112 @@ +package main + +// +// Builds for Linux assuming static linking makes sense. +// + +import ( + "fmt" + "path/filepath" + "runtime" + "strconv" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/shellx" + "github.com/spf13/cobra" +) + +// linuxStaticSubcommand returns the linuxStatic sucommand. +func linuxStaticSubcommand() *cobra.Command { + config := &linuxStaticBuilder{ + goarm: 0, + } + cmd := &cobra.Command{ + Use: "static", + Short: "Builds ooniprobe and miniooni for linux with static linking assuming alpine", + Run: config.main, + Args: cobra.NoArgs, + } + cmd.Flags().Int64Var(&config.goarm, "goarm", 0, "specifies the arm subarchitecture") + return cmd +} + +// linuxStaticBuilder is the build configuration. +type linuxStaticBuilder struct { + goarm int64 +} + +// main is the main function of the linuxStatic subcommand. +func (b *linuxStaticBuilder) main(*cobra.Command, []string) { + linuxStaticBuilAll(&buildDeps{}, runtime.GOARCH, b.goarm) +} + +// linuxStaticBuildAll builds all the packages on a linux-static environment. +func linuxStaticBuilAll(deps buildtoolmodel.Dependencies, goarch string, goarm int64) { + deps.PsiphonMaybeCopyConfigFiles() + deps.GolangCheck() + + // TODO(bassosimone): I am running the container with the right userID but + // apparently this is not enough to make git happy--why? + log.Infof("working around git file ownership checks") + must.Run(log.Log, "git", "config", "--global", "--add", "safe.directory", "/ooni") + + products := []*product{productMiniooni, productOoniprobe} + cacheprefix := runtimex.Try1(filepath.Abs("GOCACHE")) + for _, product := range products { + linuxStaticBuildPackage(deps, product, goarch, goarm, cacheprefix) + } +} + +// linuxStaticBuildPackage builds a package in a linux static environment. +func linuxStaticBuildPackage( + deps buildtoolmodel.Dependencies, + product *product, + goarch string, + goarm int64, + cacheprefix string, +) { + log.Infof("building %s for linux/%s with static linking", product.Pkg, goarch) + + ooniArch := linuxStaticBuildOONIArch(goarch, goarm) + + argv := runtimex.Try1(shellx.NewArgv("go", "build")) + if deps.PsiphonFilesExist() { + argv.Append("-tags", "ooni_psiphon_config") + } + argv.Append("-ldflags", "-s -w -extldflags -static") + argv.Append("-o", product.DestinationPath("linux", ooniArch)) + argv.Append(product.Pkg) + + envp := &shellx.Envp{} + envp.Append("CGO_ENABLED", "1") + envp.Append("GOOS", "linux") + envp.Append("GOARCH", goarch) + if goarm > 0 { + envp.Append("GOARM", strconv.FormatInt(goarm, 10)) + } + cachedirbase := filepath.Join(cacheprefix, "oonibuild", "v1", ooniArch) + envp.Append("GOCACHE", filepath.Join(cachedirbase, "buildcache")) + envp.Append("GOMODCACHE", filepath.Join(cachedirbase, "modcache")) + + config := &shellx.Config{ + Logger: log.Log, + Flags: shellx.FlagShowStdoutStderr, + } + + runtimex.Try0(shellx.RunEx(config, argv, envp)) +} + +// linuxStaticBuildOONIArch returns the OONI arch name. This is equal +// to the GOARCH but for arm where we use armv6 and armv7. +func linuxStaticBuildOONIArch(goarch string, goarm int64) string { + switch goarch { + case "arm": + runtimex.Assert(goarm > 0, "expected a > 0 goarm value") + return fmt.Sprintf("armv%d", goarm) + default: + return goarch + } +} diff --git a/internal/cmd/buildtool/linuxstatic_test.go b/internal/cmd/buildtool/linuxstatic_test.go new file mode 100644 index 0000000000..314d251c34 --- /dev/null +++ b/internal/cmd/buildtool/linuxstatic_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtooltest" + "github.com/ooni/probe-cli/v3/internal/shellx/shellxtesting" +) + +func TestLinuxStaticBuildAll(t *testing.T) { + + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + // testspec specifies a test case for this test + type testspec struct { + // name is the name of the test case + name string + + // goarch is the GOARCH value + goarch string + + // goarm is the GOARM value + goarm int64 + + // hasPsiphon indicates whether we should build with psiphon config + hasPsiphon bool + + // expectations contains the commands we expect to see + expect []buildtooltest.ExecExpectations + } + + var testcases = []testspec{{ + name: "build for arm64 where we have the psiphon config", + goarch: "arm64", + goarm: 0, + hasPsiphon: true, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"git", "config", "--global", "--add", "safe.directory", "/ooni"}, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/arm64/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/arm64/modcache", + "CGO_ENABLED=1", + "GOARCH=arm64", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-tags", "ooni_psiphon_config", + "-ldflags", "-s -w -extldflags -static", "-o", "CLI/miniooni-linux-arm64", + "./internal/cmd/miniooni", + }, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/arm64/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/arm64/modcache", + "CGO_ENABLED=1", + "GOARCH=arm64", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-tags", "ooni_psiphon_config", + "-ldflags", "-s -w -extldflags -static", "-o", "CLI/ooniprobe-linux-arm64", + "./cmd/ooniprobe", + }, + }}, + }, { + name: "build for amd64 where we don't have the psiphon config", + goarch: "amd64", + goarm: 0, + hasPsiphon: false, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"git", "config", "--global", "--add", "safe.directory", "/ooni"}, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/amd64/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/amd64/modcache", + "CGO_ENABLED=1", + "GOARCH=amd64", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-ldflags", "-s -w -extldflags -static", + "-o", "CLI/miniooni-linux-amd64", "./internal/cmd/miniooni", + }, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/amd64/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/amd64/modcache", + "CGO_ENABLED=1", + "GOARCH=amd64", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-ldflags", "-s -w -extldflags -static", + "-o", "CLI/ooniprobe-linux-amd64", "./cmd/ooniprobe", + }, + }}, + }, { + name: "build for armv7 where we have the psiphon config", + goarch: "arm", + goarm: 7, + hasPsiphon: true, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"git", "config", "--global", "--add", "safe.directory", "/ooni"}, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv7/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv7/modcache", + "CGO_ENABLED=1", + "GOARCH=arm", + "GOARM=7", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-tags", "ooni_psiphon_config", + "-ldflags", "-s -w -extldflags -static", "-o", "CLI/miniooni-linux-armv7", + "./internal/cmd/miniooni", + }, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv7/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv7/modcache", + "CGO_ENABLED=1", + "GOARCH=arm", + "GOARM=7", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-tags", "ooni_psiphon_config", + "-ldflags", "-s -w -extldflags -static", "-o", "CLI/ooniprobe-linux-armv7", + "./cmd/ooniprobe", + }, + }}, + }, { + name: "build for armv6 where we don't have the psiphon config", + goarch: "arm", + goarm: 6, + hasPsiphon: false, + expect: []buildtooltest.ExecExpectations{{ + Env: []string{}, + Argv: []string{"git", "config", "--global", "--add", "safe.directory", "/ooni"}, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv6/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv6/modcache", + "CGO_ENABLED=1", + "GOARCH=arm", + "GOARM=6", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-ldflags", "-s -w -extldflags -static", + "-o", "CLI/miniooni-linux-armv6", "./internal/cmd/miniooni", + }, + }, { + Env: []string{ + "GOCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv6/buildcache", + "GOMODCACHE=" + cwd + "/GOCACHE/oonibuild/v1/armv6/modcache", + "CGO_ENABLED=1", + "GOARCH=arm", + "GOARM=6", + "GOOS=linux", + }, + Argv: []string{ + "go", "build", "-ldflags", "-s -w -extldflags -static", + "-o", "CLI/ooniprobe-linux-armv6", "./cmd/ooniprobe", + }, + }}, + }} + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + + cc := &buildtooltest.SimpleCommandCollector{} + + deps := &buildtooltest.DependenciesCallCounter{ + HasPsiphon: testcase.hasPsiphon, + } + + shellxtesting.WithCustomLibrary(cc, func() { + linuxStaticBuilAll(deps, testcase.goarch, testcase.goarm) + }) + + expectCalls := map[string]int{ + buildtooltest.TagGolangCheck: 1, + buildtooltest.TagPsiphonMaybeCopyConfigFiles: 1, + buildtooltest.TagPsiphonFilesExist: 2, + } + + if diff := cmp.Diff(expectCalls, deps.Counter); diff != "" { + t.Fatal(diff) + } + + if err := buildtooltest.CheckManyCommands(cc.Commands, testcase.expect); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/cmd/buildtool/main.go b/internal/cmd/buildtool/main.go index 6227932f22..b607152b47 100644 --- a/internal/cmd/buildtool/main.go +++ b/internal/cmd/buildtool/main.go @@ -18,12 +18,13 @@ func main() { } root.AddCommand(darwinSubcommand()) root.AddCommand(genericSubcommand()) + root.AddCommand(linuxSubcommand()) root.AddCommand(windowsSubcommand()) logHandler := logx.NewHandlerWithDefaultSettings() logHandler.Emoji = true log.Log = &log.Logger{Level: log.InfoLevel, Handler: logHandler} - go func() { + defer func() { if r := recover(); r != nil { log.Fatalf("%+v", r) }