From 1f5cee53b8a852ef3805ddc138fdc84cb9c2f2e4 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 16:03:25 +0200 Subject: [PATCH 01/16] fix: attempt to make linux builds faster --- .github/workflows/linux.yml | 52 +++++++++++++++++++++++++++++++++++++ CLI/go-build-alpine | 2 ++ CLI/go-build-linux-static | 17 +++++++++--- Makefile | 35 +++++++++++++++---------- 4 files changed, 89 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ad4e686f29..74726b0758 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -5,6 +5,7 @@ on: branches: - "release/**" - "fullbuild" + - "linuxbuild" tags: - "v*" @@ -17,14 +18,23 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: ${{ env.HOME }}/.ooniprobe-build/cache + key: linux-build-cache-386 + - run: make CLI/linux-static-386 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,14 +48,23 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: ${{ env.HOME }}/.ooniprobe-build/cache + key: linux-build-cache-amd64 + - run: make CLI/linux-static-amd64 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-amd64 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-amd64 ./CLI/miniooni-linux-amd64 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -58,16 +77,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: ${{ env.HOME }}/.ooniprobe-build/cache + key: linux-build-cache-armv6 + - run: make CLI/linux-static-armv6 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv6 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv6 ./CLI/miniooni-linux-armv6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -80,16 +110,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: ${{ env.HOME }}/.ooniprobe-build/cache + key: linux-build-cache-armv7 + - run: make CLI/linux-static-armv7 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-armv7 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-armv7 ./CLI/miniooni-linux-armv7 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -102,16 +143,27 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + - run: sudo apt-get update -q + - run: sudo apt-get install -y qemu-user-static + - run: | echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age env: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - uses: actions/cache@v3 + with: + path: ${{ env.HOME }}/.ooniprobe-build/cache + key: linux-build-cache-arm64 + - run: make CLI/linux-static-arm64 + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-arm64 + - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-arm64 ./CLI/miniooni-linux-arm64 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine index b1995e754a..6a2d4a3aa7 100755 --- a/CLI/go-build-alpine +++ b/CLI/go-build-alpine @@ -10,6 +10,8 @@ git config --global --add safe.directory $(pwd) # useful because they provide explicit logging export CGO_ENABLED=1 export GOARM=$GOARM +export GOCACHE=$GOCACHE +export GOMODCACHE=$GOMODCACHE export GOOS=$GOOS export GOARCH=$GOARCH for PACKAGE in $@; do diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index e4f5fc19c5..ffb5882bc9 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -2,11 +2,14 @@ set -euo pipefail -if [[ $# -lt 2 ]]; then +if [[ $# -lt 3 ]]; then echo "" 1>&2 - echo "Compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2 + echo "Docker-based compiler for a Go PACKAGE producing static linux/OONIARCH binaries." 1>&2 echo "" 1>&2 - echo "usage: $0 OONIARCH PACKAGE..." 1>&2 + echo "usage: $0 OOGOCACHEDIR OONIARCH PACKAGE..." 1>&2 + echo "" 1>&2 + echo "OOGOCACHEDIR is the directory under which to put GOCACHE and GOMODCACHE for" 1>&2 + echo "this build, which will be mounted and passed to Docker." 1>&2 echo "" 1>&2 echo "OONIARCH must be one of: 386, amd64, arm64, armv6, armv7." 1>&2 echo "" 1>&2 @@ -25,6 +28,8 @@ fi GOLANG_DOCKER_IMAGE=golang:$(cat GOVERSION)-alpine GOOS=linux +OOGOCACHEDIR=$1 +shift OONIARCH=$1 shift @@ -49,11 +54,17 @@ else OONI_PSIPHON_TAGS="" fi +GOCACHE=$OOGOCACHEDIR/oonibuild/v0/$OONIARCH/buildcache +GOMODCACHE=$OOGOCACHEDIR/oonibuild/v0/$OONIARCH/modcache + set -x +mkdir -p $OOGOCACHEDIR docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE docker run --platform linux/$DOCKER_ARCH \ + -e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \ + -v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \ -e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \ -e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \ -e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \ diff --git a/Makefile b/Makefile index a87514b941..3b4c022cab 100644 --- a/Makefile +++ b/Makefile @@ -26,19 +26,26 @@ help: #help: on the command line as a key-value pairs (see usage above). #help: -#help: * GIT_CLONE_DIR : directory where to clone repositories, by default -#help: set to `$HOME/.ooniprobe-build/src`. +#help: * GIT_CLONE_DIR : directory where to clone repositories, by default +#help: set to `$HOME/.ooniprobe-build/src`. GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src #help: -#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause -#help: the build to embed a psiphon configuration file -#help: into the generated binaries. This build tag -#help: implies cloning the git@github.com:ooni/probe-private -#help: repository. If you do not have the permission to -#help: clone it, just clear this variable, e.g.: +#help: * OONI_GO_DOCKER_GOCACHE : base directory to put GOMODCACHE and GOCACHE +#help: when building using Docker. By default this +#help: is set to `$HOME/.ooniprobe-build/cache` #help: -#help: make OONI_PSIPHON_TAGS="" CLI/miniooni +OONI_GO_DOCKER_GOCACHE = $(HOME)/.ooniprobe-build/cache + +#help: +#help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause +#help: the build to embed a psiphon configuration file +#help: into the generated binaries. This build tag +#help: implies cloning the git@github.com:ooni/probe-private +#help: repository. If you do not have the permission to +#help: clone it, just clear this variable, e.g.: +#help: +#help: make OONI_PSIPHON_TAGS="" CLI/miniooni OONI_PSIPHON_TAGS = ooni_psiphon_config #quickhelp: @@ -92,35 +99,35 @@ CLI/darwin: search/for/go maybe/copypsiphon #help: ooniprobe and miniooni binaries for linux/386. .PHONY: CLI/linux-static-386 CLI/linux-static-386: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static 386 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) 386 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-amd64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/amd64. .PHONY: CLI/linux-static-amd64 CLI/linux-static-amd64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static amd64 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) amd64 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-armv6` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v6. .PHONY: CLI/linux-static-armv6 CLI/linux-static-armv6: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static armv6 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv6 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-armv7` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v7. .PHONY: CLI/linux-static-armv7 CLI/linux-static-armv7: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static armv7 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) armv7 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/linux-static-arm64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm64. .PHONY: CLI/linux-static-arm64 CLI/linux-static-arm64: search/for/docker maybe/copypsiphon - ./CLI/go-build-linux-static arm64 ./cmd/ooniprobe ./internal/cmd/miniooni + ./CLI/go-build-linux-static $(OONI_GO_DOCKER_GOCACHE) arm64 ./cmd/ooniprobe ./internal/cmd/miniooni #help: #help: The `make CLI/windows` command builds the ooniprobe and miniooni From a24fdb415777af13d353c3822eb692fc31dba70c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 16:54:32 +0200 Subject: [PATCH 02/16] try to get real homedir --- .github/workflows/linux.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 74726b0758..122a1ee3f7 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -26,9 +26,13 @@ jobs: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - name: Get HOME value + id: homedir + run: echo ::set-output name=home::$HOME + - uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.ooniprobe-build/cache + path: ${{ steps.homedir.home }}/.ooniprobe-build/cache key: linux-build-cache-386 - run: make CLI/linux-static-386 @@ -45,6 +49,7 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: + - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -74,6 +79,7 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: + - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -107,6 +113,7 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: + - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -140,6 +147,7 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: + - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 From 2873cc09d9c6925620a86787d9e39e4a94ce36c8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 16:57:48 +0200 Subject: [PATCH 03/16] x --- .github/workflows/linux.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 122a1ee3f7..7d4a2dd1a6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,7 +28,13 @@ jobs: - name: Get HOME value id: homedir - run: echo ::set-output name=home::$HOME + run: | + echo ::set-output name=home::$HOME + + - run: | + echo $HOME + + - run: exit 1 - uses: actions/cache@v3 with: From fd30a5e55b2f994ba2670d7471db4664abbc7dc9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 17:14:59 +0200 Subject: [PATCH 04/16] x --- .github/workflows/linux.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7d4a2dd1a6..1385741898 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -26,19 +26,9 @@ jobs: PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} - - name: Get HOME value - id: homedir - run: | - echo ::set-output name=home::$HOME - - - run: | - echo $HOME - - - run: exit 1 - - uses: actions/cache@v3 with: - path: ${{ steps.homedir.home }}/.ooniprobe-build/cache + path: /home/runner/.ooniprobe-build/cache # how to get $HOME here? key: linux-build-cache-386 - run: make CLI/linux-static-386 From aab0d88365641bca0fe4daafe59e867a53792e0a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 17:23:08 +0200 Subject: [PATCH 05/16] re-enable amd64 --- .github/workflows/linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1385741898..74e798cd7d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -45,7 +45,6 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: - - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -59,7 +58,7 @@ jobs: - uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.ooniprobe-build/cache + path: /home/runner/.ooniprobe-build/cache # how to get $HOME here? key: linux-build-cache-amd64 - run: make CLI/linux-static-amd64 From a3bfd34d3e32fd87cb49cd67020c7909ee16c9b2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 17:28:52 +0200 Subject: [PATCH 06/16] x --- .github/workflows/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 74e798cd7d..a6cd2968f1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -33,6 +33,8 @@ jobs: - run: make CLI/linux-static-386 + - run: ls -lha /home/runner/.ooniprobe-build/cache + - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386 - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386 From ad775b232d2df6db2c6c11c976674ab9181adbac Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 17:36:00 +0200 Subject: [PATCH 07/16] x --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a6cd2968f1..c084e70f7c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/cache@v3 with: - path: /home/runner/.ooniprobe-build/cache # how to get $HOME here? + path: ~/.ooniprobe-build/cache # how to get $HOME here? key: linux-build-cache-386 - run: make CLI/linux-static-386 From b0879ca0e2ef40f28b2b6d5cf9a8de8ffc09b957 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 18:59:43 +0200 Subject: [PATCH 08/16] fixed locally, now let's see --- .github/workflows/linux.yml | 4 ++-- CLI/.gitignore | 1 + CLI/go-build-alpine | 5 ----- CLI/go-build-linux-static | 26 ++++++++++++++++++++++---- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c084e70f7c..6a3c4e5b04 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,12 +28,12 @@ jobs: - uses: actions/cache@v3 with: - path: ~/.ooniprobe-build/cache # how to get $HOME here? + path: ~/.ooniprobe-build/cache key: linux-build-cache-386 - run: make CLI/linux-static-386 - - run: ls -lha /home/runner/.ooniprobe-build/cache + - run: ls -lha ~/.ooniprobe-build/cache - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386 diff --git a/CLI/.gitignore b/CLI/.gitignore index 34bcffc10f..93e7f32f25 100644 --- a/CLI/.gitignore +++ b/CLI/.gitignore @@ -1,2 +1,3 @@ +/Dockerfile /miniooni-* /ooniprobe-* diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine index 6a2d4a3aa7..5663cbae77 100755 --- a/CLI/go-build-alpine +++ b/CLI/go-build-alpine @@ -1,10 +1,5 @@ #!/bin/sh set -euxo pipefail -apk update -apk upgrade -apk add --no-progress gcc git linux-headers musl-dev -# We need to force git to look into this repository owned by the -# user outside docker rather than by the user running docker git config --global --add safe.directory $(pwd) # Some of the following exports are redundant but are however # useful because they provide explicit logging diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index ffb5882bc9..0576118e66 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -54,18 +54,36 @@ else OONI_PSIPHON_TAGS="" fi -GOCACHE=$OOGOCACHEDIR/oonibuild/v0/$OONIARCH/buildcache -GOMODCACHE=$OOGOCACHEDIR/oonibuild/v0/$OONIARCH/modcache +GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache +GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache + +cat > CLI/Dockerfile << EOF +FROM $GOLANG_DOCKER_IMAGE +RUN apk update +RUN apk upgrade +RUN apk add --no-progress gcc git linux-headers musl-dev +RUN adduser -D -h /home/oobuild -G nobody -u $(id -u) oobuild +ENV HOME=/home/oobuild +EOF + +TAGGED_IMAGE=oobuild-$OONIARCH-$(date +%Y%m%d%H) + +DOCKER_USER_OPTS="--user $(id -u):$(id -g)" set -x mkdir -p $OOGOCACHEDIR + docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE -docker run --platform linux/$DOCKER_ARCH \ +if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then + docker build -t $TAGGED_IMAGE CLI +fi + +docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \ -e GOCACHE=/__gocache -e GOMODCACHE=/__gomodcache \ -v "$GOCACHE:/__gocache" -v "$GOMODCACHE:/__gomodcache" \ -e GOARM=$GOARM -e GOOS=$GOOS -e GOARCH=$GOARCH \ -e OONI_PSIPHON_TAGS=$OONI_PSIPHON_TAGS \ -e OONIARCH=$OONIARCH -e GOLANG_EXTRA_FLAGS="${GOLANG_EXTRA_FLAGS:-}" \ - -v $(pwd):/ooni -w /ooni $GOLANG_DOCKER_IMAGE ./CLI/go-build-alpine "$@" + -v $(pwd):/ooni -w /ooni $TAGGED_IMAGE ./CLI/go-build-alpine "$@" From 0b9e7bf93797eb766e0ffb872dd1b26d4bab5445 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 19:25:45 +0200 Subject: [PATCH 09/16] try to use cache in the same directory --- .editorconfig | 15 +++++++++++++++ .vscode/settings.json | 9 +++++++-- CLI/go-build-linux-static | 2 +- GOCACHE/.gitignore | 1 + Makefile | 2 +- 5 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 .editorconfig create mode 100644 GOCACHE/.gitignore diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..ca2efb8916 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true + +[*.{py}] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.vscode/settings.json b/.vscode/settings.json index de288e1eab..c57221a264 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,8 @@ { - "python.formatting.provider": "black" -} \ No newline at end of file + "python.formatting.provider": "black", + "gopls": { + "build.directoryFilters": [ + "-GOCACHE" + ] + } +} diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index 0576118e66..fd97d54cd5 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -77,7 +77,7 @@ mkdir -p $OOGOCACHEDIR docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE if ! docker inspect --type=image $TAGGED_IMAGE 1>/dev/null 2>/dev/null; then - docker build -t $TAGGED_IMAGE CLI + docker build --platform linux/$DOCKER_ARCH -t $TAGGED_IMAGE CLI fi docker run --platform linux/$DOCKER_ARCH $DOCKER_USER_OPTS \ diff --git a/GOCACHE/.gitignore b/GOCACHE/.gitignore new file mode 100644 index 0000000000..72e8ffc0db --- /dev/null +++ b/GOCACHE/.gitignore @@ -0,0 +1 @@ +* diff --git a/Makefile b/Makefile index 3b4c022cab..95784c321e 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ GIT_CLONE_DIR = $(HOME)/.ooniprobe-build/src #help: when building using Docker. By default this #help: is set to `$HOME/.ooniprobe-build/cache` #help: -OONI_GO_DOCKER_GOCACHE = $(HOME)/.ooniprobe-build/cache +OONI_GO_DOCKER_GOCACHE = $$(pwd)/GOCACHE #help: #help: * OONI_PSIPHON_TAGS : build tags for `go build -tags ...` that cause From f2d691e85bd0a55deda3da67853bbee2e30a94ab Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 19:51:14 +0200 Subject: [PATCH 10/16] this seems to fix docker on linux --- CLI/go-build-linux-static | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index fd97d54cd5..7d73e5d76e 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -58,7 +58,7 @@ GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache cat > CLI/Dockerfile << EOF -FROM $GOLANG_DOCKER_IMAGE +FROM --platform=linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE RUN apk update RUN apk upgrade RUN apk add --no-progress gcc git linux-headers musl-dev @@ -72,7 +72,7 @@ DOCKER_USER_OPTS="--user $(id -u):$(id -g)" set -x -mkdir -p $OOGOCACHEDIR +mkdir -p $GOCACHE $GOMODCACHE docker pull --platform linux/$DOCKER_ARCH $GOLANG_DOCKER_IMAGE From 84641b0728511fee859d398f82c56a7ecb7f2300 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 19:54:32 +0200 Subject: [PATCH 11/16] another attempt --- .github/workflows/linux.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6a3c4e5b04..ef0b0ca180 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,13 +28,11 @@ jobs: - uses: actions/cache@v3 with: - path: ~/.ooniprobe-build/cache + path: GOCACHE key: linux-build-cache-386 - run: make CLI/linux-static-386 - - run: ls -lha ~/.ooniprobe-build/cache - - run: ./E2E/ooniprobe.sh ./CLI/ooniprobe-linux-386 - run: ./script/ghpublish.bash ./CLI/ooniprobe-linux-386 ./CLI/miniooni-linux-386 @@ -60,7 +58,7 @@ jobs: - uses: actions/cache@v3 with: - path: /home/runner/.ooniprobe-build/cache # how to get $HOME here? + path: GOCACHE key: linux-build-cache-amd64 - run: make CLI/linux-static-amd64 From 967c947149d5dbec9ddf21940fa37216b5cc0920 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 19:58:31 +0200 Subject: [PATCH 12/16] enable more builds --- .github/workflows/linux.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ef0b0ca180..f2217b0e3b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -74,7 +74,6 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: - - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -92,7 +91,7 @@ jobs: - uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.ooniprobe-build/cache + path: GOCACHE key: linux-build-cache-armv6 - run: make CLI/linux-static-armv6 @@ -108,7 +107,6 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: - - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -126,7 +124,7 @@ jobs: - uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.ooniprobe-build/cache + path: GOCACHE key: linux-build-cache-armv7 - run: make CLI/linux-static-armv7 From fc72a89e16ae4bf88e9723b118a476220f807b21 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 20:24:48 +0200 Subject: [PATCH 13/16] give it another spin --- .github/workflows/linux.yml | 3 +-- GOCACHE/README.md | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 GOCACHE/README.md diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f2217b0e3b..a99e8f9293 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -140,7 +140,6 @@ jobs: permissions: # See https://github.com/ooni/probe/issues/2154 contents: write steps: - - run: exit 1 - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -158,7 +157,7 @@ jobs: - uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.ooniprobe-build/cache + path: GOCACHE key: linux-build-cache-arm64 - run: make CLI/linux-static-arm64 diff --git a/GOCACHE/README.md b/GOCACHE/README.md new file mode 100644 index 0000000000..99646ff9c4 --- /dev/null +++ b/GOCACHE/README.md @@ -0,0 +1,8 @@ +# Directory GOCACHE + +This directory contains the GOCACHE and GOMODCACHE we use when +statically compiling Linux binaries using Docker. + +If you keep the context of this directory subsequent builds will be +faster. You will notice this especially for builds using qemu-user-static +to build for different architectures. From 0eac12ce7448e7b060e21c857d9f9d5d40aa5cc8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 20:51:24 +0200 Subject: [PATCH 14/16] okay, let's finish this --- CLI/go-build-linux-static | 5 +++++ GOCACHE/.gitignore | 2 +- GOCACHE/README.md | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CLI/go-build-linux-static b/CLI/go-build-linux-static index 7d73e5d76e..3ef177b28b 100755 --- a/CLI/go-build-linux-static +++ b/CLI/go-build-linux-static @@ -54,6 +54,11 @@ else OONI_PSIPHON_TAGS="" fi +# Implementation note: we must run docker as the user that invokes +# it for actions/cache@v3 to be able to cache OOGOCACHEDIR. This +# constraint forces us to run all privileged operations early +# using a Dockerfile, so the build proper runs as $(id -u):$(id -g). + GOCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/buildcache GOMODCACHE=$OOGOCACHEDIR/oonibuild/v1/$OONIARCH/modcache diff --git a/GOCACHE/.gitignore b/GOCACHE/.gitignore index 72e8ffc0db..61ce03b71d 100644 --- a/GOCACHE/.gitignore +++ b/GOCACHE/.gitignore @@ -1 +1 @@ -* +/oonibuild diff --git a/GOCACHE/README.md b/GOCACHE/README.md index 99646ff9c4..2316f87242 100644 --- a/GOCACHE/README.md +++ b/GOCACHE/README.md @@ -3,6 +3,6 @@ This directory contains the GOCACHE and GOMODCACHE we use when statically compiling Linux binaries using Docker. -If you keep the context of this directory subsequent builds will be +If you keep the content of this directory, subsequent builds will be faster. You will notice this especially for builds using qemu-user-static to build for different architectures. From 934951e97384ffc0f574528db5eaca6c2e8ba695 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 21:03:27 +0200 Subject: [PATCH 15/16] experiment w/o using git --- CLI/go-build-alpine | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine index 5663cbae77..68b2fd8eb5 100755 --- a/CLI/go-build-alpine +++ b/CLI/go-build-alpine @@ -1,6 +1,8 @@ #!/bin/sh set -euxo pipefail -git config --global --add safe.directory $(pwd) +# We need to force git, used by go to generate build info. Without the +# following setting, we see git complaining about dubious ownership. +#git config --global --add safe.directory $(pwd) # Some of the following exports are redundant but are however # useful because they provide explicit logging export CGO_ENABLED=1 From 7d874fa3c850e859c293bec5121fd61541e98b96 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Aug 2022 21:07:19 +0200 Subject: [PATCH 16/16] we don't actually need to force git --- CLI/go-build-alpine | 3 --- 1 file changed, 3 deletions(-) diff --git a/CLI/go-build-alpine b/CLI/go-build-alpine index 68b2fd8eb5..6ea546c646 100755 --- a/CLI/go-build-alpine +++ b/CLI/go-build-alpine @@ -1,8 +1,5 @@ #!/bin/sh set -euxo pipefail -# We need to force git, used by go to generate build info. Without the -# following setting, we see git complaining about dubious ownership. -#git config --global --add safe.directory $(pwd) # Some of the following exports are redundant but are however # useful because they provide explicit logging export CGO_ENABLED=1