From 46c3689b7506c8166f8f96f42d597ad6a816257f Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 30 Mar 2022 11:07:26 -0400 Subject: [PATCH] feat: re-enable docker sharness tests (#8808) The Docker sharness tests were disabled years ago when go-ipfs moved from Travis to CircleCI. This makes the tweaks necessary to re-enable them. The Docker image has since moved to be based on BusyBox which doesn't have the requisite wget version for the existing tests to work, so this adds some functionality to the pollEndpoint program to support polling HTTP endpoints as well. --- .circleci/main.yml | 2 +- .dockerignore | 2 + test/dependencies/pollEndpoint/main.go | 72 ++++++++++++++++++++++++-- test/ipfs-test-lib.sh | 7 +-- test/sharness/t0300-docker-image.sh | 23 ++++---- test/sharness/t0301-docker-migrate.sh | 14 ++--- 6 files changed, 89 insertions(+), 31 deletions(-) diff --git a/.circleci/main.yml b/.circleci/main.yml index 7e87098cf6a..4c01be86fe3 100644 --- a/.circleci/main.yml +++ b/.circleci/main.yml @@ -154,7 +154,7 @@ jobs: working_directory: ~/ipfs/go-ipfs environment: <<: *default_environment - TEST_NO_DOCKER: 1 + TEST_NO_DOCKER: 0 TEST_NO_FUSE: 1 TEST_VERBOSE: 1 steps: diff --git a/.dockerignore b/.dockerignore index 10dd5fd886d..1e149a2bddb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,8 @@ Dockerfile.fast !.git/refs/ !.git/packed-refs test/sharness/lib/sharness/ +test/sharness/trash* +rb-pinning-service-api/ # The Docker client might not be running on Linux # so delete any compiled binaries diff --git a/test/dependencies/pollEndpoint/main.go b/test/dependencies/pollEndpoint/main.go index 7eadcd3f702..f3c19dbaf4a 100644 --- a/test/dependencies/pollEndpoint/main.go +++ b/test/dependencies/pollEndpoint/main.go @@ -2,7 +2,11 @@ package main import ( + "context" "flag" + "io" + "net" + "net/http" "os" "time" @@ -15,6 +19,8 @@ var ( host = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on") tries = flag.Int("tries", 10, "how many tries to make before failing") timeout = flag.Duration("tout", time.Second, "how long to wait between attempts") + httpURL = flag.String("http-url", "", "HTTP URL to fetch") + httpOut = flag.Bool("http-out", false, "Print the HTTP response body to stdout") verbose = flag.Bool("v", false, "verbose logging") ) @@ -37,18 +43,76 @@ func main() { start := time.Now() log.Debugf("starting at %s, tries: %d, timeout: %s, addr: %s", start, *tries, *timeout, addr) - for *tries > 0 { + connTries := *tries + for connTries > 0 { c, err := manet.Dial(addr) if err == nil { log.Debugf("ok - endpoint reachable with %d tries remaining, took %s", *tries, time.Since(start)) c.Close() - os.Exit(0) + break } log.Debug("connect failed: ", err) time.Sleep(*timeout) - *tries-- + connTries-- } - log.Error("failed.") + if err != nil { + goto Fail + } + + if *httpURL != "" { + dialer := &connDialer{addr: addr} + httpClient := http.Client{Transport: &http.Transport{ + DialContext: dialer.DialContext, + }} + reqTries := *tries + for reqTries > 0 { + try := (*tries - reqTries) + 1 + log.Debugf("trying HTTP req %d: '%s'", try, *httpURL) + if tryHTTPGet(&httpClient, *httpURL) { + log.Debugf("HTTP req %d to '%s' succeeded", try, *httpURL) + goto Success + } + log.Debugf("HTTP req %d to '%s' failed", try, *httpURL) + time.Sleep(*timeout) + reqTries-- + } + goto Fail + } + +Success: + os.Exit(0) + +Fail: + log.Error("failed") os.Exit(1) } + +func tryHTTPGet(client *http.Client, url string) bool { + resp, err := client.Get(*httpURL) + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return false + } + if resp.StatusCode != http.StatusOK { + return false + } + if *httpOut { + _, err := io.Copy(os.Stdout, resp.Body) + if err != nil { + panic(err) + } + } + + return true +} + +type connDialer struct { + addr ma.Multiaddr +} + +func (d connDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return (&manet.Dialer{}).DialContext(ctx, d.addr) +} diff --git a/test/ipfs-test-lib.sh b/test/ipfs-test-lib.sh index 38c62d33177..960cf7caa86 100644 --- a/test/ipfs-test-lib.sh +++ b/test/ipfs-test-lib.sh @@ -62,12 +62,7 @@ docker_run() { # This takes a docker ID and a command as arguments docker_exec() { - if test "$CIRCLE" = 1 - then - sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $1)" -- /bin/bash -c "$2" - else - docker exec -t "$1" /bin/bash -c "$2" - fi + docker exec -t "$1" /bin/sh -c "$2" } # This takes a docker ID as argument diff --git a/test/sharness/t0300-docker-image.sh b/test/sharness/t0300-docker-image.sh index 3d8f573f2f5..2a25dd28406 100755 --- a/test/sharness/t0300-docker-image.sh +++ b/test/sharness/t0300-docker-image.sh @@ -29,30 +29,28 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >actual || + docker_build "$TEST_TESTS_DIR/../Dockerfile" "$APP_ROOT_DIR" >build-actual || test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" || test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" || - test_fsh cat actual + test_fsh cat build-actual ' test_expect_success "docker image build output looks good" ' - SUCCESS_LINE=$(egrep "^Successfully built" actual) && + SUCCESS_LINE=$(egrep "^Successfully built" build-actual) && IMAGE_ID=$(expr "$SUCCESS_LINE" : "^Successfully built \(.*\)") || - test_fsh cat actual + test_fsh cat build-actual ' test_expect_success "docker image runs" ' - DOC_ID=$(docker_run "$IMAGE_ID") + DOC_ID=$(docker run -d -p 127.0.0.1:5001:5001 -p 127.0.0.1:8080:8080 "$IMAGE_ID") ' -test_expect_success "docker image gateway is up" ' - docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \ - -q -O - http://localhost:8080/version >/dev/null" +test_expect_success "docker container gateway is up" ' + pollEndpoint -host=/ip4/127.0.0.1/tcp/8080 -http-url http://localhost:8080/api/v0/version -v -tries 30 -tout 1s ' -test_expect_success "docker image API is up" ' - docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \ - -q -O - http://localhost:5001/api/v0/version >/dev/null" +test_expect_success "docker container API is up" ' + pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -v -tries 30 -tout 1s ' test_expect_success "simple ipfs add/cat can be run in docker container" ' @@ -63,8 +61,7 @@ test_expect_success "simple ipfs add/cat can be run in docker container" ' ' read testcode <actual ; \ + pollEndpoint -host=/ip4/127.0.0.1/tcp/5001 -http-url http://localhost:5001/version -http-out | grep Commit | cut -d" " -f2 >actual ; \ test -s actual ; \ docker exec -i "$DOC_ID" ipfs version --enc json \ | sed 's/^.*"Commit":"\\\([^"]*\\\)".*$/\\\1/g' >expected ; \ diff --git a/test/sharness/t0301-docker-migrate.sh b/test/sharness/t0301-docker-migrate.sh index 4f47310d181..4fb9ab155e4 100755 --- a/test/sharness/t0301-docker-migrate.sh +++ b/test/sharness/t0301-docker-migrate.sh @@ -32,6 +32,10 @@ test_expect_success "docker image build succeeds" ' test_init_ipfs +test_expect_success "configure migration sources" ' + ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]" +' + test_expect_success "make repo be version 4" ' echo 4 > "$IPFS_PATH/version" ' @@ -43,17 +47,13 @@ test_expect_success "setup http response" ' echo "v1.1.1" >> vers_resp ' -pretend_server() { - socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr 'SYSTEM:cat vers_resp'!!STDERR & -} - test_expect_success "startup fake dists server" ' - pretend_server > dist_serv_out & + ( socat tcp-listen:17233,fork,bind=127.0.0.1,reuseaddr "SYSTEM:cat vers_resp"!!STDERR 2> dist_serv_out ) & echo $! > netcat_pid ' test_expect_success "docker image runs" ' - DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host -e IPFS_DIST_PATH="http://localhost:17233" "$IMAGE_ID" --migrate) + DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_ID") ' test_expect_success "docker container tries to pull migrations from netcat" ' @@ -74,7 +74,7 @@ test_expect_success "kill the net cat" ' ' test_expect_success "correct version was requested" ' - grep "/fs-repo-migrations/v1.1.1/fs-repo-migrations_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null + grep "/fs-repo-6-to-7/v1.1.1/fs-repo-6-to-7_v1.1.1_linux-amd64.tar.gz" dist_serv_out > /dev/null ' test_done