Skip to content

Commit

Permalink
Merge pull request #667 from kradalby/rerun-docker
Browse files Browse the repository at this point in the history 
Make integration tests retry on failure.
  • Loading branch information
@juanfont
juanfont authored Jun 27, 2022
2 parents 4a200c3 + 566b8c3 commit e0b15c1
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 44 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/test-integration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,9 @@ jobs:

- name: Run Integration tests
if: steps.changed-files.outputs.any_changed == 'true'
run: nix develop --command -- make test_integration
uses: nick-fields/retry@v2
with:
timeout_minutes: 240
max_attempts: 5
retry_on: error
command: nix develop --command -- make test_integration
13 changes: 9 additions & 4 deletions integration_cli_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ func (s *IntegrationCLITestSuite) SetupTest() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}

if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
s.network = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}

headscaleBuildOptions := &dockertest.BuildOptions{
Expand All @@ -56,7 +56,7 @@ func (s *IntegrationCLITestSuite) SetupTest() {

currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}

headscaleOptions := &dockertest.RunOptions{
Expand All @@ -68,11 +68,16 @@ func (s *IntegrationCLITestSuite) SetupTest() {
Cmd: []string{"headscale", "serve"},
}

err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}

fmt.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
fmt.Println("Created headscale container")

Expand Down
31 changes: 30 additions & 1 deletion integration_common_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ package headscale
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"

Expand All @@ -16,9 +19,13 @@ import (
"inet.af/netaddr"
)

const DOCKER_EXECUTE_TIMEOUT = 10 * time.Second
const (
DOCKER_EXECUTE_TIMEOUT = 10 * time.Second
)

var (
errEnvVarEmpty = errors.New("getenv: environment variable empty")

IpPrefix4 = netaddr.MustParseIPPrefix("100.64.0.0/10")
IpPrefix6 = netaddr.MustParseIPPrefix("fd7a:115c:a1e0::/48")

Expand Down Expand Up @@ -283,3 +290,25 @@ func getMagicFQDN(

return hostnames, nil
}

func GetEnvStr(key string) (string, error) {
v := os.Getenv(key)
if v == "" {
return v, errEnvVarEmpty
}

return v, nil
}

func GetEnvBool(key string) (bool, error) {
s, err := GetEnvStr(key)
if err != nil {
return false, err
}
v, err := strconv.ParseBool(s)
if err != nil {
return false, err
}

return v, nil
}
69 changes: 50 additions & 19 deletions integration_embedded_derp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,41 +40,50 @@ type IntegrationDERPTestSuite struct {
pool dockertest.Pool
networks map[int]dockertest.Network // so we keep the containers isolated
headscale dockertest.Resource
saveLogs bool

tailscales map[string]dockertest.Resource
joinWaitGroup sync.WaitGroup
}

func TestDERPIntegrationTestSuite(t *testing.T) {
saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
if err != nil {
saveLogs = false
}

s := new(IntegrationDERPTestSuite)

s.tailscales = make(map[string]dockertest.Resource)
s.networks = make(map[int]dockertest.Network)
s.saveLogs = saveLogs

suite.Run(t, s)

// HandleStats, which allows us to check if we passed and save logs
// is called after TearDown, so we cannot tear down containers before
// we have potentially saved the logs.
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
if s.saveLogs {
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
}

if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}

for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
}
Expand All @@ -83,14 +92,14 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}

for i := 0; i < totalContainers; i++ {
if pnetwork, err := s.pool.CreateNetwork(fmt.Sprintf("headscale-derp-%d", i)); err == nil {
s.networks[i] = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}
}

Expand All @@ -101,7 +110,7 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {

currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}

headscaleOptions := &dockertest.RunOptions{
Expand All @@ -120,11 +129,16 @@ func (s *IntegrationDERPTestSuite) SetupSuite() {
},
}

err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}

log.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
log.Println("Created headscale container to test DERP")

Expand Down Expand Up @@ -290,6 +304,23 @@ func (s *IntegrationDERPTestSuite) tailscaleContainer(
}

func (s *IntegrationDERPTestSuite) TearDownSuite() {
if !s.saveLogs {
for _, tailscale := range s.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}

if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}

for _, network := range s.networks {
if err := network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}
}

func (s *IntegrationDERPTestSuite) HandleStats(
Expand Down
69 changes: 50 additions & 19 deletions integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,19 @@ type IntegrationTestSuite struct {
pool dockertest.Pool
network dockertest.Network
headscale dockertest.Resource
saveLogs bool

namespaces map[string]TestNamespace

joinWaitGroup sync.WaitGroup
}

func TestIntegrationTestSuite(t *testing.T) {
saveLogs, err := GetEnvBool("HEADSCALE_INTEGRATION_SAVE_LOG")
if err != nil {
saveLogs = false
}

s := new(IntegrationTestSuite)

s.namespaces = map[string]TestNamespace{
Expand All @@ -55,32 +61,35 @@ func TestIntegrationTestSuite(t *testing.T) {
tailscales: make(map[string]dockertest.Resource),
},
}
s.saveLogs = saveLogs

suite.Run(t, s)

// HandleStats, which allows us to check if we passed and save logs
// is called after TearDown, so we cannot tear down containers before
// we have potentially saved the logs.
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
if s.saveLogs {
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
}
}

if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
if !s.stats.Passed() {
err := s.saveLog(&s.headscale, "test_output")
if err != nil {
log.Printf("Could not save log: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}

if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}

Expand Down Expand Up @@ -209,13 +218,13 @@ func (s *IntegrationTestSuite) SetupSuite() {
if ppool, err := dockertest.NewPool(""); err == nil {
s.pool = *ppool
} else {
log.Fatalf("Could not connect to docker: %s", err)
s.FailNow(fmt.Sprintf("Could not connect to docker: %s", err), "")
}

if pnetwork, err := s.pool.CreateNetwork("headscale-test"); err == nil {
s.network = *pnetwork
} else {
log.Fatalf("Could not create network: %s", err)
s.FailNow(fmt.Sprintf("Could not create network: %s", err), "")
}

headscaleBuildOptions := &dockertest.BuildOptions{
Expand All @@ -225,7 +234,7 @@ func (s *IntegrationTestSuite) SetupSuite() {

currentPath, err := os.Getwd()
if err != nil {
log.Fatalf("Could not determine current path: %s", err)
s.FailNow(fmt.Sprintf("Could not determine current path: %s", err), "")
}

headscaleOptions := &dockertest.RunOptions{
Expand All @@ -237,11 +246,16 @@ func (s *IntegrationTestSuite) SetupSuite() {
Cmd: []string{"headscale", "serve"},
}

err = s.pool.RemoveContainerByName(headscaleHostname)
if err != nil {
s.FailNow(fmt.Sprintf("Could not remove existing container before building test: %s", err), "")
}

log.Println("Creating headscale container")
if pheadscale, err := s.pool.BuildAndRunWithBuildOptions(headscaleBuildOptions, headscaleOptions, DockerRestartPolicy); err == nil {
s.headscale = *pheadscale
} else {
log.Fatalf("Could not start headscale container: %s", err)
s.FailNow(fmt.Sprintf("Could not start headscale container: %s", err), "")
}
log.Println("Created headscale container")

Expand Down Expand Up @@ -338,6 +352,23 @@ func (s *IntegrationTestSuite) SetupSuite() {
}

func (s *IntegrationTestSuite) TearDownSuite() {
if !s.saveLogs {
for _, scales := range s.namespaces {
for _, tailscale := range scales.tailscales {
if err := s.pool.Purge(&tailscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}
}
}

if err := s.pool.Purge(&s.headscale); err != nil {
log.Printf("Could not purge resource: %s\n", err)
}

if err := s.network.Close(); err != nil {
log.Printf("Could not close network: %s\n", err)
}
}
}

func (s *IntegrationTestSuite) HandleStats(
Expand Down

0 comments on commit e0b15c1

Please sign in to comment.