Merge pull request #60 from daimg/develop

run e2e case parallelly

internal/controllers/ingress/netv1/ingress_controller.go

@@ -27,7 +27,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/bfenetworks/ingress-bfe/internal/bfeConfig"
@@ -147,22 +146,6 @@ func ReconcileV1Ingress(ctx context.Context, r client.Client, configBuilder *bfe
 	return nil
 }
 
-func NamespaceFilter() predicate.Funcs {
-	funcs := predicate.NewPredicateFuncs(func(obj client.Object) bool {
-		if len(option.Opts.NamespaceList) == 0 {
-			return true
-		}
-		for _, ns := range option.Opts.NamespaceList {
-			if ns == obj.GetNamespace() {
-				return true
-			}
-		}
-		return false
-	})
-
-	return funcs
-}
-
 func getIngressBackends(ctx context.Context, r client.Reader, ingress *netv1.Ingress) (map[string]*corev1.Service, map[string]*corev1.Endpoints, error) {
 	services := make(map[string]*corev1.Service)
 	endpoints := make(map[string]*corev1.Endpoints)

test/e2e/README.md

@@ -1,17 +1,18 @@
 # BFE ingress controller e2e test
 
-The e2e test follows K8s project [ingress-controller-conformance](https://github.com/kubernetes-sigs/ingress-controller-conformance), and add more test cases for ingress-bfe special features.
+This test follows K8s project [ingress-controller-conformance](https://github.com/kubernetes-sigs/ingress-controller-conformance), and add more test cases for ingress-bfe special features.
 
-## Running
+## How to run
+
+To run all e2e test cases, execute following command in ingress-bfe project's top directory:
 
-In ingress-bfe project's top directory, execute:
 ``` 
 $ make e2e-test
 ```
 It would automatically start the whole testing with following procedures:
 
 - Build bfe-ingress-controller docker image
-- Prepare test environment, including setting up a local k8s cluster by [Kind](https://kind.sigs.k8s.io/), loading and applying the docker images, etc. The scripts used for preparing environment is located in [test/script](../script).
+- Prepare test environment, including spining up a local k8s cluster with [Kind](https://kind.sigs.k8s.io/), loading docker images, etc. All scripts used to prepare environment are located in [test/script](../script).
 - Execute test cases by running [run.sh](./run.sh), which actually build and execute program e2e_test.
 
 ## Contributing
@@ -26,40 +27,52 @@ Steps to add new test case as below:
 
 * Create feature file to describe your test case. 
 
-All existing feature files is under directory [test/e2e/features](./features). Please put your feature file into proper directory.
-  > Reuse existing steps from existing feature files.
+All feature files are under directory [test/e2e/features](./features). Please put your feature file into proper sub-directory. For example, features/<your-new-feature>/<your-new-feature>.feature
+  > Try to reuse steps from existing feature files if possible.
 
 ### Step2: Create steps definition
 
 * Generate steps.go for your case. Under directory test/e2e, run:
 
 ```bash
-$ go run hack/codegen.go -dest-path=steps/<dest-dir>  features/<xxx>.feature
+$ go run hack/codegen.go -dest-path=steps/<your-new-feature>  features/<your-new-feature>/<your-new-feature>.feature
 ```
 
 
-* Edit generated code, implement all generated functions. If you reuse step in other feature file, you also can reuse the logic from exist steps files.
+* Edit generated code, implement all generated functions. If you reuse step description from other feature file, you can also reuse corresponding function from that `step.go` file in this step.
 
-### Step3: Add step into e2e_test.go
+### Step3: Add Init function into e2e_test.go
 
 * In e2e_test.go, add generated feature file and InitializeScenario function into map `features`.
 
 ```go
 var (
-	features = map[string]func(*godog.ScenarioContext){
-		"features/conformance/host_rules.feature":           hostrules.InitializeScenario,
+	features = map[string]InitialFunc{
+		"features/conformance/host_rules.feature":           {hostrules.InitializeScenario, nil},
     ...
 	}
 )
 
 ```
 
-### Step4: Build and run case
+### Step4: Build and run
+
+
+
+* Build e2e_test:
 
-* Build e2e_test
 ```bash
 $ make build
-$ ./e2e_test --feature features/<xxx>.feature
 ```
 
-After your test case pass, commit and push the code.
+* Run your case, using `--feature` to specify the feature file.
+
+```bash
+$ ./e2e_test --feature features/<your-new-feature>/<your-new-feature>.feature
+```
+> Before running, your testing environment must be ready.
+
+* Run all cases
+```bash
+$ ./run.sh
+```

test/e2e/e2e_test.go

@@ -45,9 +45,11 @@ import (
 	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/ingressclass"
 	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/loadbalancing"
 	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/conformance/pathrules"
-	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/host"
+	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/host1"
+	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/host2"
 	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/multipleingress"
-	rules_path "github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/path"
+	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/path1"
+	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/path2"
 	"github.com/bfenetworks/ingress-bfe/test/e2e/steps/rules/patherr"
 )
 
@@ -58,6 +60,7 @@ var (
 	godogNoColors      bool
 	godogOutput        string
 	godogTestFeature   string
+	FeatureParallel    int
 )
 
 func TestMain(m *testing.M) {
@@ -76,6 +79,7 @@ func TestMain(m *testing.M) {
 	flag.StringVar(&kubernetes.IngressControllerServiceName, "ingress-controller-service-name", "bfe-controller-service", "Sets the name of the service for ingress controller")
 	flag.StringVar(&kubernetes.K8sNodeAddr, "k8s-node-addr", "127.0.0.1", "Sets the ip address of one k8s node")
 	flag.StringVar(&godogTestFeature, "feature", "", "Sets the file to test")
+	flag.IntVar(&FeatureParallel, "feature-parallel", 1, "Sets the file to test")
 	flag.BoolVar(&http.EnableDebug, "enable-http-debug", false, "Enable dump of requests and responses of HTTP requests (useful for debug)")
 	flag.BoolVar(&kubernetes.EnableOutputYamlDefinitions, "enable-output-yaml-definitions", false, "Dump yaml definitions of Kubernetes objects before creation")
 
@@ -114,43 +118,59 @@ func setup() error {
 	return nil
 }
 
+type InitialFunc struct {
+	Scenario func(*godog.ScenarioContext)
+	Suite    func(*godog.TestSuiteContext)
+}
+
 var (
-	features = map[string]func(*godog.ScenarioContext){
-		"features/conformance/host_rules.feature":           hostrules.InitializeScenario,
-		"features/conformance/ingress_class.feature":        ingressclass.InitializeScenario,
-		"features/conformance/load_balancing.feature":       loadbalancing.InitializeScenario,
-		"features/conformance/path_rules.feature":           pathrules.InitializeScenario,
-		"features/rules/host_rule1.feature":                 host.InitializeScenario,
-		"features/rules/host_rule2.feature":                 host.InitializeScenario,
-		"features/rules/multiple_ingress.feature":           multipleingress.InitializeScenario,
-		"features/rules/path_rule1.feature":                 rules_path.InitializeScenario,
-		"features/rules/path_rule2.feature":                 rules_path.InitializeScenario,
-		"features/rules/path_err.feature":                   patherr.InitializeScenario,
-		"features/annotations/route/cookie.feature":         cookie.InitializeScenario,
-		"features/annotations/route/header.feature":         header.InitializeScenario,
-		"features/annotations/route/priority.feature":       priority.InitializeScenario,
-		"features/annotations/balance/load_balance.feature": loadbalance.InitializeScenario,
+	features = map[string]InitialFunc{
+		"features/conformance/host_rules.feature":           {hostrules.InitializeScenario, nil},
+		"features/conformance/ingress_class.feature":        {ingressclass.InitializeScenario, nil},
+		"features/conformance/load_balancing.feature":       {loadbalancing.InitializeScenario, nil},
+		"features/conformance/path_rules.feature":           {pathrules.InitializeScenario, pathrules.InitializeSuite},
+		"features/rules/host_rule1.feature":                 {host1.InitializeScenario, nil},
+		"features/rules/host_rule2.feature":                 {host2.InitializeScenario, nil},
+		"features/rules/multiple_ingress.feature":           {multipleingress.InitializeScenario, nil},
+		"features/rules/path_rule1.feature":                 {path1.InitializeScenario, path1.InitializeSuite},
+		"features/rules/path_rule2.feature":                 {path2.InitializeScenario, path2.InitializeSuite},
+		"features/rules/path_err.feature":                   {patherr.InitializeScenario, nil},
+		"features/annotations/route/cookie.feature":         {cookie.InitializeScenario, nil},
+		"features/annotations/route/header.feature":         {header.InitializeScenario, nil},
+		"features/annotations/route/priority.feature":       {priority.InitializeScenario, nil},
+		"features/annotations/balance/load_balance.feature": {loadbalance.InitializeScenario, nil},
 	}
 )
 
 func TestSuite(t *testing.T) {
 	var failed bool
 
-	activeFeatures := make(map[string]func(*godog.ScenarioContext))
+	activeFeatures := make(map[string]InitialFunc)
 	for file, init := range features {
 		if strings.HasPrefix(file, godogTestFeature) {
 			activeFeatures[file] = init
 		}
 	}
 
-	for feature, scenarioContext := range activeFeatures {
-		err := testFeature(feature, scenarioContext)
-		if err != nil {
-			if godogStopOnFailure {
-				t.Fatal(err)
+	queue := make(chan int, FeatureParallel)
+
+	for i := 0; i < FeatureParallel; i++ {
+		queue <- 1
+	}
+
+	for feature, initFunc := range activeFeatures {
+		<-queue
+		go func(feature string, init InitialFunc) {
+			err := testFeature(feature, init)
+			if err != nil {
+				failed = true
 			}
-			failed = true
-		}
+			queue <- 1
+		}(feature, initFunc)
+	}
+
+	for i := 0; i < FeatureParallel; i++ {
+		<-queue
 	}
 
 	if failed {
@@ -159,7 +179,7 @@ func TestSuite(t *testing.T) {
 
 }
 
-func testFeature(feature string, scenarioInitializer func(*godog.ScenarioContext)) error {
+func testFeature(feature string, initFunc InitialFunc) error {
 	var testOutput io.Writer
 	// default output is stdout
 	testOutput = os.Stdout
@@ -190,9 +210,10 @@ func testFeature(feature string, scenarioInitializer func(*godog.ScenarioContext
 	}
 
 	exitCode := godog.TestSuite{
-		Name:                "e2e-test",
-		ScenarioInitializer: scenarioInitializer,
-		Options:             &opts,
+		Name:                 "e2e-test",
+		TestSuiteInitializer: initFunc.Suite,
+		ScenarioInitializer:  initFunc.Scenario,
+		Options:              &opts,
 	}.Run()
 	if exitCode > 0 {
 		return fmt.Errorf("unexpected exit code testing %v: %v", feature, exitCode)

test/e2e/images/reports/src/index.js

@@ -7,7 +7,7 @@ assert(process.env.OUTPUT_DIRECTORY, "Environment variable OUTPUT_DIRECTORY is n
 report.generate({
   jsonDir: process.env.INPUT_DIRECTORY,
   reportPath: process.env.OUTPUT_DIRECTORY,
-  pageFooter: '<p><a href="https://github.com/bfenetworks/ingress-bfe/test/e2e">BFE ingress controller e2e test</a></p>',
+  pageFooter: '<p><a href="https://github.com/bfenetworks/ingress-bfe/tree/develop/test/e2e">BFE ingress controller e2e test</a></p>',
   ingress: {
     controller: process.env.INGRESS_CONTROLLER || 'N/A',
     version: process.env.CONTROLLER_VERSION || 'N/A'

test/e2e/pkg/kubernetes/kubernetes.go

@@ -46,6 +46,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/flowcontrol"
 	"sigs.k8s.io/yaml"
 
 	// ensure auth plugins are loaded
@@ -87,6 +88,8 @@ func LoadClientset() (*clientset.Clientset, error) {
 		}
 	}
 
+	config.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(100, 100)
+
 	// TODO: add version information?
 	config.UserAgent = fmt.Sprintf(
 		"%s (%s/%s) ingress-conformance",
@@ -105,7 +108,7 @@ func LoadClientset() (*clientset.Clientset, error) {
 
 // NewNamespace creates a new namespace using ingress-conformance- as prefix.
 func NewNamespace(c kubernetes.Interface) (string, error) {
-	ns := &corev1.Namespace{
+	nsParam := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "ingress-conformance-",
 			Labels: map[string]string{
@@ -116,12 +119,12 @@ func NewNamespace(c kubernetes.Interface) (string, error) {
 
 	var err error
 
-	err = displayYamlDefinition(ns)
+	err = displayYamlDefinition(nsParam)
 	if err != nil {
 		return "", fmt.Errorf("unable show yaml definition: %v", err)
 	}
 
-	ns, err = c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
+	ns, err := c.CoreV1().Namespaces().Create(context.TODO(), nsParam, metav1.CreateOptions{})
 	if err != nil {
 		return "", fmt.Errorf("unable to create namespace: %v", err)
 	}
@@ -130,7 +133,7 @@ func NewNamespace(c kubernetes.Interface) (string, error) {
 }
 
 // DeleteNamespace deletes a namespace and all the objects inside
-func DeleteNamespace(c kubernetes.Interface, namespace string) error {
+func DeleteNamespaceBlocking(c kubernetes.Interface, namespace string) error {
 	grace := int64(0)
 	pb := metav1.DeletePropagationForeground
 
@@ -145,7 +148,7 @@ func DeleteNamespace(c kubernetes.Interface, namespace string) error {
 }
 
 // DeleteNamespace deletes a namespace and all the objects inside
-func DeleteNamespaceNonBlock(c kubernetes.Interface, namespace string) error {
+func DeleteNamespace(c kubernetes.Interface, namespace string) error {
 	grace := int64(0)
 	pb := metav1.DeletePropagationBackground
 
@@ -179,7 +182,7 @@ func CleanupNamespaces(c kubernetes.Interface) error {
 	}
 
 	for _, namespace := range namespaces.Items {
-		err := DeleteNamespaceNonBlock(c, namespace.Name)
+		err := DeleteNamespace(c, namespace.Name)
 		if err != nil {
 			return err
 		}
@@ -435,7 +438,7 @@ func isRetryableAPIError(err error) bool {
 	}
 
 	// in case backend start slowly
-	if err != nil && strings.Contains(err.Error(), "no avail backend") {
+	if err != nil && (strings.Contains(err.Error(), "no avail backend") || strings.Contains(err.Error(), "conflict with")) {
 		return true
 	}
 

test/e2e/run.sh

@@ -32,14 +32,16 @@ make build
 CUCUMBER_OUTPUT_FORMAT="${CUCUMBER_OUTPUT_FORMAT:-pretty}"
 WAIT_FOR_STATUS_TIMEOUT="${WAIT_FOR_STATUS_TIMEOUT:-5m}"
 TEST_TIMEOUT="${TEST_TIMEOUT:-0}"
+TEST_PARALLEL="${TEST_PARALLEL:-5}"
 
 ./e2e_test \
     --output-directory="${RESULTS_DIR}" \
     --feature="${CUCUMBER_FEATURE}" \
     --format="${CUCUMBER_OUTPUT_FORMAT}" \
     --wait-time-for-ingress-status="${WAIT_FOR_STATUS_TIMEOUT}" \
     --wait-time-for-ready="${WAIT_FOR_STATUS_TIMEOUT}" \
-    --test.timeout="${TEST_TIMEOUT}"
+    --test.timeout="${TEST_TIMEOUT}" \
+    --feature-parallel="${TEST_PARALLEL}"
 ret=$?
 
 exit 0