From e139cedc2690923476b04bf7cf7bca6d8f4494ab Mon Sep 17 00:00:00 2001 From: Shawn Hurley Date: Fri, 1 Apr 2022 12:12:07 -0400 Subject: [PATCH] adding a test case for validating webhook --- test/e2e/conformance/webhook_test.go | 284 +++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 test/e2e/conformance/webhook_test.go diff --git a/test/e2e/conformance/webhook_test.go b/test/e2e/conformance/webhook_test.go new file mode 100644 index 000000000000..2c493d692314 --- /dev/null +++ b/test/e2e/conformance/webhook_test.go @@ -0,0 +1,284 @@ +/* +Copyright 2022 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/kcp-dev/apimachinery/pkg/logicalcluster" + "github.com/stretchr/testify/require" + + v1 "k8s.io/api/admission/v1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest" + "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest/v1alpha1" + client "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/client/clientset/versioned" + "github.com/kcp-dev/kcp/test/e2e/framework" +) + +func TestWebhookInWorkspace(t *testing.T) { + t.Parallel() + + server := framework.SharedKcpServer(t) + dirPath := filepath.Dir(server.KubeconfigPath()) + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + // using known path to cert and key + cfg := server.DefaultConfig(t) + cfg.CertFile = filepath.Join(dirPath, "apiserver.crt") + cfg.KeyFile = filepath.Join(dirPath, "apiserver.key") + + scheme := runtime.NewScheme() + err := admissionregistrationv1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission registration v1 scheme") + err = v1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission v1 scheme") + err = v1alpha1.AddToScheme(scheme) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") + + testWebhook := testWebhookServer{ + Response: v1.AdmissionResponse{ + Allowed: true, + }, + ObjectGVK: schema.GroupVersionKind{ + Group: "wildwest.dev", + Version: "v1alpha1", + Kind: "Cowboy", + }, + T: t, + Lock: sync.Mutex{}, + Scheme: scheme, + } + + port, err := framework.GetFreePort(t) + require.NoError(t, err, "failed to get free port for test webhook") + testWebhook.StartServer(ctx, cfg, port) + + organization := framework.NewOrganizationFixture(t, server) + logicalClusters := []logicalcluster.LogicalCluster{ + framework.NewWorkspaceFixture(t, server, organization, "Universal"), + framework.NewWorkspaceFixture(t, server, organization, "Universal"), + } + + kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + cowbyClients, err := client.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct cowboy client for server") + apiExtensionsClients, err := apiextensionsclient.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct apiextensions client for server") + + t.Logf("Install the Cowboy resources into logical clusters") + for _, logicalCluster := range logicalClusters { + t.Logf("Bootstrapping ClusterWorkspace CRDs in logical cluster %s", logicalCluster) + crdClient := apiExtensionsClients.Cluster(logicalCluster).ApiextensionsV1().CustomResourceDefinitions() + wildwest.Create(t, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"}) + } + + t.Logf("Installing webhook into the first workspace") + url := fmt.Sprintf("https://localhost:%v/hello", port) + sideEffect := admissionregistrationv1.SideEffectClassNone + webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: "test-webhook.cowboy.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: &url, + CABundle: cfg.CAData, + }, + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"wildwest.dev"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"cowboys"}, + }, + }}, + SideEffects: &sideEffect, + AdmissionReviewVersions: []string{"v1"}, + }}, + } + _, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) + require.NoError(t, err, "failed to add validating webhook configurations") + + cowboy := v1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing", + }, + Spec: v1alpha1.CowboySpec{}, + } + + // Avoid race condition here by making sure that CRD is served after installing the types into logical clusters + t.Logf("Creating cowboy resource in second logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(logicalClusters[1]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + require.Equal(t, 0, testWebhook.Calls, "expected that the webhook is not called for logical cluster where webhook is not installed") + + t.Logf("Creating cowboy resource in first logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(logicalClusters[0]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + //Avoid race condition where webhook informer is not updated before the call to create was made. + t.Log("Verify webhook is eventually called") + require.Eventually(t, func() bool { + return testWebhook.Calls == 1 + }, wait.ForeverTestTimeout, 100*time.Millisecond) + +} + +type testWebhookServer struct { + Response v1.AdmissionResponse + ObjectGVK schema.GroupVersionKind + T *testing.T + Scheme *runtime.Scheme + + Lock sync.Mutex + Calls int +} + +func (t *testWebhookServer) StartServer(ctx context.Context, cfg *rest.Config, port string) { + serv := &http.Server{Addr: fmt.Sprintf(":%v", port), Handler: t} + go func() { + <-ctx.Done() + t.T.Logf("Shutting down the HTTP server") + err := serv.Shutdown(context.TODO()) + if err != nil { + t.T.Errorf("failure to shutdown server: %v", err) + } + }() + go func() { + err := serv.ListenAndServeTLS(cfg.CertFile, cfg.KeyFile) + if err != nil && err != http.ErrServerClosed { + t.T.Error(err) + } + }() +} + +func (t *testWebhookServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + // Make sure that this is a request for the object that was set. + if req.Body == nil { + msg := "Expected request body to be non-empty" + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + } + + data, err := ioutil.ReadAll(req.Body) + if err != nil { + msg := fmt.Sprintf("Request could not be decoded: %v", err) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + } + + // verify the content type is accurate + contentType := req.Header.Get("Content-Type") + if contentType != "application/json" { + msg := fmt.Sprintf("contentType=%s, expect application/json", contentType) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + var codecs = serializer.NewCodecFactory(t.Scheme) + deserializer := codecs.UniversalDeserializer() + obj, gvk, err := deserializer.Decode(data, nil, nil) + if err != nil { + t.T.Errorf("%v", err) + } + + if *gvk != v1.SchemeGroupVersion.WithKind("AdmissionReview") { + msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) + if !ok { + //return an error + msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + obj, objGVK, err := deserializer.Decode(requestedAdmissionReview.Request.Object.Raw, nil, nil) + if err != nil { + t.T.Errorf("%v", err) + } + + if t.ObjectGVK != *objGVK { + //return an error + msg := fmt.Sprintf("Expected ObjectGVK: %v but got: %T", t.ObjectGVK, obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + responseAdmissionReview := &v1.AdmissionReview{ + TypeMeta: requestedAdmissionReview.TypeMeta, + } + responseAdmissionReview.Response = &t.Response + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + + respBytes, err := json.Marshal(responseAdmissionReview) + if err != nil { + t.T.Logf("%v", err) + http.Error(resp, err.Error(), http.StatusInternalServerError) + return + } + + t.Lock.Lock() + t.Calls = t.Calls + 1 + t.Lock.Unlock() + + resp.Header().Set("Content-Type", "application/json") + if _, err := resp.Write(respBytes); err != nil { + t.T.Logf("%v", err) + } +}