Skip to content

Commit

Permalink
[FEAT] added version cleanup monitoring
Browse files Browse the repository at this point in the history
- `CAPApplicationVersion` enhanced with
  - deletion rules based on metric rules or PromQL expression
  - scrape configuration
- New routine to periodically collect relevant versions and queue them for evaluation
- New routine to process queued versions by evaluating metrics rules
- added service monitor creation for workload services based on scrape configuration
  • Loading branch information
skrishnan-sap committed Sep 17, 2024
1 parent 9ac11b5 commit 2a3049c
Show file tree
Hide file tree
Showing 36 changed files with 2,656 additions and 29 deletions.
15 changes: 14 additions & 1 deletion cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import (
"github.com/sap/cap-operator/pkg/client/clientset/versioned"
istio "istio.io/client-go/pkg/clientset/versioned"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

promop "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
)

const (
Expand All @@ -52,6 +55,16 @@ func main() {
klog.Fatal("could not create client for custom resources: ", err.Error())
}

apiExtClient, err := apiext.NewForConfig(config)
if err != nil {
klog.Fatal("could not create client for api-extensions: ", err.Error())
}

promClient, err := promop.NewForConfig(config)
if err != nil {
klog.Fatal("could not create client for prometheus-operator resources: ", err.Error())
}

istioClient, err := istio.NewForConfig(config)
if err != nil {
klog.Fatal("could not create client for istio resources: ", err.Error())
Expand Down Expand Up @@ -107,7 +120,7 @@ func main() {
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
klog.InfoS("Started leading: ", LeaseLockName, leaseLockId)
c := controller.NewController(coreClient, crdClient, istioClient, certClient, certManagerClient, dnsClient)
c := controller.NewController(coreClient, crdClient, istioClient, certClient, certManagerClient, dnsClient, apiExtClient, promClient)
go c.Start(ctx)
},
OnStoppedLeading: func() {
Expand Down
51 changes: 51 additions & 0 deletions crds/sme.sap.com_capapplicationversions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,57 @@ spec:
format: int32
type: integer
type: object
monitoring:
properties:
deletionRules:
oneOf:
- required:
- metrics
- required:
- expression
properties:
expression:
type: string
metrics:
items:
properties:
calculationPeriod:
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
type: string
name:
type: string
thresholdValue:
format: double
type: string
type:
enum:
- Gauge
- Counter
type: string
required:
- calculationPeriod
- name
- thresholdValue
- type
type: object
type: array
type: object
scrapeConfig:
properties:
interval:
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
type: string
path:
type: string
port:
type: string
scrapeTimeout:
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
type: string
required:
- port
type: object
type: object
nodeName:
type: string
nodeSelector:
Expand Down
12 changes: 9 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,23 @@ require (
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/lestrrat-go/jwx/v2 v2.1.1
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2
github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2
github.com/prometheus/client_golang v1.20.2
github.com/prometheus/common v0.55.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
golang.org/x/mod v0.21.0
google.golang.org/protobuf v1.34.2
istio.io/api v1.23.1
istio.io/client-go v1.23.1
k8s.io/api v0.31.1
k8s.io/apiextensions-apiserver v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/client-go v0.31.1
k8s.io/code-generator v0.31.1
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/structured-merge-diff/v4 v4.4.1
)

require (
Expand Down Expand Up @@ -52,12 +59,12 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
golang.org/x/net v0.29.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
Expand All @@ -71,12 +78,11 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.31.1 // indirect
k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect
k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
sigs.k8s.io/controller-runtime v0.18.5 // indirect
sigs.k8s.io/gateway-api v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1
sigs.k8s.io/yaml v1.4.0 // indirect
)
23 changes: 23 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k=
github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cert-manager/cert-manager v1.15.3 h1:/u9T0griwd5MegPfWbB7v0KcVcT9OJrEvPNhc9tl7xQ=
github.com/cert-manager/cert-manager v1.15.3/go.mod h1:stBge/DTvrhfQMB/93+Y62s+gQgZBsfL1o0C/4AL/mI=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
Expand All @@ -26,6 +30,7 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF
github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
Expand Down Expand Up @@ -54,6 +59,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
Expand Down Expand Up @@ -83,6 +90,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os=
Expand All @@ -92,6 +101,18 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2 h1:BpGDC87A2SaxbKgONsFLEX3kRcRJee2aLQbjXsuz0hA=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2/go.mod h1:Rd8YnCqz+2FYsiGmE2DMlaLjQRB4v2jFNnzCt9YY4IM=
github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2 h1:yncs8NglhE3hB+viNsabCAF9TBBDOBljHUyxHC5fSGY=
github.com/prometheus-operator/prometheus-operator/pkg/client v0.76.2/go.mod h1:AfbzyEUFxJmSoTiMcgNHHjDKcorBVd9TIwx0viURgEw=
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
Expand Down Expand Up @@ -200,6 +221,8 @@ k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUx
k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA=
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk=
sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM=
sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
Expand Down
63 changes: 50 additions & 13 deletions internal/controller/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ import (
gardenerdnsscheme "github.com/gardener/external-dns-management/pkg/client/dns/clientset/versioned/scheme"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
promopFake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake"
promopScheme "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/scheme"
"github.com/sap/cap-operator/pkg/apis/sme.sap.com/v1alpha1"
copfake "github.com/sap/cap-operator/pkg/client/clientset/versioned/fake"
smeScheme "github.com/sap/cap-operator/pkg/client/clientset/versioned/scheme"
Expand All @@ -42,6 +45,9 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextFake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
apiExtScheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -210,13 +216,17 @@ func initializeControllerForReconciliationTests(t *testing.T, items []ResourceAc
gardenerdnsscheme.AddToScheme(scheme.Scheme)
istioscheme.AddToScheme(scheme.Scheme)
certManagerScheme.AddToScheme(scheme.Scheme)
apiExtScheme.AddToScheme(scheme.Scheme)
promopScheme.AddToScheme(scheme.Scheme)

coreClient := k8sfake.NewSimpleClientset()
copClient := copfake.NewSimpleClientset()
istioClient := istiofake.NewSimpleClientset()
gardenerCertClient := gardenercertfake.NewSimpleClientset()
gardenerDNSClient := gardenerdnsfake.NewSimpleClientset()
certManagerClient := certManagerFake.NewSimpleClientset()
apiExtClient := apiextFake.NewSimpleClientset()
promopClient := promopFake.NewSimpleClientset()

copClient.PrependReactor("create", "*", generateNameCreateHandler)
copClient.PrependReactor("update", "*", removeStatusTimestampHandler)
Expand All @@ -239,7 +249,7 @@ func initializeControllerForReconciliationTests(t *testing.T, items []ResourceAc
gardenerCertClient.PrependReactor("*", "*", getErrorReactorWithResources(t, items))
gardenerCertClient.PrependReactor("delete-collection", "*", getDeleteCollectionHandler(t, gardenerDNSClient))

c := NewController(coreClient, copClient, istioClient, gardenerCertClient, certManagerClient, gardenerDNSClient)
c := NewController(coreClient, copClient, istioClient, gardenerCertClient, certManagerClient, gardenerDNSClient, apiExtClient, promopClient)
c.eventRecorder = events.NewFakeRecorder(10)
return c
}
Expand Down Expand Up @@ -408,22 +418,16 @@ func processTestData(t *testing.T, c *Controller, data TestData, dataType TestDa

var processFile = func(file string) {
defer wg.Done()
i, err := os.ReadFile(file)

resources, err := readYAMLResourcesFromFile(file)
if err != nil {
t.Error(err.Error())
}

fileContents := string(i)
splits := strings.Split(fileContents, "---")
for _, part := range splits {
if part == "\n" || part == "" {
continue
}

for i := range resources {
if dataType == TestDataTypeInitial {
err = addInitialObjectToStore(t, []byte(part), c)
err = addInitialObjectToStore(resources[i], c)
} else {
err = compareExpectedWithStore(t, []byte(part), c)
err = compareExpectedWithStore(t, resources[i], c)
}
if err != nil {
t.Error(err.Error())
Expand All @@ -438,7 +442,25 @@ func processTestData(t *testing.T, c *Controller, data TestData, dataType TestDa
wg.Wait()
}

func addInitialObjectToStore(t *testing.T, resource []byte, c *Controller) error {
func readYAMLResourcesFromFile(file string) ([][]byte, error) {
i, err := os.ReadFile(file)
if err != nil {
return nil, err
}

resources := [][]byte{}
fileContents := string(i)
splits := strings.Split(fileContents, "---")
for _, part := range splits {
if part == "\n" || part == "" {
continue
}
resources = append(resources, []byte(part))
}
return resources, nil
}

func addInitialObjectToStore(resource []byte, c *Controller) error {
decoder := scheme.Codecs.UniversalDeserializer().Decode
obj, _, err := decoder(resource, nil, nil)
if err != nil {
Expand Down Expand Up @@ -533,6 +555,18 @@ func addInitialObjectToStore(t *testing.T, resource []byte, c *Controller) error
case *v1alpha1.CAPTenantOperation:
err = c.crdInformerFactory.Sme().V1alpha1().CAPTenantOperations().Informer().GetIndexer().Add(obj)
}
case *apiextv1.CustomResourceDefinition:
fakeClient, ok := c.apiExtClient.(*apiextFake.Clientset)
if !ok {
return fmt.Errorf("controller is not using a fake clientset")
}
fakeClient.Tracker().Add(obj)
case *monv1.ServiceMonitor:
fakeClient, ok := c.promClient.(*promopFake.Clientset)
if !ok {
return fmt.Errorf("controller is not using a fake clientset")
}
fakeClient.Tracker().Add(obj)
default:
return fmt.Errorf("unknown object type")
}
Expand Down Expand Up @@ -592,6 +626,9 @@ func compareExpectedWithStore(t *testing.T, resource []byte, c *Controller) erro
case *v1alpha1.CAPTenantOperation:
actual, err = fakeClient.Tracker().Get(gvk.GroupVersion().WithResource("captenantoperations"), mo.GetNamespace(), mo.GetName())
}
case *monv1.ServiceMonitor:
fakeClient := c.promClient.(*promopFake.Clientset)
actual, err = fakeClient.Tracker().Get(gvk.GroupVersion().WithResource("servicemonitors"), mo.GetNamespace(), mo.GetName())
default:
return fmt.Errorf("unknown expected object type")
}
Expand Down
Loading

0 comments on commit 2a3049c

Please sign in to comment.