From 93fa2ca69ee37919c3cb48639bc995df62f4bef1 Mon Sep 17 00:00:00 2001 From: Heba Elayoty <31887807+helayoty@users.noreply.github.com> Date: Mon, 22 Aug 2022 22:31:50 +0200 Subject: [PATCH] enhancement: Upgrade to nodeutil (#211) --- cmd/virtual-kubelet/context.go | 27 ++ cmd/virtual-kubelet/main.go | 304 +++++++++++++++++++---- cmd/virtual-kubelet/ocagent.go | 67 +++-- cmd/virtual-kubelet/signal_unix.go | 13 + cmd/virtual-kubelet/signal_windows.go | 7 + go.mod | 10 +- go.sum | 111 ++++++++- provider/aci.go | 75 ++++-- provider/aci_test.go | 340 ++++++++++++-------------- provider/config.go | 39 +-- provider/metrics/metrics.go | 47 ++-- provider/metrics/metrics_test.go | 34 +-- provider/metrics/mock_metrics_test.go | 53 ++-- provider/mocks_k8s_listers_test.go | 28 +-- provider/podsTracker.go | 64 +++-- 15 files changed, 820 insertions(+), 399 deletions(-) create mode 100644 cmd/virtual-kubelet/context.go create mode 100644 cmd/virtual-kubelet/signal_unix.go create mode 100644 cmd/virtual-kubelet/signal_windows.go diff --git a/cmd/virtual-kubelet/context.go b/cmd/virtual-kubelet/context.go new file mode 100644 index 00000000..8967888e --- /dev/null +++ b/cmd/virtual-kubelet/context.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "os" + "os/signal" +) + +func BaseContext(ctx context.Context) (context.Context, func()) { + sigC := make(chan os.Signal, 1) + ctx, cancel := context.WithCancel(ctx) + + go func() { + for { + select { + case <-ctx.Done(): + signal.Stop(sigC) + return + case <-sigC: + cancel() + } + } + }() + + signal.Notify(sigC, cancelSigs()...) + return ctx, cancel +} diff --git a/cmd/virtual-kubelet/main.go b/cmd/virtual-kubelet/main.go index 1949c7f6..2a93181f 100644 --- a/cmd/virtual-kubelet/main.go +++ b/cmd/virtual-kubelet/main.go @@ -16,71 +16,285 @@ package main import ( "context" + "crypto/tls" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strconv" "strings" + "time" + "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" + "github.com/spf13/cobra" azprovider "github.com/virtual-kubelet/azure-aci/provider" - cli "github.com/virtual-kubelet/node-cli" - logruscli "github.com/virtual-kubelet/node-cli/logrus" - opencensuscli "github.com/virtual-kubelet/node-cli/opencensus" - "github.com/virtual-kubelet/node-cli/opts" - "github.com/virtual-kubelet/node-cli/provider" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + "github.com/virtual-kubelet/virtual-kubelet/node" + "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" + v1 "k8s.io/api/core/v1" + "github.com/virtual-kubelet/virtual-kubelet/log" logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" - "github.com/virtual-kubelet/virtual-kubelet/trace" - "github.com/virtual-kubelet/virtual-kubelet/trace/opencensus" + "k8s.io/klog" ) var ( - buildVersion = "N/A" - buildTime = "N/A" - k8sVersion = "v1.19.10" // This should follow the version of k8s.io/client-go we are importing - numberOfWorkers = 50 + buildVersion = "N/A" + k8sVersion = "v1.19.10" // This should follow the version of k8s.io/client-go we are importing ) func main() { - ctx := cli.ContextWithCancelOnSignal(context.Background()) + prog := filepath.Base(os.Args[0]) + desc := prog + " implements a node on a Kubernetes cluster using Azure Container Instances to run pods." + + var ( + taintKey = envOrDefault("VKUBELET_TAINT_KEY", "virtual-kubelet.io/provider") + taintEffect = envOrDefault("VKUBELET_TAINT_EFFECT", string(v1.TaintEffectNoSchedule)) + taintValue = envOrDefault("VKUBELET_TAINT_VALUE", "azure") + + logLevel = "info" + traceSampleRate string + + // for aci + kubeconfigPath = os.Getenv("KUBECONFIG") + cfgPath string + clusterDomain = "cluster.local" + startupTimeout time.Duration + disableTaint bool + operatingSystem = "Linux" + numberOfWorkers = 50 + resync time.Duration + + certPath = os.Getenv("APISERVER_CERT_LOCATION") + keyPath = os.Getenv("APISERVER_KEY_LOCATION") + clientCACert = os.Getenv("APISERVER_CA_CERT_LOCATION") + clientNoVerify bool - logger := logrus.StandardLogger() - log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) - logConfig := &logruscli.Config{LogLevel: "info"} + webhookAuth bool + webhookAuthnCacheTTL time.Duration + webhookAuthzUnauthedCacheTTL time.Duration + webhookAuthzAuthedCacheTTL time.Duration + nodeName string + listenPort = 10250 + + // deprecated + namespace string + metricsAddr string + provider string + leases bool + ) + + if kubeconfigPath == "" { + home, _ := homedir.Dir() + if home != "" { + kubeconfigPath = filepath.Join(home, ".kube", "config") + } + } - trace.T = opencensus.Adapter{} - traceConfig := opencensuscli.Config{ - AvailableExporters: map[string]opencensuscli.ExporterInitFunc{ - "ocagent": initOCAgent, + withTaint := func(cfg *nodeutil.NodeConfig) error { + if disableTaint { + return nil + } + + taint := v1.Taint{ + Key: taintKey, + Value: taintValue, + } + switch taintEffect { + case "NoSchedule": + taint.Effect = v1.TaintEffectNoSchedule + case "NoExecute": + taint.Effect = v1.TaintEffectNoExecute + case "PreferNoSchedule": + taint.Effect = v1.TaintEffectPreferNoSchedule + default: + return errdefs.InvalidInputf("taint effect %q is not supported", taintEffect) + } + cfg.NodeSpec.Spec.Taints = append(cfg.NodeSpec.Spec.Taints, taint) + return nil + } + withVersion := func(cfg *nodeutil.NodeConfig) error { + cfg.NodeSpec.Status.NodeInfo.KubeletVersion = strings.Join([]string{k8sVersion, "vk-azure-aci", buildVersion}, "-") + return nil + } + + withWebhookAuth := func(cfg *nodeutil.NodeConfig) error { + if !webhookAuth { + return nil + } + auth, err := nodeutil.WebhookAuth(cfg.Client, cfg.NodeSpec.Name, func(cfg *nodeutil.WebhookAuthConfig) error { + if webhookAuthnCacheTTL > 0 { + cfg.AuthnConfig.CacheTTL = webhookAuthnCacheTTL + } + if webhookAuthzAuthedCacheTTL > 0 { + cfg.AuthzConfig.AllowCacheTTL = webhookAuthzAuthedCacheTTL + } + if webhookAuthzUnauthedCacheTTL > 0 { + cfg.AuthzConfig.AllowCacheTTL = webhookAuthzUnauthedCacheTTL + } + return nil + }) + if err != nil { + return err + } + cfg.TLSConfig.ClientAuth = tls.RequestClientCert + cfg.Handler = nodeutil.WithAuth(auth, cfg.Handler) + return nil + } + + withCA := func(cfg *tls.Config) error { + if clientCACert == "" { + return nil + } + if err := nodeutil.WithCAFromPath(clientCACert)(cfg); err != nil { + return fmt.Errorf("error getting CA from path: %w", err) + } + if clientNoVerify { + cfg.ClientAuth = tls.NoClientCert + } + return nil + } + withClient := func(cfg *nodeutil.NodeConfig) error { + client, err := nodeutil.ClientsetFromEnv(kubeconfigPath) + if err != nil { + return err + } + return nodeutil.WithClient(client)(cfg) + } + + run := func(ctx context.Context) error { + if err := configureTracing(nodeName, traceSampleRate); err != nil { + return err + } + node, err := nodeutil.NewNode(nodeName, func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) { + if p := os.Getenv("KUBELET_PORT"); p != "" { + var err error + listenPort, err = strconv.Atoi(p) + if err != nil { + return nil, nil, err + } + } + p, err := azprovider.NewACIProvider(cfgPath, cfg, os.Getenv("VKUBELET_POD_IP"), int32(listenPort), clusterDomain) + return p, nil, err }, + withClient, + withTaint, + withVersion, + nodeutil.WithTLSConfig(nodeutil.WithKeyPairFromPath(certPath, keyPath), withCA), + withWebhookAuth, + func(cfg *nodeutil.NodeConfig) error { + cfg.InformerResyncPeriod = resync + cfg.NumWorkers = numberOfWorkers + cfg.HTTPListenAddr = fmt.Sprintf(":%d", listenPort) + return nil + }, + ) + if err != nil { + return err + } + + go func() error { + err = node.Run(ctx) + if err != nil { + return fmt.Errorf("error running the node: %w", err) + } + return nil + }() + + if err := node.WaitReady(ctx, startupTimeout); err != nil { + return fmt.Errorf("error waiting for node to be ready: %w", err) + } + + <-node.Done() + return node.Err() } - o, err := opts.FromEnv() - if err != nil { - log.G(ctx).Fatal(err) + cmd := &cobra.Command{ + Use: prog, + Short: desc, + Long: desc, + Run: func(cmd *cobra.Command, args []string) { + logger := logrus.StandardLogger() + lvl, err := logrus.ParseLevel(logLevel) + if err != nil { + logrus.WithError(err).Fatal("Error parsing log level") + } + logger.SetLevel(lvl) + + ctx := log.WithLogger(cmd.Context(), logruslogger.FromLogrus(logrus.NewEntry(logger))) + + if err := run(ctx); err != nil { + if !errors.Is(err, context.Canceled) { + log.G(ctx).Fatal(err) + } + log.G(ctx).Debug(err) + } + }, } - o.Provider = "azure" - o.Version = strings.Join([]string{k8sVersion, "vk-azure-aci", buildVersion}, "-") - o.PodSyncWorkers = numberOfWorkers - - node, err := cli.New(ctx, - cli.WithBaseOpts(o), - cli.WithCLIVersion(buildVersion, buildTime), - cli.WithProvider("azure", func(cfg provider.InitConfig) (provider.Provider, error) { - return azprovider.NewACIProvider(cfg.ConfigPath, cfg.ResourceManager, cfg.NodeName, cfg.OperatingSystem, cfg.InternalIP, cfg.DaemonPort, cfg.KubeClusterDomain) - }), - cli.WithPersistentFlags(logConfig.FlagSet()), - cli.WithPersistentPreRunCallback(func() error { - return logruscli.Configure(logConfig, logger) - }), - cli.WithPersistentFlags(traceConfig.FlagSet()), - cli.WithPersistentPreRunCallback(func() error { - return opencensuscli.Configure(ctx, &traceConfig, o) - }), - ) - if err != nil { - log.G(ctx).Fatal(err) + flags := cmd.Flags() + + klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(klogFlags) + klogFlags.VisitAll(func(f *flag.Flag) { + f.Name = "klog." + f.Name + flags.AddGoFlag(f) + }) + + flags.StringVar(&nodeName, "nodename", nodeName, "kubernetes node name") + flags.StringVar(&cfgPath, "provider-config", cfgPath, "cloud provider configuration file") + flags.StringVar(&clusterDomain, "cluster-domain", clusterDomain, "kubernetes cluster-domain") + flag.DurationVar(&startupTimeout, "startup-timeout", startupTimeout, "How long to wait for the virtual-kubelet to start") + flags.BoolVar(&disableTaint, "disable-taint", disableTaint, "disable the node taint") + flag.StringVar(&operatingSystem, "os", operatingSystem, "Operating System (Linux/Windows)") + flags.IntVar(&numberOfWorkers, "pod-sync-workers", numberOfWorkers, `set the number of pod synchronization workers`) + flags.DurationVar(&resync, "full-resync-period", resync, "how often to perform a full resync of pods between kubernetes and the provider") + + flags.StringVar(&clientCACert, "client-verify-ca", clientCACert, "CA cert to use to verify client requests") + flags.BoolVar(&clientNoVerify, "no-verify-clients", clientNoVerify, "Do not require client certificate validation") + flags.BoolVar(&webhookAuth, "authentication-token-webhook", webhookAuth, ""+ + "Use the TokenReview API to determine authentication for bearer tokens.") + flags.DurationVar(&webhookAuthnCacheTTL, "authentication-token-webhook-cache-ttl", webhookAuthnCacheTTL, + "The duration to cache responses from the webhook token authenticator.") + flags.DurationVar(&webhookAuthzAuthedCacheTTL, "authorization-webhook-cache-authorized-ttl", webhookAuthzAuthedCacheTTL, + "The duration to cache 'authorized' responses from the webhook authorizer.") + flags.DurationVar(&webhookAuthzUnauthedCacheTTL, "authorization-webhook-cache-unauthorized-ttl", webhookAuthzUnauthedCacheTTL, + "The duration to cache 'unauthorized' responses from the webhook authorizer.") + + flags.StringVar(&traceSampleRate, "trace-sample-rate", traceSampleRate, "set probability of tracing samples") + + // deprecated flags + flags.StringVar(&namespace, "namespace", namespace, "set namespace to watch for pods") + flags.MarkDeprecated("namespace", "cannot set namespace, all namespaces watched") + flags.MarkHidden("namespace") + flags.StringVar(&metricsAddr, "metrics-addr", metricsAddr, "address to listen for metrics/stats requests") + flags.MarkDeprecated("metrics-addr", "metrics are only available on the main api port") + flags.MarkHidden("metrics-addr") + flags.StringVar(&provider, "provider", provider, "cloud provider") + flags.MarkDeprecated("provider", "only one provider is supported") + flags.MarkHidden("provider") + flags.BoolVar(&leases, "enable-node-lease", leases, "use node leases for heartbeats") + flags.MarkDeprecated("leases", "Leases are always enabled") + flags.MarkHidden("leases") + flags.StringVar(&taintKey, "taint", taintKey, "Set node taint key") + flags.MarkDeprecated("taint", "Taint key should now be configured using the VKUBELET_TAINT_KEY environment variable") + + ctx, cancel := BaseContext(context.Background()) + defer cancel() + + if err := cmd.ExecuteContext(ctx); err != nil { + if !errors.Is(err, context.Canceled) { + logrus.WithError(err).Fatal("Error running command") + } } +} - if err := node.Run(ctx); err != nil { - log.G(ctx).Fatal(err) +func envOrDefault(key string, defaultValue string) string { + v, set := os.LookupEnv(key) + if set { + return v } + return defaultValue } diff --git a/cmd/virtual-kubelet/ocagent.go b/cmd/virtual-kubelet/ocagent.go index e4d2088e..3f3eb540 100644 --- a/cmd/virtual-kubelet/ocagent.go +++ b/cmd/virtual-kubelet/ocagent.go @@ -1,30 +1,69 @@ package main import ( + "fmt" "os" - + "strconv" + "strings" + "contrib.go.opencensus.io/exporter/ocagent" - opencensuscli "github.com/virtual-kubelet/node-cli/opencensus" "github.com/virtual-kubelet/virtual-kubelet/errdefs" - "go.opencensus.io/trace" + "github.com/virtual-kubelet/virtual-kubelet/trace" + "github.com/virtual-kubelet/virtual-kubelet/trace/opencensus" + octrace "go.opencensus.io/trace" ) -func initOCAgent(c *opencensuscli.Config) (trace.Exporter, error) { - agentOpts := append([]ocagent.ExporterOption{}, ocagent.WithServiceName(c.ServiceName)) - - if endpoint := os.Getenv("OCAGENT_ENDPOINT"); endpoint != "" { - agentOpts = append(agentOpts, ocagent.WithAddress(endpoint)) - } else { - return nil, errdefs.InvalidInput("must set endpoint address in OCAGENT_ENDPOINT") +func initOCAgent(service string) error { + endpoint := os.Getenv("OCAGENT_ENDPOINT") + if endpoint == "" { + return nil + } + options := []ocagent.ExporterOption{ + ocagent.WithAddress(endpoint), + ocagent.WithServiceName(service), } - switch os.Getenv("OCAGENT_INSECURE") { case "0", "no", "n", "off", "": case "1", "yes", "y", "on": - agentOpts = append(agentOpts, ocagent.WithInsecure()) + options = append(options, ocagent.WithInsecure()) default: - return nil, errdefs.InvalidInput("invalid value for OCAGENT_INSECURE") + return errdefs.InvalidInput("invalid value for OCAGENT_INSECURE") + } + + exporter, err := ocagent.NewExporter(options...) + if err != nil { + return err } + + octrace.RegisterExporter(exporter) + return nil +} - return ocagent.NewExporter(agentOpts...) +func configureTracing(service string, rate string) error { + var s octrace.Sampler + switch strings.ToLower(rate) { + case "": + case "always": + s = octrace.AlwaysSample() + case "never": + s = octrace.NeverSample() + default: + rate, err := strconv.Atoi(rate) + if err != nil { + return errdefs.AsInvalidInput(fmt.Errorf("unsupported trace sample rate: %w", err)) + } + if rate < 0 || rate > 100 { + return errdefs.AsInvalidInput(fmt.Errorf("trace sample rate must be between 0 and 100: %w", err)) + } + s = octrace.ProbabilitySampler(float64(rate) / 100) + } + + if err := initOCAgent(service); err != nil { + return err + } + trace.T = opencensus.Adapter{} + octrace.ApplyConfig(octrace.Config{ + DefaultSampler: s, + }) + return nil } diff --git a/cmd/virtual-kubelet/signal_unix.go b/cmd/virtual-kubelet/signal_unix.go new file mode 100644 index 00000000..6673a627 --- /dev/null +++ b/cmd/virtual-kubelet/signal_unix.go @@ -0,0 +1,13 @@ +//go:build !windows +// +build !windows + +package main + +import ( + "os" + "syscall" +) + +func cancelSigs() []os.Signal { + return []os.Signal{os.Interrupt, syscall.SIGTERM} +} diff --git a/cmd/virtual-kubelet/signal_windows.go b/cmd/virtual-kubelet/signal_windows.go new file mode 100644 index 00000000..a4600322 --- /dev/null +++ b/cmd/virtual-kubelet/signal_windows.go @@ -0,0 +1,7 @@ +package main + +import "os" + +func cancelSigs() []os.Signal { + return []os.Signal{os.Interrupt} +} diff --git a/go.mod b/go.mod index a27c833d..5e5ebac6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/virtual-kubelet/azure-aci go 1.16 require ( - contrib.go.opencensus.io/exporter/ocagent v0.4.12 + contrib.go.opencensus.io/exporter/ocagent v0.7.0 github.com/Azure/azure-sdk-for-go v43.0.0+incompatible github.com/Azure/go-autorest/autorest v0.11.0 github.com/Azure/go-autorest/autorest/adal v0.9.0 @@ -12,26 +12,30 @@ require ( github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/BurntSushi/toml v0.3.1 + github.com/cpuguy83/dockercfg v0.3.0 github.com/dimchansky/utfbom v1.1.0 github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/mock v1.6.0 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 + github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.16.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 + github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.5 // indirect github.com/thoas/go-funk v0.9.1 - github.com/virtual-kubelet/node-cli v0.7.0 github.com/virtual-kubelet/virtual-kubelet v1.6.0 - go.opencensus.io v0.22.2 + go.opencensus.io v0.22.3 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gotest.tools v2.2.0+incompatible k8s.io/api v0.19.10 k8s.io/apimachinery v0.19.10 k8s.io/client-go v0.19.10 + k8s.io/klog v1.0.0 ) // fix cve diff --git a/go.sum b/go.sum index 76b5d74a..d4687082 100644 --- a/go.sum +++ b/go.sum @@ -5,14 +5,26 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= -contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= +contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v43.0.0+incompatible h1:/wSNCu0e6EsHFR4Qa3vBEBbicaprEHMyyga9g8RTULI= github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -70,6 +82,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -94,6 +107,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -104,6 +118,8 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/dockercfg v0.3.0 h1:rsyeW1hvsI0JMAZpQGB3lzb47MSq32Gz/pfBoV5TBxM= +github.com/cpuguy83/dockercfg v0.3.0/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -129,7 +145,9 @@ github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:/Zj4wYkg github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= @@ -142,7 +160,9 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -211,22 +231,25 @@ github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkS github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -254,6 +277,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -277,8 +302,9 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.14.6 h1:8ERzHx8aj1Sc47mu9n/AksaKCSWrMchFtkdrS4BIj5o= +github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -405,6 +431,7 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -447,12 +474,11 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/virtual-kubelet/node-cli v0.7.0 h1:a04KMO8dnf8TnAaOkaCWwnaSLYzTLMNmuClhY04fxa4= -github.com/virtual-kubelet/node-cli v0.7.0/go.mod h1:kT/vPiu2h9peA/wbaTfbH3F2Eo/Rv3xXe0ZO+s0gXFY= github.com/virtual-kubelet/virtual-kubelet v1.6.0 h1:BJAsHB3ItrXVhzQszWiIqmwVkuxrtu4f0Xz5hjMPR7c= github.com/virtual-kubelet/virtual-kubelet v1.6.0/go.mod h1:Uf0+/KqEDz5Tz6TNQSok5B4A+guFZV6cs4JohQTHdEQ= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -467,8 +493,9 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -494,7 +521,12 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -505,11 +537,14 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -531,12 +566,18 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -546,14 +587,16 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -573,18 +616,26 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -596,6 +647,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -634,9 +686,21 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -654,9 +718,15 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.1 h1:5mMS6mYvK5LVB8+ujVBC33Y8gltBo/kT6HBm6kU80G4= google.golang.org/api v0.15.1/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.25.0 h1:LodzhlzZEUfhXzNUMIfVlf9Gr6Ua5MMtoFWh7+f47qA= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -672,9 +742,20 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece h1:1YM0uhfumvoDu9sx8+RyWwTI63zoCQvI23IYFRlvte0= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -682,9 +763,13 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -715,6 +800,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -732,6 +818,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.10 h1:AGQnvGl693Kxib2Byxc7lTcNm1/KdHMht9xDb5GcOmY= k8s.io/api v0.19.10/go.mod h1:FSHnCfzQ9/b9qSSF1A1+QWk5cdO1auExjLcc/RJ/zB0= @@ -763,6 +850,8 @@ k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= diff --git a/provider/aci.go b/provider/aci.go index b253654b..e007779d 100644 --- a/provider/aci.go +++ b/provider/aci.go @@ -19,16 +19,18 @@ import ( "time" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/labels" + "github.com/cpuguy83/dockercfg" "github.com/gorilla/websocket" client "github.com/virtual-kubelet/azure-aci/client" "github.com/virtual-kubelet/azure-aci/client/aci" "github.com/virtual-kubelet/azure-aci/client/network" "github.com/virtual-kubelet/azure-aci/provider/metrics" - "github.com/virtual-kubelet/node-cli/manager" "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/node/api" + "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" "github.com/virtual-kubelet/virtual-kubelet/trace" v1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" @@ -36,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" @@ -78,8 +81,11 @@ const ( // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs. type ACIProvider struct { - aciClient *aci.Client - resourceManager *manager.ResourceManager + aciClient *aci.Client + + secretL corev1listers.SecretLister + configL corev1listers.ConfigMapLister + podsL corev1listers.PodLister resourceGroup string region string nodeName string @@ -167,11 +173,14 @@ func isValidACIRegion(region string) bool { } // NewACIProvider creates a new ACIProvider. -func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) { +func NewACIProvider(config string, pCfg nodeutil.ProviderConfig, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) { var p ACIProvider var err error - p.resourceManager = rm + p.configL = pCfg.ConfigMaps + p.secretL = pCfg.Secrets + p.podsL = pCfg.Pods + p.clusterDomain = clusterDomain if config != "" { @@ -321,7 +330,7 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat p.diagnostics.LogAnalytics.LogType = aci.LogAnlyticsLogTypeContainerInsights p.diagnostics.LogAnalytics.Metadata = map[string]string{ aci.LogAnalyticsMetadataKeyClusterResourceID: clusterResourceID, - aci.LogAnalyticsMetadataKeyNodeName: nodeName, + aci.LogAnalyticsMetadataKeyNodeName: pCfg.Node.Name, } } } @@ -350,8 +359,8 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat return nil, err } - p.operatingSystem = operatingSystem - p.nodeName = nodeName + p.operatingSystem = strings.Title(pCfg.Node.Status.NodeInfo.OperatingSystem) + p.nodeName = pCfg.Node.Name p.internalIP = internalIP p.daemonEndpointPort = daemonEndpointPort @@ -411,7 +420,7 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat } } - p.ACIPodMetricsProvider = *metrics.NewACIPodMetricsProvider(nodeName, p.resourceGroup, p.resourceManager, p.aciClient, p.aciClient) + p.ACIPodMetricsProvider = *metrics.NewACIPodMetricsProvider(p.nodeName, p.resourceGroup, p.podsL, p.aciClient, p.aciClient) return &p, err } @@ -1166,7 +1175,7 @@ func (p *ACIProvider) NotifyPods(ctx context.Context, notifierCb func(*v1.Pod)) // Capture the notifier to be used for communicating updates to VK p.tracker = &PodsTracker{ - rm: p.resourceManager, + pods: p.podsL, updateCb: notifierCb, handler: p, } @@ -1314,7 +1323,7 @@ func (p *ACIProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints { func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]aci.ImageRegistryCredential, error) { ips := make([]aci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) for _, ref := range pod.Spec.ImagePullSecrets { - secret, err := p.resourceManager.GetSecret(ref.Name, pod.Namespace) + secret, err := p.secretL.Secrets(pod.Namespace).Get(ref.Name) if err != nil { return ips, err } @@ -1370,15 +1379,25 @@ func makeRegistryCredential(server string, authConfig AuthConfig) (*aci.ImageReg return &cred, nil } -func makeRegistryCredentialFromDockerConfig(server string, configEntry DockerConfigEntry) (*aci.ImageRegistryCredential, error) { +func makeRegistryCredentialFromDockerConfig(server string, configEntry dockercfg.AuthConfig) (*aci.ImageRegistryCredential, error) { if configEntry.Username == "" { return nil, fmt.Errorf("no username present in auth config for server: %s", server) } + u := configEntry.Username + p := configEntry.Password + if configEntry.Auth != "" { + var err error + u, p, err = dockercfg.DecodeBase64Auth(configEntry) + if err != nil { + return nil, fmt.Errorf("error decoding docker auth: %w", err) + } + } + cred := aci.ImageRegistryCredential{ Server: server, - Username: configEntry.Username, - Password: configEntry.Password, + Username: u, + Password: p, } return &cred, nil @@ -1419,15 +1438,15 @@ func readDockerConfigJSONSecret(secret *v1.Secret, ips []aci.ImageRegistryCreden } // Will use K8s config models to handle marshaling (including auth field handling). - var cfgJson DockerConfigJSON + var cfgJson dockercfg.Config err = json.Unmarshal(repoData, &cfgJson) if err != nil { return ips, err } - auths := cfgJson.Auths - if len(cfgJson.Auths) == 0 { + auths := cfgJson.AuthConfigs + if len(auths) == 0 { return ips, fmt.Errorf("malformed dockerconfigjson in secret") } @@ -1664,10 +1683,14 @@ func (p *ACIProvider) getAzureFileCSI(volume v1.Volume, namespace string) (*aci. return nil, fmt.Errorf("secret name for AzureFile CSI driver %s cannot be empty or nil", volume.Name) } - secret, err := p.resourceManager.GetSecret(secretName, namespace) + secret, err := p.secretL.Secrets(namespace).Get(secretName) + + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("the secret %s for AzureFile CSI driver %s is not found", secretName, volume.Name)) + } - if err != nil || secret == nil { - return nil, fmt.Errorf("the secret %s for AzureFile CSI driver %s is not found", secretName, volume.Name) + if secret == nil { + return nil, fmt.Errorf("getting secret for AzureFile CSI driver %s returned an empty secret", volume.Name) } return &aci.Volume{ @@ -1699,7 +1722,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { // Handle the case for the AzureFile volume. if v.AzureFile != nil { - secret, err := p.resourceManager.GetSecret(v.AzureFile.SecretName, pod.Namespace) + secret, err := p.secretL.Secrets(pod.Namespace).Get(v.AzureFile.SecretName) if err != nil { return volumes, err } @@ -1745,7 +1768,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { // Handle the case for Secret volume. if v.Secret != nil { paths := make(map[string]string) - secret, err := p.resourceManager.GetSecret(v.Secret.SecretName, pod.Namespace) + secret, err := p.secretL.Secrets(pod.Namespace).Get(v.Secret.SecretName) if v.Secret.Optional != nil && !*v.Secret.Optional && k8serr.IsNotFound(err) { return nil, fmt.Errorf("Secret %s is required by Pod %s and does not exist", v.Secret.SecretName, pod.Name) } @@ -1769,7 +1792,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { // Handle the case for ConfigMap volume. if v.ConfigMap != nil { paths := make(map[string]string) - configMap, err := p.resourceManager.GetConfigMap(v.ConfigMap.Name, pod.Namespace) + configMap, err := p.configL.ConfigMaps(pod.Namespace).Get(v.ConfigMap.Name) if v.ConfigMap.Optional != nil && !*v.ConfigMap.Optional && k8serr.IsNotFound(err) { return nil, fmt.Errorf("ConfigMap %s is required by Pod %s and does not exist", v.ConfigMap.Name, pod.Name) } @@ -1801,7 +1824,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { switch { case source.ServiceAccountToken != nil: // This is still stored in a secret, hence the dance to figure out what secret. - secrets, err := p.resourceManager.GetSecrets(pod.Namespace) + secrets, err := p.secretL.Secrets(pod.Namespace).List(labels.Everything()) if err != nil { return nil, err } @@ -1831,7 +1854,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { } case source.Secret != nil: - secret, err := p.resourceManager.GetSecret(source.Secret.Name, pod.Namespace) + secret, err := p.secretL.Secrets(pod.Namespace).Get(source.Secret.Name) if source.Secret.Optional != nil && !*source.Secret.Optional && k8serr.IsNotFound(err) { return nil, fmt.Errorf("projected secret %s is required by pod %s and does not exist", source.Secret.Name, pod.Name) } @@ -1858,7 +1881,7 @@ func (p *ACIProvider) getVolumes(pod *v1.Pod) ([]aci.Volume, error) { } case source.ConfigMap != nil: - configMap, err := p.resourceManager.GetConfigMap(source.ConfigMap.Name, pod.Namespace) + configMap, err := p.configL.ConfigMaps(pod.Namespace).Get(source.ConfigMap.Name) if source.ConfigMap.Optional != nil && !*source.ConfigMap.Optional && k8serr.IsNotFound(err) { return nil, fmt.Errorf("projected configMap %s is required by pod %s and does not exist", source.ConfigMap.Name, pod.Name) } diff --git a/provider/aci_test.go b/provider/aci_test.go index 71a59058..d92b4996 100644 --- a/provider/aci_test.go +++ b/provider/aci_test.go @@ -15,19 +15,19 @@ import ( "os" "strings" "testing" - + "github.com/google/uuid" azure "github.com/virtual-kubelet/azure-aci/client" "github.com/virtual-kubelet/azure-aci/client/aci" - "github.com/virtual-kubelet/node-cli/manager" + "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" "gotest.tools/assert" - + is "gotest.tools/assert/cmp" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" ) @@ -49,7 +49,7 @@ func TestMakeRegistryCredential(t *testing.T) { username := "user-" + uuid.New().String() password := "pass-" + uuid.New().String() auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) - + tt := []struct { name string authConfig AuthConfig @@ -87,17 +87,17 @@ func TestMakeRegistryCredential(t *testing.T) { "malformed auth for server", }, } - + for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { cred, err := makeRegistryCredential(server, tc.authConfig) - + if tc.shouldFail { assert.Check(t, err != nil, "conversion should fail") assert.Check(t, strings.Contains(err.Error(), tc.failMessage), "failed message is not expected") return } - + assert.Check(t, err, "conversion should not fail") assert.Check(t, cred != nil, "credential should not be nil") assert.Check(t, is.Equal(server, cred.Server), "server doesn't match") @@ -110,14 +110,14 @@ func TestMakeRegistryCredential(t *testing.T) { // Tests create pod without resource spec func TestCreatePodWithoutResourceSpec(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -130,10 +130,10 @@ func TestCreatePodWithoutResourceSpec(t *testing.T) { assert.Check(t, is.Equal(1.0, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") assert.Check(t, is.Equal(1.5, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -147,7 +147,7 @@ func TestCreatePodWithoutResourceSpec(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -156,14 +156,14 @@ func TestCreatePodWithoutResourceSpec(t *testing.T) { // Tests create pod with resource request only func TestCreatePodWithResourceRequestOnly(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -176,10 +176,10 @@ func TestCreatePodWithResourceRequestOnly(t *testing.T) { assert.Check(t, is.Equal(1.98, cg.ContainerGroupProperties.Containers[0].Resources.Requests.CPU), "Request CPU is not expected") assert.Check(t, is.Equal(3.4, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") assert.Check(t, is.Nil(cg.ContainerGroupProperties.Containers[0].Resources.Limits), "Limits should be nil") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -199,7 +199,7 @@ func TestCreatePodWithResourceRequestOnly(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -209,32 +209,32 @@ func TestCreatePodWithResourceRequestOnly(t *testing.T) { func TestCreatePodWithGPU(t *testing.T) { aadServerMocker := NewAADMock() aciServerMocker := NewACIMock() - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() gpuSKU := aci.GPUSKU("sku-" + uuid.New().String()) - + aciServerMocker.OnGetRPManifest = func() (int, interface{}) { manifest := &aci.ResourceProviderManifest{ Metadata: &aci.ResourceProviderMetadata{ GPURegionalSKUs: []*aci.GPURegionalSKU{ - &aci.GPURegionalSKU{ + { Location: fakeRegion, SKUs: []aci.GPUSKU{gpuSKU, aci.K80, aci.P100}, }, }, }, } - + return http.StatusOK, manifest } - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil) + + provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil, nil, nil) if err != nil { t.Fatalf("failed to create the test provider. %s", err.Error()) return } - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -251,10 +251,10 @@ func TestCreatePodWithGPU(t *testing.T) { assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU != nil, "Limits GPU is not expected") assert.Check(t, is.Equal(int32(10), cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU.Count), "Requests GPU Count is not expected") assert.Check(t, is.Equal(gpuSKU, cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU.SKU), "Requests GPU SKU is not expected") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -277,7 +277,7 @@ func TestCreatePodWithGPU(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -287,11 +287,11 @@ func TestCreatePodWithGPU(t *testing.T) { func TestCreatePodWithGPUSKU(t *testing.T) { aadServerMocker := NewAADMock() aciServerMocker := NewACIMock() - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() gpuSKU := aci.GPUSKU("sku-" + uuid.New().String()) - + aciServerMocker.OnGetRPManifest = func() (int, interface{}) { manifest := &aci.ResourceProviderManifest{ Metadata: &aci.ResourceProviderMetadata{ @@ -303,16 +303,16 @@ func TestCreatePodWithGPUSKU(t *testing.T) { }, }, } - + return http.StatusOK, manifest } - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil) + + provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil, nil, nil) if err != nil { t.Fatalf("failed to create the test provider. %s", err.Error()) return } - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -330,10 +330,10 @@ func TestCreatePodWithGPUSKU(t *testing.T) { assert.Check(t, cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU != nil, "Limits GPU is not expected") assert.Check(t, is.Equal(int32(1), cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU.Count), "Requests GPU Count is not expected") assert.Check(t, is.Equal(gpuSKU, cg.ContainerGroupProperties.Containers[0].Resources.Limits.GPU.SKU), "Requests GPU SKU is not expected") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -359,7 +359,7 @@ func TestCreatePodWithGPUSKU(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -368,14 +368,14 @@ func TestCreatePodWithGPUSKU(t *testing.T) { // Tests create pod with both resource request and limit. func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -389,10 +389,10 @@ func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { assert.Check(t, is.Equal(3.4, cg.ContainerGroupProperties.Containers[0].Resources.Requests.MemoryInGB), "Request Memory is not expected") assert.Check(t, is.Equal(3.999, cg.ContainerGroupProperties.Containers[0].Resources.Limits.CPU), "Limit CPU is not expected") assert.Check(t, is.Equal(8.0, cg.ContainerGroupProperties.Containers[0].Resources.Limits.MemoryInGB), "Limit Memory is not expected") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -416,7 +416,7 @@ func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -425,25 +425,25 @@ func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { // Tests get pods with empty list. func TestGetPodsWithEmptyList(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + aciServerMocker.OnGetContainerGroups = func(subscription, resourceGroup string) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - + return http.StatusOK, aci.ContainerGroupListResult{ Value: []aci.ContainerGroup{}, } } - + pods, err := provider.GetPods(context.Background()) if err != nil { t.Fatal("Failed to get pods", err) } - + assert.Check(t, pods != nil, "Response pods should not be nil") assert.Check(t, is.Equal(0, len(pods)), "No pod should be returned") } @@ -451,15 +451,15 @@ func TestGetPodsWithEmptyList(t *testing.T) { // Tests get pods without requests limit. func TestGetPodsWithoutResourceRequestsLimits(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + aciServerMocker.OnGetContainerGroups = func(subscription, resourceGroup string) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") - + var cg = aci.ContainerGroup{ Name: "default-nginx", Tags: map[string]string{ @@ -490,22 +490,22 @@ func TestGetPodsWithoutResourceRequestsLimits(t *testing.T) { }, }, } - + return http.StatusOK, aci.ContainerGroupListResult{ Value: []aci.ContainerGroup{cg}, } } - + pods, err := provider.GetPods(context.Background()) if err != nil { t.Fatal("Failed to get pods", err) } - + assert.Check(t, pods != nil, "Response pods should not be nil") assert.Check(t, is.Equal(1, len(pods)), "No pod should be returned") - + pod := pods[0] - + assert.Check(t, pod != nil, "Response pod should not be nil") assert.Check(t, pod.Spec.Containers != nil, "Containers should not be nil") assert.Check(t, is.Nil(pod.Spec.Containers[0].Resources.Limits), "Containers[0].Resources.Limits should be nil") @@ -519,19 +519,19 @@ func TestGetPodsWithoutResourceRequestsLimits(t *testing.T) { // Tests get pod without requests limit. func TestGetPodWithoutResourceRequestsLimits(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnGetContainerGroup = func(subscription, resourceGroup, containerGroup string) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - + return http.StatusOK, aci.ContainerGroup{ Tags: map[string]string{ "NodeName": fakeNodeName, @@ -562,12 +562,12 @@ func TestGetPodWithoutResourceRequestsLimits(t *testing.T) { }, } } - + pod, err := provider.GetPod(context.Background(), podNamespace, podName) if err != nil { t.Fatal("Failed to get pod", err) } - + assert.Check(t, pod != nil, "Response pod should not be nil") assert.Check(t, pod.Spec.Containers != nil, "Containers should not be nil") assert.Check(t, is.Nil(pod.Spec.Containers[0].Resources.Limits), "Containers[0].Resources.Limits should be nil") @@ -581,19 +581,19 @@ func TestGetPodWithoutResourceRequestsLimits(t *testing.T) { // Tests get pod with GPU. func TestGetPodWithGPU(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnGetContainerGroup = func(subscription, resourceGroup, containerGroup string) (int, interface{}) { assert.Equal(t, fakeSubscription, subscription, "Subscription doesn't match") assert.Equal(t, fakeResourceGroup, resourceGroup, "Resource group doesn't match") assert.Equal(t, podNamespace+"-"+podName, containerGroup, "Container group name is not expected") - + return http.StatusOK, aci.ContainerGroup{ Tags: map[string]string{ "NodeName": fakeNodeName, @@ -634,12 +634,12 @@ func TestGetPodWithGPU(t *testing.T) { }, } } - + pod, err := provider.GetPod(context.Background(), podNamespace, podName) if err != nil { t.Fatal("Failed to get pod", err) } - + assert.Check(t, pod != nil, "Response pod should not be nil") assert.Check(t, pod.Spec.Containers != nil, "Containers should not be nil") assert.Check(t, pod.Spec.Containers[0].Resources.Requests != nil, "Containers[0].Resources.Requests should not be nil") @@ -668,23 +668,23 @@ func TestGetPodWithGPU(t *testing.T) { func TestGetPodWithContainerID(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() containerName := "c-" + uuid.New().String() containerImage := "ci-" + uuid.New().String() - + cgID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerInstance/containerGroups/%s-%s", fakeSubscription, fakeResourceGroup, podNamespace, podName) - + aciServerMocker.OnGetContainerGroup = func(subscription, resourceGroup, containerGroup string) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") assert.Check(t, is.Equal(podNamespace+"-"+podName, containerGroup), "Container group name is not expected") - + return http.StatusOK, aci.ContainerGroup{ ID: cgID, Tags: map[string]string{ @@ -716,12 +716,12 @@ func TestGetPodWithContainerID(t *testing.T) { }, } } - + pod, err := provider.GetPod(context.Background(), podNamespace, podName) if err != nil { t.Fatal("Failed to get pod", err) } - + assert.Check(t, pod != nil, "Response pod should not be nil") assert.Check(t, is.Equal(1, len(pod.Status.ContainerStatuses)), "1 container status is expected") assert.Check(t, is.Equal(containerName, pod.Status.ContainerStatuses[0].Name), "Container name in the container status doesn't match") @@ -730,10 +730,10 @@ func TestGetPodWithContainerID(t *testing.T) { } func TestPodToACISecretEnvVar(t *testing.T) { - + testKey := "testVar" testVal := "testVal" - + e := v1.EnvVar{ Name: testKey, Value: testVal, @@ -742,40 +742,40 @@ func TestPodToACISecretEnvVar(t *testing.T) { }, } aciEnvVar := getACIEnvVar(e) - + if aciEnvVar.Value != "" { t.Fatalf("ACI Env Variable Value should be empty for a secret") } - + if aciEnvVar.Name != testKey { t.Fatalf("ACI Env Variable Name does not match expected Name") } - + if aciEnvVar.SecureValue != testVal { t.Fatalf("ACI Env Variable Secure Value does not match expected value") } } func TestPodToACIEnvVar(t *testing.T) { - + testKey := "testVar" testVal := "testVal" - + e := v1.EnvVar{ Name: testKey, Value: testVal, ValueFrom: &v1.EnvVarSource{}, } aciEnvVar := getACIEnvVar(e) - + if aciEnvVar.SecureValue != "" { t.Fatalf("ACI Env Variable Secure Value should be empty for non-secret variables") } - + if aciEnvVar.Name != testKey { t.Fatalf("ACI Env Variable Name does not match expected Name") } - + if aciEnvVar.Value != testVal { t.Fatalf("ACI Env Variable Value does not match expected value") } @@ -784,7 +784,7 @@ func TestPodToACIEnvVar(t *testing.T) { func prepareMocks() (*AADMock, *ACIMock, *ACIProvider, error) { aadServerMocker := NewAADMock() aciServerMocker := NewACIMock() - + aciServerMocker.OnGetRPManifest = func() (int, interface{}) { manifest := &aci.ResourceProviderManifest{ Metadata: &aci.ResourceProviderMetadata{ @@ -796,19 +796,19 @@ func prepareMocks() (*AADMock, *ACIMock, *ACIProvider, error) { }, }, } - + return http.StatusOK, manifest } - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil) + + provider, err := createTestProvider(aadServerMocker, aciServerMocker, nil, nil, nil) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create the test provider %s", err.Error()) } - + return aadServerMocker, aciServerMocker, provider, nil } -func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock, resourceManager *manager.ResourceManager) (*ACIProvider, error) { +func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock, configMapMocker *MockConfigMapLister, secretMocker *MockSecretLister, podMocker *MockPodLister) (*ACIProvider, error) { auth := azure.NewAuthentication( azure.PublicCloud.Name, fakeClientID, @@ -816,41 +816,43 @@ func createTestProvider(aadServerMocker *AADMock, aciServerMocker *ACIMock, reso fakeSubscription, fakeTenantID, fakeUserIdentity) - + auth.ActiveDirectoryEndpoint = aadServerMocker.GetServerURL() auth.ResourceManagerEndpoint = aciServerMocker.GetServerURL() - + file, err := ioutil.TempFile("", "auth.json") if err != nil { return nil, err } - + defer os.Remove(file.Name()) - + b := new(bytes.Buffer) json.NewEncoder(b).Encode(auth) - + if _, err := file.Write(b.Bytes()); err != nil { return nil, err } - + os.Setenv("AZURE_AUTH_LOCATION", file.Name()) os.Setenv("ACI_RESOURCE_GROUP", fakeResourceGroup) os.Setenv("ACI_REGION", fakeRegion) - - if resourceManager == nil { - resourceManager, err = manager.NewResourceManager(nil, nil, nil, nil, nil, nil) - - if err != nil { - return nil, err - } - } - - provider, err := NewACIProvider("example.toml", resourceManager, fakeNodeName, "Linux", "0.0.0.0", 10250, "cluster.local") + + cfg := nodeutil.ProviderConfig{ + ConfigMaps: configMapMocker, + Secrets: secretMocker, + Pods: podMocker, + } + + cfg.Node = &v1.Node{} + cfg.Node.Name = fakeNodeName + cfg.Node.Status.NodeInfo.OperatingSystem = "Linux" + + provider, err := NewACIProvider("example.toml", cfg, "0.0.0.0", 10250, "cluster.local") if err != nil { return nil, err } - + return provider, nil } @@ -863,7 +865,7 @@ func TestConfigureNode(t *testing.T) { if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "virtual-kubelet", @@ -889,14 +891,14 @@ func TestConfigureNode(t *testing.T) { func TestCreatePodWithNamedLivenessProbe(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, cg.Containers[0].LivenessProbe != nil, "Liveness probe expected") assert.Check(t, is.Equal(10, cg.Containers[0].LivenessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") @@ -938,25 +940,25 @@ func TestCreatePodWithNamedLivenessProbe(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } - + return http.StatusOK, cg } } func TestCreatePodWithLivenessProbe(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -972,10 +974,10 @@ func TestCreatePodWithLivenessProbe(t *testing.T) { assert.Check(t, is.Equal(int32(3), cg.Containers[0].LivenessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") assert.Check(t, is.Equal(int32(5), cg.Containers[0].LivenessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") assert.Check(t, cg.Containers[0].LivenessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -1002,7 +1004,7 @@ func TestCreatePodWithLivenessProbe(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -1010,14 +1012,14 @@ func TestCreatePodWithLivenessProbe(t *testing.T) { func TestCreatePodWithReadinessProbe(t *testing.T) { _, aciServerMocker, provider, err := prepareMocks() - + if err != nil { t.Fatal("Unable to prepare the mocks", err) } - + podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -1033,10 +1035,10 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { assert.Check(t, is.Equal(int32(3), cg.Containers[0].ReadinessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") assert.Check(t, is.Equal(int32(5), cg.Containers[0].ReadinessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") assert.Check(t, cg.Containers[0].ReadinessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -1063,7 +1065,7 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -1072,10 +1074,10 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { func TestCreatePodWithProjectedVolume(t *testing.T) { podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() - + aadServerMocker := NewAADMock() aciServerMocker := NewACIMock() - + aciServerMocker.OnGetRPManifest = func() (int, interface{}) { manifest := &aci.ResourceProviderManifest{ Metadata: &aci.ResourceProviderMetadata{ @@ -1087,23 +1089,12 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { }, }, } - + return http.StatusOK, manifest } - + mockCtrl := gomock.NewController(GinkgoT()) - podLister := NewMockPodLister(mockCtrl) - secretLister := NewMockSecretLister(mockCtrl) configMapLister := NewMockConfigMapLister(mockCtrl) - serviceLister := NewMockServiceLister(mockCtrl) - pvcLister := NewMockPersistentVolumeClaimLister(mockCtrl) - pvLister := NewMockPersistentVolumeLister(mockCtrl) - - resourceManager, err := manager.NewResourceManager(podLister, secretLister, configMapLister, serviceLister, pvcLister, pvLister) - if err != nil { - t.Fatal("Unable to prepare the mocks for resourceManager", err) - } - configMapNamespaceLister := NewMockConfigMapNamespaceLister(mockCtrl) configMapLister.EXPECT().ConfigMaps(podNamespace).Return(configMapNamespaceLister) configMapNamespaceLister.EXPECT().Get("kube-root-ca.crt").Return(&v1.ConfigMap{ @@ -1115,12 +1106,12 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { "foo": "bar", }, }, nil) - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, resourceManager) + + provider, err := createTestProvider(aadServerMocker, aciServerMocker, configMapLister, nil, nil) if err != nil { t.Fatal("Unable to create test provider", err) } - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -1132,10 +1123,10 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { assert.Check(t, is.Equal(1, len(cg.Volumes)), "volume count not match") assert.Check(t, is.Equal("projectedvolume", cg.Volumes[0].Name), "volume name doesn't match") assert.Check(t, is.Equal(base64.StdEncoding.EncodeToString([]byte("fake-ca-data")), cg.Volumes[0].Secret["ca.crt"]), "configmap data doesn't match") - + return http.StatusOK, cg } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -1186,7 +1177,7 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { }, }, } - + if err := provider.CreatePod(context.Background(), pod); err != nil { t.Fatal("Failed to create pod", err) } @@ -1198,18 +1189,14 @@ func TestCreatePodWithCSIVolume(t *testing.T) { fakeVolumeSecret := "fake-volume-secret" fakeShareName := "aksshare" azureFileVolumeName := "azure" - + aadServerMocker := NewAADMock() aciServerMocker := NewACIMock() mockCtrl := gomock.NewController(GinkgoT()) - mockPodLister := NewMockPodLister(mockCtrl) mockSecretLister := NewMockSecretLister(mockCtrl) mockSecretNamespaceLister := NewMockSecretNamespaceLister(mockCtrl) mockConfigMapLister := NewMockConfigMapLister(mockCtrl) - mockServiceLister := NewMockServiceLister(mockCtrl) - mockPvcLister := NewMockPersistentVolumeClaimLister(mockCtrl) - mockPvLister := NewMockPersistentVolumeLister(mockCtrl) - + aciServerMocker.OnGetRPManifest = func() (int, interface{}) { manifest := &aci.ResourceProviderManifest{ Metadata: &aci.ResourceProviderMetadata{ @@ -1223,7 +1210,7 @@ func TestCreatePodWithCSIVolume(t *testing.T) { } return http.StatusOK, manifest } - + aciServerMocker.OnCreate = func(subscription, resourceGroup, containerGroup string, cg *aci.ContainerGroup) (int, interface{}) { assert.Check(t, is.Equal(fakeSubscription, subscription), "Subscription doesn't match") assert.Check(t, is.Equal(fakeResourceGroup, resourceGroup), "Resource group doesn't match") @@ -1233,10 +1220,10 @@ func TestCreatePodWithCSIVolume(t *testing.T) { assert.Check(t, is.Equal(1, len(cg.ContainerGroupProperties.Containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", cg.ContainerGroupProperties.Containers[0].Name), "Container nginx is expected") assert.Check(t, is.Equal(1, len(cg.Volumes)), "volume count not match") - + return http.StatusOK, cg } - + configMapNamespaceLister := NewMockConfigMapNamespaceLister(mockCtrl) mockConfigMapLister.EXPECT().ConfigMaps(podNamespace).Return(configMapNamespaceLister) configMapNamespaceLister.EXPECT().Get("kube-root-ca.crt").Return(&v1.ConfigMap{ @@ -1248,7 +1235,12 @@ func TestCreatePodWithCSIVolume(t *testing.T) { "foo": "bar", }, }, nil) - + + provider, err := createTestProvider(aadServerMocker, aciServerMocker, mockConfigMapLister, mockSecretLister, nil) + if err != nil { + t.Fatal("Unable to create test provider", err) + } + fakeSecret := v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fakeVolumeSecret, @@ -1258,12 +1250,12 @@ func TestCreatePodWithCSIVolume(t *testing.T) { azureFileStorageAccountName: []byte("azure storage account name"), azureFileStorageAccountKey: []byte("azure storage account key")}, } - + fakeVolumeMount := v1.VolumeMount{ Name: azureFileVolumeName, MountPath: "/mnt/azure", } - + fakePodVolume := v1.Volume{ Name: azureFileVolumeName, VolumeSource: v1.VolumeSource{ @@ -1276,7 +1268,7 @@ func TestCreatePodWithCSIVolume(t *testing.T) { }, }, } - + cases := []struct { description string secretVolume *v1.Secret @@ -1287,7 +1279,7 @@ func TestCreatePodWithCSIVolume(t *testing.T) { description: "Secret is nil", secretVolume: nil, volume: fakePodVolume, - expectedError: fmt.Errorf("the secret %s for AzureFile CSI driver %s is not found", fakeSecret.Name, fakePodVolume.Name), + expectedError: fmt.Errorf("getting secret for AzureFile CSI driver %s returned an empty secret", fakePodVolume.Name), }, { description: "Volume has a secret with a valid value", @@ -1342,17 +1334,7 @@ func TestCreatePodWithCSIVolume(t *testing.T) { } for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - - resourceManager, err := manager.NewResourceManager(mockPodLister, mockSecretLister, mockConfigMapLister, mockServiceLister, mockPvcLister, mockPvLister) - if err != nil { - t.Fatal("Unable to prepare the mocks for resourceManager", err) - } - - provider, err := createTestProvider(aadServerMocker, aciServerMocker, resourceManager) - if err != nil { - t.Fatal("Unable to create test provider", err) - } - + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -1379,18 +1361,18 @@ func TestCreatePodWithCSIVolume(t *testing.T) { }, }, } - + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, fakeVolumeMount) - + if tc.volume.CSI.VolumeAttributes != nil && len(tc.volume.CSI.VolumeAttributes) != 0 { mockSecretLister.EXPECT().Secrets(podNamespace).Return(mockSecretNamespaceLister) mockSecretNamespaceLister.EXPECT().Get(tc.volume.CSI.VolumeAttributes[azureFileSecretName]).Return(tc.secretVolume, nil) } - + pod.Spec.Volumes = append(pod.Spec.Volumes, tc.volume) - + err = provider.CreatePod(context.Background(), pod) - + if tc.expectedError != nil { assert.Equal(t, tc.expectedError.Error(), err.Error()) } else { diff --git a/provider/config.go b/provider/config.go index 34dd368c..78b6641f 100644 --- a/provider/config.go +++ b/provider/config.go @@ -4,21 +4,24 @@ import ( "fmt" "io" "net" - "strings" - + "github.com/BurntSushi/toml" - "github.com/virtual-kubelet/node-cli/provider" ) type providerConfig struct { - ResourceGroup string - Region string - OperatingSystem string - CPU string - Memory string - Pods string - SubnetName string - SubnetCIDR string + ResourceGroup string + Region string + OperatingSystem string + CPU string + Memory string + Pods string + SubnetName string + SubnetCIDR string +} + +var validOS = map[string]bool{ + "Linux": true, + "Windows": true, } func (p *ACIProvider) loadConfig(r io.Reader) error { @@ -28,7 +31,7 @@ func (p *ACIProvider) loadConfig(r io.Reader) error { } p.region = config.Region p.resourceGroup = config.ResourceGroup - + // Default to 20 mcpu p.cpu = "20" if config.CPU != "" { @@ -44,18 +47,18 @@ func (p *ACIProvider) loadConfig(r io.Reader) error { if config.Pods != "" { p.pods = config.Pods } - + // Default to Linux if the operating system was not defined in the config. if config.OperatingSystem == "" { - config.OperatingSystem = provider.OperatingSystemLinux + config.OperatingSystem = "Linux" } else { // Validate operating system from config. - ok := provider.ValidOperatingSystems[config.OperatingSystem] + ok := validOS[config.OperatingSystem] if !ok { - return fmt.Errorf("%q is not a valid operating system, try one of the following instead: %s", config.OperatingSystem, strings.Join(provider.ValidOperatingSystems.Names(), " | ")) + return fmt.Errorf("%q is not a valid operating system", config.OperatingSystem) } } - + // default subnet name if config.SubnetName != "" { p.subnetName = config.SubnetName @@ -68,7 +71,7 @@ func (p *ACIProvider) loadConfig(r io.Reader) error { return fmt.Errorf("error parsing provided subnet CIDR: %v", err) } } - + p.operatingSystem = config.OperatingSystem return nil } diff --git a/provider/metrics/metrics.go b/provider/metrics/metrics.go index d76a6581..c99a9805 100644 --- a/provider/metrics/metrics.go +++ b/provider/metrics/metrics.go @@ -6,7 +6,7 @@ import ( "net/http" "sync" "time" - + "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/client/aci" @@ -15,6 +15,8 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/trace" "golang.org/x/sync/errgroup" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + corev1listers "k8s.io/client-go/listers/core/v1" ) const ( @@ -22,7 +24,7 @@ const ( ) // package dependency: query the Pods in current virtual nodes. it usually is ResourceManager -type PodGetter interface { +type PodLister interface { GetPods() []*v1.Pod } @@ -47,20 +49,20 @@ type podStatsGetter interface { type ACIPodMetricsProvider struct { nodeName string metricsSync sync.Mutex - podGetter PodGetter + podLister corev1listers.PodLister aciCGGetter ContainerGroupGetter aciCGMetricsGetter MetricsGetter podStatsGetter podStatsGetter } -func NewACIPodMetricsProvider(nodeName, aciResourcegroup string, podGetter PodGetter, aciCGGetter ContainerGroupGetter, aciCGMetricsGetter MetricsGetter) *ACIPodMetricsProvider { +func NewACIPodMetricsProvider(nodeName, aciResourcegroup string, podLister corev1listers.PodLister, aciCGGetter ContainerGroupGetter, aciCGMetricsGetter MetricsGetter) *ACIPodMetricsProvider { provider := ACIPodMetricsProvider{ nodeName: nodeName, - podGetter: podGetter, + podLister: podLister, aciCGGetter: aciCGGetter, aciCGMetricsGetter: aciCGMetricsGetter, } - + containerInsightGetter := WrapCachedPodStatsGetter( 30, NewContainerInsightsMetricsProvider(aciCGMetricsGetter, aciResourcegroup)) @@ -75,23 +77,26 @@ func NewACIPodMetricsProvider(nodeName, aciResourcegroup string, podGetter PodGe func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *stats.Summary, err error) { ctx, span := trace.StartSpan(ctx, "GetSummaryStats") defer span.End() - + p.metricsSync.Lock() defer p.metricsSync.Unlock() - + log.G(ctx).Debug("acquired metrics mutex") - + select { case <-ctx.Done(): return nil, ctx.Err() default: } - - pods := p.podGetter.GetPods() - + + pods, err := p.podLister.List(labels.Everything()) + if err != nil { + log.L.WithError(err).Errorf("failed to retrieve pods list") + } + var errGroup errgroup.Group chResult := make(chan stats.PodStats, len(pods)) - + sema := make(chan struct{}, 10) for _, pod := range pods { if pod.Status.Phase != v1.PodRunning { @@ -106,7 +111,7 @@ func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *s "Name": pod.Name, "Namespace": pod.Namespace, }) - + select { case <-ctx.Done(): return ctx.Err() @@ -115,37 +120,37 @@ func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *s defer func() { <-sema }() - + logger.Debug("Acquired semaphore") - + podMetrics, err := p.podStatsGetter.getPodStats(ctx, pod) if err != nil { span.SetStatus(err) return errors.Wrapf(err, "error fetching metrics for pods '%s'", pod.Name) } - + chResult <- *podMetrics return nil }) } - + if err := errGroup.Wait(); err != nil { span.SetStatus(err) return nil, errors.Wrap(err, "error in request to fetch container group metrics") } close(chResult) log.G(ctx).Debugf("Collected status from azure for %d pods", len(pods)) - + var s stats.Summary s.Node = stats.NodeStats{ NodeName: p.nodeName, } s.Pods = make([]stats.PodStats, 0, len(chResult)) - + for stat := range chResult { s.Pods = append(s.Pods, stat) } - + return &s, nil } diff --git a/provider/metrics/metrics_test.go b/provider/metrics/metrics_test.go index 3a53102d..98526c15 100644 --- a/provider/metrics/metrics_test.go +++ b/provider/metrics/metrics_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" "time" - + gomock "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/virtual-kubelet/azure-aci/client/aci" @@ -29,7 +29,7 @@ func (m podNameMatcher) String() string { return fmt.Sprintf("pod name is equal to %v (%T)", m.podName, m.podName) } func podNameEq(podName string) gomock.Matcher { - + return podNameMatcher{ podName: podName, } @@ -42,17 +42,17 @@ func TestGetStatsSummary(t *testing.T) { "pod2": uint64(2000), }, } - + for testName, test := range testCases { t.Run(testName, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockedPodGetter := NewMockPodGetter(ctrl) + + podLister := NewMockPodLister(ctrl) mockedPodStatsGetter := NewMockpodStatsGetter(ctrl) - podMetricsProvider := NewACIPodMetricsProvider("node-1", "rg", mockedPodGetter, nil, nil) + podMetricsProvider := NewACIPodMetricsProvider("node-1", "rg", podLister, nil, nil) podMetricsProvider.podStatsGetter = mockedPodStatsGetter - mockedPodGetter.EXPECT().GetPods().Return(fakePod(getMapKeys(test))).AnyTimes() + podLister.EXPECT().List(gomock.Any()).Return(fakePod(getMapKeys(test)), nil) for podName, cpu := range test { mockedPodStatsGetter.EXPECT().getPodStats(gomock.Any(), podNameEq(podName)).Return(fakePodStatus(podName, cpu), nil) } @@ -70,10 +70,10 @@ func TestPodStatsGetterDecider(t *testing.T) { t.Run("useContainerInsightAndContainerGroupCacheNotTakeEffective", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - + mockedAciCgGetter := NewMockContainerGroupGetter(ctrl) httpStatus := 200 - + // Times(2) because we expect the Container Group not be cached mockedAciCgGetter.EXPECT().GetContainerGroup( gomock.Any(), gomock.Any(), gomock.Any()).Return(&aci.ContainerGroup{ @@ -88,29 +88,29 @@ func TestPodStatsGetterDecider(t *testing.T) { }, }, }, &httpStatus, nil).Times(2) - + mockedContainerInsights := NewMockpodStatsGetter(ctrl) mockedContainerInsights.EXPECT().getPodStats(gomock.Any(), gomock.Any()).Return(fakePodStatus("pod-1", 0), nil).Times(2) - + decider := NewPodStatsGetterDecider(mockedContainerInsights, nil, "rg", mockedAciCgGetter) ctx := context.Background() pod1 := fakePod([]string{"pod-1"})[0] pod2 := fakePod([]string{"pod-1"})[0] decider.getPodStats(ctx, pod1) - + /* this time use a pod with new UID but same name. we expect the Container Group will not be cached */ decider.getPodStats(ctx, pod2) }) - + t.Run("useRealtimeMetricsAndContainerGroupCacheTakeEffective", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - + mockedAciCgGetter := NewMockContainerGroupGetter(ctrl) httpStatus := 200 - + // Times(1) here because we expect the Container Group be cached mockedAciCgGetter.EXPECT().GetContainerGroup( gomock.Any(), gomock.Any(), gomock.Any()).Return(&aci.ContainerGroup{ @@ -131,11 +131,11 @@ func TestPodStatsGetterDecider(t *testing.T) { }, }, }, &httpStatus, nil).Times(1) - + mockedRealtime := NewMockpodStatsGetter(ctrl) mockedRealtime.EXPECT().getPodStats(gomock.Any(), gomock.Any()).Return(fakePodStatus("pod-1", 0), nil).Times(1) mockedRealtime.EXPECT().getPodStats(gomock.Any(), gomock.Any()).Return(fakePodStatus("pod-1", 0), nil).Times(1) - + decider := NewPodStatsGetterDecider(nil, mockedRealtime, "rg", mockedAciCgGetter) ctx := context.Background() pod := fakePod([]string{"pod-1"})[0] diff --git a/provider/metrics/mock_metrics_test.go b/provider/metrics/mock_metrics_test.go index 0ac9070d..424f650e 100644 --- a/provider/metrics/mock_metrics_test.go +++ b/provider/metrics/mock_metrics_test.go @@ -12,43 +12,48 @@ import ( aci "github.com/virtual-kubelet/azure-aci/client/aci" statsv1alpha1 "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + corev1listers "k8s.io/client-go/listers/core/v1" ) -// MockPodGetter is a mock of PodGetter interface. -type MockPodGetter struct { +// MockPodLister is a mock of PodLister interface. +type MockPodLister struct { ctrl *gomock.Controller - recorder *MockPodGetterMockRecorder + recorder *MockPodListerMockRecorder } -// MockPodGetterMockRecorder is the mock recorder for MockPodGetter. -type MockPodGetterMockRecorder struct { - mock *MockPodGetter +func (m *MockPodLister) List(selector labels.Selector) (ret []*v1.Pod, err error) { + m.ctrl.T.Helper() + mList := m.ctrl.Call(m, "List", selector) + mList0 := mList[0].([]*v1.Pod) + mList1, _ := mList[1].(error) + return mList0, mList1 } -// NewMockPodGetter creates a new mock instance. -func NewMockPodGetter(ctrl *gomock.Controller) *MockPodGetter { - mock := &MockPodGetter{ctrl: ctrl} - mock.recorder = &MockPodGetterMockRecorder{mock} - return mock +func (m *MockPodLister) Pods(namespace string) corev1listers.PodNamespaceLister { +return nil } -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPodGetter) EXPECT() *MockPodGetterMockRecorder { - return m.recorder +func (m *MockPodListerMockRecorder) List(selector interface{}) *gomock.Call { + m.mock.ctrl.T.Helper() + return m.mock.ctrl.RecordCallWithMethodType(m.mock, "List", reflect.TypeOf((*MockPodLister)(nil).List), selector) } -// GetPods mocks base method. -func (m *MockPodGetter) GetPods() []*v1.Pod { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPods") - ret0, _ := ret[0].([]*v1.Pod) - return ret0 +// MockPodListerMockRecorder is the mock recorder for MockPodLister. +type MockPodListerMockRecorder struct { + mock *MockPodLister } -// GetPods indicates an expected call of GetPods. -func (mr *MockPodGetterMockRecorder) GetPods() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPods", reflect.TypeOf((*MockPodGetter)(nil).GetPods)) +// NewMockPodGetter creates a new mock instance. +func NewMockPodLister(ctrl *gomock.Controller) *MockPodLister { + mock := &MockPodLister{ctrl: ctrl} + mock.recorder = &MockPodListerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPodLister) EXPECT() *MockPodListerMockRecorder { + return m.recorder } // MockMetricsGetter is a mock of MetricsGetter interface. diff --git a/provider/mocks_k8s_listers_test.go b/provider/mocks_k8s_listers_test.go index d36c9ce5..8c622208 100644 --- a/provider/mocks_k8s_listers_test.go +++ b/provider/mocks_k8s_listers_test.go @@ -89,20 +89,6 @@ func (m *MockConfigMapLister) EXPECT() *MockConfigMapListerMockRecorder { return m.recorder } -// ConfigMaps mocks base method. -func (m *MockConfigMapLister) ConfigMaps(arg0 string) v10.ConfigMapNamespaceLister { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ConfigMaps", arg0) - ret0, _ := ret[0].(v10.ConfigMapNamespaceLister) - return ret0 -} - -// ConfigMaps indicates an expected call of ConfigMaps. -func (mr *MockConfigMapListerMockRecorder) ConfigMaps(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigMaps", reflect.TypeOf((*MockConfigMapLister)(nil).ConfigMaps), arg0) -} - // List mocks base method. func (m *MockConfigMapLister) List(arg0 labels.Selector) ([]*v1.ConfigMap, error) { m.ctrl.T.Helper() @@ -118,6 +104,20 @@ func (mr *MockConfigMapListerMockRecorder) List(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockConfigMapLister)(nil).List), arg0) } +// ConfigMaps mocks base method. +func (m *MockConfigMapLister) ConfigMaps(arg0 string) v10.ConfigMapNamespaceLister { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigMaps", arg0) + ret0, _ := ret[0].(v10.ConfigMapNamespaceLister) + return ret0 +} + +// ConfigMaps indicates an expected call of ConfigMaps. +func (mr *MockConfigMapListerMockRecorder) ConfigMaps(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigMaps", reflect.TypeOf((*MockConfigMapLister)(nil).ConfigMaps), arg0) +} + // MockConfigMapNamespaceLister is a mock of ConfigMapNamespaceLister interface. type MockConfigMapNamespaceLister struct { ctrl *gomock.Controller diff --git a/provider/podsTracker.go b/provider/podsTracker.go index 5af5dc00..f93315b7 100644 --- a/provider/podsTracker.go +++ b/provider/podsTracker.go @@ -3,13 +3,14 @@ package provider import ( "context" "time" - - "github.com/virtual-kubelet/node-cli/manager" + errdef "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/trace" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corev1listers "k8s.io/client-go/listers/core/v1" ) const ( @@ -17,7 +18,7 @@ const ( statusReasonNotFound = "NotFound" statusMessageNotFound = "The pod may have been deleted from the provider" containerExitCodeNotFound int32 = -137 - + statusUpdatesInterval = 5 * time.Second cleanupInterval = 5 * time.Minute ) @@ -34,7 +35,7 @@ type PodsTrackerHandler interface { } type PodsTracker struct { - rm *manager.ResourceManager + pods corev1listers.PodLister updateCb func(*v1.Pod) handler PodsTrackerHandler } @@ -43,15 +44,15 @@ type PodsTracker struct { func (pt *PodsTracker) StartTracking(ctx context.Context) { ctx, span := trace.StartSpan(ctx, "PodsTracker.StartTracking") defer span.End() - + statusUpdatesTimer := time.NewTimer(statusUpdatesInterval) cleanupTimer := time.NewTimer(cleanupInterval) defer statusUpdatesTimer.Stop() defer cleanupTimer.Stop() - + for { log.G(ctx).Debug("Pod status updates & cleanup loop start") - + select { case <-ctx.Done(): log.G(ctx).WithError(ctx.Err()).Debug("Pod status update loop exiting") @@ -68,18 +69,21 @@ func (pt *PodsTracker) StartTracking(ctx context.Context) { // UpdatePodStatus updates the status of a pod, by posting to update callback. func (pt *PodsTracker) UpdatePodStatus(ns, name string, updateHandler func(*v1.PodStatus), forceUpdate bool) error { - k8sPods := pt.rm.GetPods() + k8sPods, err := pt.pods.List(labels.Everything()) + if err != nil { + log.L.WithError(err).Errorf("failed to retrieve pods list") + } pod := getPodFromList(k8sPods, ns, name) - + if pod == nil { return errdef.NotFound("pod not found") } - + updatedPod := pod.DeepCopy() if forceUpdate { updatedPod.ResourceVersion = "" } - + updateHandler(&updatedPod.Status) pt.updateCb(updatedPod) return nil @@ -88,8 +92,11 @@ func (pt *PodsTracker) UpdatePodStatus(ns, name string, updateHandler func(*v1.P func (pt *PodsTracker) updatePodsLoop(ctx context.Context) { ctx, span := trace.StartSpan(ctx, "PodsTracker.updatePods") defer span.End() - - k8sPods := pt.rm.GetPods() + + k8sPods, err := pt.pods.List(labels.Everything()) + if err != nil { + log.L.WithError(err).Errorf("failed to retrieve pods list") + } for _, pod := range k8sPods { updatedPod := pod.DeepCopy() ok := pt.processPodUpdates(ctx, updatedPod) @@ -102,23 +109,26 @@ func (pt *PodsTracker) updatePodsLoop(ctx context.Context) { func (pt *PodsTracker) cleanupDanglingPods(ctx context.Context) { ctx, span := trace.StartSpan(ctx, "PodsTracker.cleanupDanglingPods") defer span.End() - - k8sPods := pt.rm.GetPods() + + k8sPods, err := pt.pods.List(labels.Everything()) + if err != nil { + log.L.WithError(err).Errorf("failed to retrieve pods list") + } activePods, err := pt.handler.ListActivePods(ctx) if err != nil { log.G(ctx).WithError(err).Errorf("failed to retrieve active container groups list") return } - + if len(activePods) > 0 { for i := range activePods { pod := getPodFromList(k8sPods, activePods[i].namespace, activePods[i].name) if pod != nil { continue } - + log.G(ctx).Errorf("cleaning up dangling pod %v", activePods[i].name) - + err := pt.handler.CleanupPod(ctx, activePods[i].namespace, activePods[i].name) if err != nil && !errdef.IsNotFound(err) { log.G(ctx).WithError(err).Errorf("failed to cleanup pod %v", activePods[i].name) @@ -130,17 +140,17 @@ func (pt *PodsTracker) cleanupDanglingPods(ctx context.Context) { func (pt *PodsTracker) processPodUpdates(ctx context.Context, pod *v1.Pod) bool { ctx, span := trace.StartSpan(ctx, "PodsTracker.processPodUpdates") defer span.End() - + if pt.shouldSkipPodStatusUpdate(pod) { return false } - + podStatusFromProvider, err := pt.handler.FetchPodStatus(ctx, pod.Namespace, pod.Name) if err == nil && podStatusFromProvider != nil { podStatusFromProvider.DeepCopyInto(&pod.Status) return true } - + if errdef.IsNotFound(err) || (err == nil && podStatusFromProvider == nil) { // Only change the status when the pod was already up if pod.Status.Phase == v1.PodRunning { @@ -153,7 +163,7 @@ func (pt *PodsTracker) processPodUpdates(ctx context.Context, pod *v1.Pod) bool if pod.Status.ContainerStatuses[i].State.Running == nil { continue } - + pod.Status.ContainerStatuses[i].State.Terminated = &v1.ContainerStateTerminated{ ExitCode: containerExitCodeNotFound, Reason: statusReasonNotFound, @@ -164,17 +174,17 @@ func (pt *PodsTracker) processPodUpdates(ctx context.Context, pod *v1.Pod) bool } pod.Status.ContainerStatuses[i].State.Running = nil } - + return true } - + return false } - + if err != nil { log.G(ctx).WithError(err).Errorf("failed to retrieve pod %v status from provider", pod.Name) } - + return false } @@ -191,6 +201,6 @@ func getPodFromList(list []*v1.Pod, ns, name string) *v1.Pod { return pod } } - + return nil }