diff --git a/Gopkg.lock b/Gopkg.lock index c8127ee1d3..7e24d1e1ef 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -73,14 +73,6 @@ revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" version = "v0.5" -[[projects]] - branch = "master" - digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" - name = "github.com/golang/glog" - packages = ["."] - pruneopts = "UT" - revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" - [[projects]] digest = "1:7672c206322f45b33fac1ae2cb899263533ce0adcc6481d207725560208ec84e" name = "github.com/golang/groupcache" @@ -1031,7 +1023,6 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ - "github.com/golang/glog", "github.com/hashicorp/go-multierror", "github.com/onsi/ginkgo", "github.com/onsi/gomega", @@ -1048,6 +1039,7 @@ "k8s.io/api/apps/v1", "k8s.io/api/batch/v1", "k8s.io/api/core/v1", + "k8s.io/api/networking/v1", "k8s.io/api/policy/v1beta1", "k8s.io/api/scheduling/v1beta1", "k8s.io/apimachinery/pkg/api/errors", @@ -1100,6 +1092,7 @@ "k8s.io/code-generator/cmd/defaulter-gen", "k8s.io/code-generator/cmd/informer-gen", "k8s.io/code-generator/cmd/lister-gen", + "k8s.io/klog", "k8s.io/kubernetes/pkg/api/v1/pod", "k8s.io/kubernetes/pkg/apis/core", "k8s.io/kubernetes/pkg/apis/core/v1", diff --git a/Gopkg.toml b/Gopkg.toml index 6d42725fd1..82100b6136 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,10 +32,6 @@ required = [ "k8s.io/code-generator/cmd/defaulter-gen", ] -[[constraint]] - branch = "master" - name = "github.com/golang/glog" - [[constraint]] name = "github.com/onsi/ginkgo" version = "1.7.0" diff --git a/Makefile b/Makefile index ca3461ded1..e9b4a45d5d 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ generate-code: ./hack/update-gencode.sh unit-test: + go clean -testcache go list ./... | grep -v e2e | xargs go test -v -race e2e-test-kind: @@ -84,7 +85,7 @@ clean: rm -rf _output/ rm -f *.log -verify: generate-code +verify: hack/verify-gofmt.sh hack/verify-golint.sh hack/verify-gencode.sh diff --git a/cmd/admission/app/options/options.go b/cmd/admission/app/options/options.go index 13227fbee9..98dd1bd8f0 100644 --- a/cmd/admission/app/options/options.go +++ b/cmd/admission/app/options/options.go @@ -17,16 +17,16 @@ limitations under the License. package options import ( - "flag" "fmt" - "github.com/golang/glog" + "github.com/spf13/pflag" "k8s.io/api/admissionregistration/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" admissionregistrationv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" + "k8s.io/klog" ) const ( @@ -58,27 +58,27 @@ func NewConfig() *Config { } // AddFlags add flags -func (c *Config) AddFlags() { - flag.StringVar(&c.Master, "master", c.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - flag.StringVar(&c.Kubeconfig, "kubeconfig", c.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ +func (c *Config) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&c.Master, "master", c.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + fs.StringVar(&c.Kubeconfig, "kubeconfig", c.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") + fs.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ "after server cert).") - flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") - flag.StringVar(&c.CaCertFile, "ca-cert-file", c.CaCertFile, "File containing the x509 Certificate for HTTPS.") - flag.IntVar(&c.Port, "port", 443, "the port used by admission-controller-server.") - flag.StringVar(&c.MutateWebhookConfigName, "mutate-webhook-config-name", "", + fs.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") + fs.StringVar(&c.CaCertFile, "ca-cert-file", c.CaCertFile, "File containing the x509 Certificate for HTTPS.") + fs.IntVar(&c.Port, "port", 443, "the port used by admission-controller-server.") + fs.StringVar(&c.MutateWebhookConfigName, "mutate-webhook-config-name", "", "Name of the mutatingwebhookconfiguration resource in Kubernetes [Deprecated]: it will be generated when not specified.") - flag.StringVar(&c.MutateWebhookName, "mutate-webhook-name", "", + fs.StringVar(&c.MutateWebhookName, "mutate-webhook-name", "", "Name of the webhook entry in the webhook config. [Deprecated]: it will be generated when not specified") - flag.StringVar(&c.ValidateWebhookConfigName, "validate-webhook-config-name", "", + fs.StringVar(&c.ValidateWebhookConfigName, "validate-webhook-config-name", "", "Name of the mutatingwebhookconfiguration resource in Kubernetes. [Deprecated]: it will be generated when not specified") - flag.StringVar(&c.ValidateWebhookName, "validate-webhook-name", "", + fs.StringVar(&c.ValidateWebhookName, "validate-webhook-name", "", "Name of the webhook entry in the webhook config. [Deprecated]: it will be generated when not specified") - flag.BoolVar(&c.PrintVersion, "version", false, "Show version and quit") - flag.StringVar(&c.AdmissionServiceNamespace, "webhook-namespace", "default", "The namespace of this webhook") - flag.StringVar(&c.AdmissionServiceName, "webhook-service-name", "admission-service", "The name of this admission service") - flag.StringVar(&c.SchedulerName, "scheduler-name", defaultSchedulerName, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name") + fs.BoolVar(&c.PrintVersion, "version", false, "Show version and quit") + fs.StringVar(&c.AdmissionServiceNamespace, "webhook-namespace", "default", "The namespace of this webhook") + fs.StringVar(&c.AdmissionServiceName, "webhook-service-name", "admission-service", "The name of this admission service") + fs.StringVar(&c.SchedulerName, "scheduler-name", defaultSchedulerName, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name") } const ( @@ -235,13 +235,13 @@ func registerMutateWebhook(client admissionregistrationv1beta1.MutatingWebhookCo return err } if err == nil && existing != nil { - glog.Infof("Updating MutatingWebhookConfiguration %v", hook) + klog.Infof("Updating MutatingWebhookConfiguration %v", hook) existing.Webhooks = hook.Webhooks if _, err := client.Update(existing); err != nil { return err } } else { - glog.Infof("Creating MutatingWebhookConfiguration %v", hook) + klog.Infof("Creating MutatingWebhookConfiguration %v", hook) if _, err := client.Create(&hook); err != nil { return err } @@ -259,12 +259,12 @@ func registerValidateWebhook(client admissionregistrationv1beta1.ValidatingWebho } if err == nil && existing != nil { existing.Webhooks = hook.Webhooks - glog.Infof("Updating ValidatingWebhookConfiguration %v", hook) + klog.Infof("Updating ValidatingWebhookConfiguration %v", hook) if _, err := client.Update(existing); err != nil { return err } } else { - glog.Infof("Creating ValidatingWebhookConfiguration %v", hook) + klog.Infof("Creating ValidatingWebhookConfiguration %v", hook) if _, err := client.Create(&hook); err != nil { return err } diff --git a/cmd/admission/app/server.go b/cmd/admission/app/server.go index 41a1389a81..14e4b045c1 100644 --- a/cmd/admission/app/server.go +++ b/cmd/admission/app/server.go @@ -19,10 +19,10 @@ package app import ( "crypto/tls" - "github.com/golang/glog" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/klog" + "volcano.sh/volcano/cmd/admission/app/options" "volcano.sh/volcano/pkg/client/clientset/versioned" ) @@ -31,7 +31,7 @@ import ( func GetClient(restConfig *rest.Config) *kubernetes.Clientset { clientset, err := kubernetes.NewForConfig(restConfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientset } @@ -40,7 +40,7 @@ func GetClient(restConfig *rest.Config) *kubernetes.Clientset { func GetVolcanoClient(restConfig *rest.Config) *versioned.Clientset { clientset, err := versioned.NewForConfig(restConfig) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return clientset } @@ -52,7 +52,7 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { if len(config.CertFile) != 0 && len(config.KeyFile) != 0 { sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return &tls.Config{ @@ -63,7 +63,7 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { if len(restConfig.CertData) != 0 && len(restConfig.KeyData) != 0 { sCert, err := tls.X509KeyPair(restConfig.CertData, restConfig.KeyData) if err != nil { - glog.Fatal(err) + klog.Fatal(err) } return &tls.Config{ @@ -71,6 +71,6 @@ func ConfigTLS(config *options.Config, restConfig *rest.Config) *tls.Config { } } - glog.Fatal("tls: failed to find any tls config data") + klog.Fatal("tls: failed to find any tls config data") return &tls.Config{} } diff --git a/cmd/admission/main.go b/cmd/admission/main.go index c5d3789c41..37bc9e2ff4 100644 --- a/cmd/admission/main.go +++ b/cmd/admission/main.go @@ -16,17 +16,21 @@ limitations under the License. package main import ( - "flag" "io/ioutil" "net/http" "os" "os/signal" + "runtime" "strconv" "syscall" + "time" - "github.com/golang/glog" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" "volcano.sh/volcano/cmd/admission/app" "volcano.sh/volcano/cmd/admission/app/options" @@ -42,26 +46,35 @@ func serveMutateJobs(w http.ResponseWriter, r *http.Request) { admission.Serve(w, r, admission.MutateJobs) } +var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes") + func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + config := options.NewConfig() - config.AddFlags() - flag.Parse() + config.AddFlags(pflag.CommandLine) + + flag.InitFlags() if config.PrintVersion { version.PrintVersionAndExit() } + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() + http.HandleFunc(admission.AdmitJobPath, serveJobs) http.HandleFunc(admission.MutateJobPath, serveMutateJobs) if err := config.CheckPortOrDie(); err != nil { - glog.Fatalf("Configured port is invalid: %v", err) + klog.Fatalf("Configured port is invalid: %v", err) } addr := ":" + strconv.Itoa(config.Port) restConfig, err := clientcmd.BuildConfigFromFlags(config.Master, config.Kubeconfig) if err != nil { - glog.Fatalf("Unable to build k8s config: %v", err) + klog.Fatalf("Unable to build k8s config: %v", err) } admission.VolcanoClientSet = app.GetVolcanoClient(restConfig) @@ -70,12 +83,12 @@ func main() { caBundle, err := ioutil.ReadFile(config.CaCertFile) if err != nil { - glog.Fatalf("Unable to read cacert file: %v", err) + klog.Fatalf("Unable to read cacert file: %v", err) } err = options.RegisterWebhooks(config, app.GetClient(restConfig), caBundle) if err != nil { - glog.Fatalf("Unable to register webhook configs: %v", err) + klog.Fatalf("Unable to register webhook configs: %v", err) } stopChannel := make(chan os.Signal) @@ -89,7 +102,7 @@ func main() { go func() { err = server.ListenAndServeTLS("", "") if err != nil && err != http.ErrServerClosed { - glog.Fatalf("ListenAndServeTLS for admission webhook failed: %v", err) + klog.Fatalf("ListenAndServeTLS for admission webhook failed: %v", err) close(webhookServeError) } }() @@ -97,7 +110,7 @@ func main() { select { case <-stopChannel: if err := server.Close(); err != nil { - glog.Fatalf("Close admission server failed: %v", err) + klog.Fatalf("Close admission server failed: %v", err) } return case <-webhookServeError: diff --git a/cmd/cli/queue.go b/cmd/cli/queue.go index 039c8672b7..46f199b747 100644 --- a/cmd/cli/queue.go +++ b/cmd/cli/queue.go @@ -23,20 +23,40 @@ import ( ) func buildQueueCmd() *cobra.Command { - jobCmd := &cobra.Command{ + queueCmd := &cobra.Command{ Use: "queue", Short: "Queue Operations", } - jobRunCmd := &cobra.Command{ + queueCreateCmd := &cobra.Command{ Use: "create", Short: "creates queue", Run: func(cmd *cobra.Command, args []string) { checkError(cmd, queue.CreateQueue()) }, } - queue.InitRunFlags(jobRunCmd) - jobCmd.AddCommand(jobRunCmd) + queue.InitCreateFlags(queueCreateCmd) + queueCmd.AddCommand(queueCreateCmd) + + queueDeleteCmd := &cobra.Command{ + Use: "delete", + Short: "delete queue", + Run: func(cmd *cobra.Command, args []string) { + checkError(cmd, queue.DeleteQueue()) + }, + } + queue.InitDeleteFlags(queueDeleteCmd) + queueCmd.AddCommand(queueDeleteCmd) + + queueOperateCmd := &cobra.Command{ + Use: "operate queue", + Short: "operate queue", + Run: func(cmd *cobra.Command, args []string) { + checkError(cmd, queue.OperateQueue()) + }, + } + queue.InitOperateFlags(queueOperateCmd) + queueCmd.AddCommand(queueOperateCmd) queueListCmd := &cobra.Command{ Use: "list", @@ -46,7 +66,7 @@ func buildQueueCmd() *cobra.Command { }, } queue.InitListFlags(queueListCmd) - jobCmd.AddCommand(queueListCmd) + queueCmd.AddCommand(queueListCmd) queueGetCmd := &cobra.Command{ Use: "get", @@ -56,7 +76,7 @@ func buildQueueCmd() *cobra.Command { }, } queue.InitGetFlags(queueGetCmd) - jobCmd.AddCommand(queueGetCmd) + queueCmd.AddCommand(queueGetCmd) - return jobCmd + return queueCmd } diff --git a/cmd/cli/vcctl.go b/cmd/cli/vcctl.go index 7d32886659..42197f88a8 100644 --- a/cmd/cli/vcctl.go +++ b/cmd/cli/vcctl.go @@ -20,11 +20,11 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" "volcano.sh/volcano/pkg/version" ) @@ -34,9 +34,11 @@ var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum func main() { // flag.InitFlags() - // The default glog flush interval is 30 seconds, which is frighteningly long. - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + klog.InitFlags(nil) + + // The default klog flush interval is 30 seconds, which is frighteningly long. + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() rootCmd := cobra.Command{ Use: "vcctl", diff --git a/cmd/controllers/app/server.go b/cmd/controllers/app/server.go index b210f0059e..5f5496dd61 100644 --- a/cmd/controllers/app/server.go +++ b/cmd/controllers/app/server.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -129,7 +129,7 @@ func Run(opt *options.ServerOption) error { Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, }) diff --git a/cmd/controllers/main.go b/cmd/controllers/main.go index 64ee534b34..20df8642e3 100644 --- a/cmd/controllers/main.go +++ b/cmd/controllers/main.go @@ -18,13 +18,14 @@ package main import ( "fmt" "os" + "runtime" "time" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/klog" "volcano.sh/volcano/cmd/controllers/app" "volcano.sh/volcano/cmd/controllers/app/options" @@ -34,6 +35,9 @@ import ( var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes") func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + s := options.NewServerOption() s.AddFlags(pflag.CommandLine) @@ -46,9 +50,9 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } - // The default glog flush interval is 30 seconds, which is frighteningly long. - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + // The default klog flush interval is 30 seconds, which is frighteningly long. + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() if err := app.Run(s); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go index d7c1784d7e..3010bbd70b 100644 --- a/cmd/scheduler/app/server.go +++ b/cmd/scheduler/app/server.go @@ -23,7 +23,6 @@ import ( "os" "time" - "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus/promhttp" "volcano.sh/volcano/cmd/scheduler/app/options" @@ -36,6 +35,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog" // Register gcp auth _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -95,7 +95,7 @@ func Run(opt *options.ServerOption) error { go func() { http.Handle("/metrics", promhttp.Handler()) - glog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil)) + klog.Fatalf("Prometheus Http Server failed %s", http.ListenAndServe(opt.ListenAddress, nil)) }() if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler"); err != nil { @@ -149,7 +149,7 @@ func Run(opt *options.ServerOption) error { Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { - glog.Fatalf("leaderelection lost") + klog.Fatalf("leaderelection lost") }, }, }) diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index 63ea46e1b3..344a99c380 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -24,11 +24,11 @@ import ( // init pprof server _ "net/http/pprof" - "github.com/golang/glog" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/util/flag" + "k8s.io/klog" "volcano.sh/volcano/cmd/scheduler/app" "volcano.sh/volcano/cmd/scheduler/app/options" @@ -46,6 +46,8 @@ var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum func main() { runtime.GOMAXPROCS(runtime.NumCPU()) + klog.InitFlags(nil) + s := options.NewServerOption() s.AddFlags(pflag.CommandLine) s.RegisterOptions() @@ -56,8 +58,8 @@ func main() { os.Exit(1) } - go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) - defer glog.Flush() + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) + defer klog.Flush() if err := app.Run(s); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) diff --git a/docs/community/adopters.md b/docs/community/adopters.md index ce6df5f64d..6564288de7 100644 --- a/docs/community/adopters.md +++ b/docs/community/adopters.md @@ -14,4 +14,5 @@ please feel free to add yourself into the following list by a pull request. | [JD Retail Infrastructure Department](https://jd.com/) |[@yuanchen8911](https://github.com/yuanchen8911)| Evaluation | Spark on K8S | | [kt NexR](https://www.ktnexr.com) |[@minyk](https://github.com/minyk), [@dieselnexr](https://github.com/dieselnexr)| Evaluation | spark scheduler of our next cloud native product. | | [QTT Bigdata Infra](https://ir.qutoutiao.net/) |[@yuzhaojing](https://github.com/yuzhaojing) | Evaluation | Spark and AI on K8S. | +| [Vivo](https://www.vivo.com/) | [@zionwu](https://github.com/zionwu) | Evaluation | Scheduler for Vtraining (deep learning training platform) | diff --git a/docs/design/queue/queue-state-management.md b/docs/design/queue/queue-state-management.md index 3a15e31bdb..8a708427e8 100644 --- a/docs/design/queue/queue-state-management.md +++ b/docs/design/queue/queue-state-management.md @@ -122,8 +122,6 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: ValidatingWebhookConfiguration metadata: name: {{ .Release.Name }}-validate-queue - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-delete webhooks: - clientConfig: caBundle: "" @@ -171,8 +169,6 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: name: {{ .Release.Name }}-mutate-queue - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-delete webhooks: - clientConfig: caBundle: "" diff --git a/docs/development/development.md b/docs/development/development.md index 490023b33c..8c88df7d24 100644 --- a/docs/development/development.md +++ b/docs/development/development.md @@ -118,7 +118,7 @@ make e2e-test-kind If you want to run e2e test in a existing cluster with volcano deployed, run the following: ```bash -export VK_BIN= need to set vcctl binary path (eg:.../src/volcano.sh/volcano/_output/bin/) +export VC_BIN= need to set vcctl binary path (eg:.../src/volcano.sh/volcano/_output/bin/) KUBECONFIG=${KUBECONFIG} go test ./test/e2e ``` diff --git a/hack/check-generated-yaml.sh b/hack/check-generated-yaml.sh index 212e352a77..adc37d3d38 100755 --- a/hack/check-generated-yaml.sh +++ b/hack/check-generated-yaml.sh @@ -26,7 +26,8 @@ if ! diff ${VK_ROOT}/installer/volcano-development.yaml ${RELEASE_FOLDER}/volcan { echo echo "The Generated yaml is different from the one in installer/volcano-development.yaml" - echo "Suggest to sync both the files" + echo "please run 'make generate-yaml TAG=latest RELEASE_DIR=installer \ + && mv ${RELEASE_FOLDER}/volcano-latest.yaml ${VK_ROOT}/installer/volcano-development.yaml' to update" echo } >&2 false diff --git a/hack/generate-yaml.sh b/hack/generate-yaml.sh index 4c489cd715..e978c4f7ac 100755 --- a/hack/generate-yaml.sh +++ b/hack/generate-yaml.sh @@ -59,7 +59,7 @@ if [[ ! -d ${RELEASE_FOLDER} ]];then fi DEPLOYMENT_FILE=${RELEASE_FOLDER}/${YAML_FILENAME} -echo "Generating volcano yaml file into ${DEPLOYMENT_FILE}}" +echo "Generating volcano yaml file into ${DEPLOYMENT_FILE}" if [[ -f ${DEPLOYMENT_FILE} ]];then rm ${DEPLOYMENT_FILE} diff --git a/hack/lib/install.sh b/hack/lib/install.sh index 13418acd84..dd95f0e008 100644 --- a/hack/lib/install.sh +++ b/hack/lib/install.sh @@ -60,7 +60,7 @@ function install-helm { HELM_TEMP_DIR=`mktemp -d` curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > ${HELM_TEMP_DIR}/get_helm.sh #TODO: There are some issue with helm's latest version, remove '--version' when it get fixed. - chmod 700 ${HELM_TEMP_DIR}/get_helm.sh && ${HELM_TEMP_DIR}/get_helm.sh --version v2.13.0 + chmod 700 ${HELM_TEMP_DIR}/get_helm.sh && ${HELM_TEMP_DIR}/get_helm.sh --version v3.0.1 else echo -n "found helm, version: " && helm version fi diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index e875451c69..530b89a2a3 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -1,7 +1,7 @@ #!/bin/bash export VK_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.. -export VK_BIN=${VK_ROOT}/${BIN_DIR}/${BIN_OSARCH} +export VC_BIN=${VK_ROOT}/${BIN_DIR}/${BIN_OSARCH} export LOG_LEVEL=3 export SHOW_VOLCANO_LOGS=${SHOW_VOLCANO_LOGS:-1} export CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-1} @@ -25,32 +25,21 @@ function kind-up-cluster { } function install-volcano { - echo "Preparing helm tiller service account" - kubectl create serviceaccount --namespace kube-system tiller - kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller - install-helm - helm init --service-account tiller --kubeconfig ${KUBECONFIG} --wait echo "Pulling required docker images" docker pull ${MPI_EXAMPLE_IMAGE} docker pull ${TF_EXAMPLE_IMAGE} - echo "Loading docker images into kind cluster" - kind load docker-image ${MPI_EXAMPLE_IMAGE} ${CLUSTER_CONTEXT} - kind load docker-image ${TF_EXAMPLE_IMAGE} ${CLUSTER_CONTEXT} - echo "Install volcano chart" - helm install installer/helm/chart/volcano --namespace kube-system --name ${CLUSTER_NAME} --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait + helm install ${CLUSTER_NAME} installer/helm/chart/volcano --namespace kube-system --kubeconfig ${KUBECONFIG} --set basic.image_tag_version=${TAG} --set basic.scheduler_config_file=config/volcano-scheduler-ci.conf --wait } function uninstall-volcano { - helm delete ${CLUSTER_NAME} --purge --kubeconfig ${KUBECONFIG} + helm uninstall ${CLUSTER_NAME} -n kube-system } function generate-log { - echo "Generating tiller log files" - kubectl logs deployment/tiller-deploy -n kube-system > helm-tiller.log echo "Generating volcano log files" kubectl logs deployment/${CLUSTER_NAME}-admission -n kube-system > volcano-admission.log kubectl logs deployment/${CLUSTER_NAME}-controllers -n kube-system > volcano-controller.log diff --git a/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml b/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml index 13a52405f2..b9ebfabdfe 100644 --- a/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml +++ b/installer/helm/chart/volcano/templates/batch_v1alpha1_job.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: jobs.batch.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: batch.volcano.sh names: diff --git a/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml b/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml index 111b55cb0a..1850904314 100644 --- a/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml +++ b/installer/helm/chart/volcano/templates/bus_v1alpha1_command.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: commands.bus.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: bus.volcano.sh names: diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml index c5f18f3238..afdee63f52 100644 --- a/installer/helm/chart/volcano/templates/controllers.yaml +++ b/installer/helm/chart/volcano/templates/controllers.yaml @@ -46,6 +46,9 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create"] --- kind: ClusterRoleBinding diff --git a/installer/helm/chart/volcano/templates/default-queue.yaml b/installer/helm/chart/volcano/templates/default-queue.yaml deleted file mode 100644 index 00475756eb..0000000000 --- a/installer/helm/chart/volcano/templates/default-queue.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: scheduling.incubator.k8s.io/v1alpha1 -kind: Queue -metadata: - name: default - annotations: - "helm.sh/hook": post-install -spec: - weight: 1 diff --git a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml index 3b60f2bb55..244459a520 100644 --- a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml +++ b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_podgroup.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: podgroups.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: diff --git a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml index 46bc0ade69..1398b51b8e 100644 --- a/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml +++ b/installer/helm/chart/volcano/templates/scheduling_v1alpha1_queue.yaml @@ -2,8 +2,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: queues.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index c842ce3699..ea384e7bbf 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -321,6 +321,9 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create"] --- kind: ClusterRoleBinding @@ -371,8 +374,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: jobs.batch.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: batch.volcano.sh names: @@ -573,8 +574,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: commands.bus.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: bus.volcano.sh names: @@ -623,8 +622,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: podgroups.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: @@ -667,8 +664,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: queues.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install spec: group: scheduling.incubator.k8s.io names: diff --git a/pkg/admission/admission_controller.go b/pkg/admission/admission_controller.go index 22a370767d..c7f9794607 100644 --- a/pkg/admission/admission_controller.go +++ b/pkg/admission/admission_controller.go @@ -19,7 +19,6 @@ package admission import ( "fmt" - "github.com/golang/glog" "github.com/hashicorp/go-multierror" "k8s.io/api/admission/v1beta1" @@ -29,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog" "k8s.io/kubernetes/pkg/apis/core/validation" batchv1alpha1 "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -97,7 +97,7 @@ func addToScheme(scheme *runtime.Scheme) { //ToAdmissionResponse updates the admission response with the input error func ToAdmissionResponse(err error) *v1beta1.AdmissionResponse { - glog.Error(err) + klog.Error(err) return &v1beta1.AdmissionResponse{ Result: &metav1.Status{ Message: err.Error(), @@ -120,7 +120,7 @@ func DecodeJob(object runtime.RawExtension, resource metav1.GroupVersionResource if _, _, err := deserializer.Decode(raw, nil, &job); err != nil { return job, err } - glog.V(3).Infof("the job struct is %+v", job) + klog.V(3).Infof("the job struct is %+v", job) return job, nil } @@ -232,25 +232,30 @@ func getValidActions() []batchv1alpha1.Action { return actions } -// ValidateIO validate IO configuration -func ValidateIO(volumes []batchv1alpha1.VolumeSpec) (string, bool) { +// validateIO validates IO configuration +func validateIO(volumes []batchv1alpha1.VolumeSpec) error { volumeMap := map[string]bool{} for _, volume := range volumes { if len(volume.MountPath) == 0 { - return " mountPath is required;", true + return fmt.Errorf(" mountPath is required;") } if _, found := volumeMap[volume.MountPath]; found { - return fmt.Sprintf(" duplicated mountPath: %s;", volume.MountPath), true + return fmt.Errorf(" duplicated mountPath: %s;", volume.MountPath) + } + if volume.VolumeClaim == nil && volume.VolumeClaimName == "" { + return fmt.Errorf(" either VolumeClaim or VolumeClaimName must be specified;") } if len(volume.VolumeClaimName) != 0 { if volume.VolumeClaim != nil { - return fmt.Sprintf("Confilct: If you want to use an existing PVC, just specify VolumeClaimName. If you want to create a new PVC, you do not need to specify VolumeClaimName."), true + return fmt.Errorf("confilct: If you want to use an existing PVC, just specify VolumeClaimName." + + "If you want to create a new PVC, you do not need to specify VolumeClaimName") } if errMsgs := validation.ValidatePersistentVolumeName(volume.VolumeClaimName, false); len(errMsgs) > 0 { - return fmt.Sprintf("Illegal VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs), true + return fmt.Errorf("invalid VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs) } } + volumeMap[volume.MountPath] = true } - return "", false + return nil } diff --git a/pkg/admission/admit_job.go b/pkg/admission/admit_job.go index 1bfc9784ab..a34e8a3bf3 100644 --- a/pkg/admission/admit_job.go +++ b/pkg/admission/admit_job.go @@ -20,13 +20,12 @@ import ( "fmt" "strings" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog" k8score "k8s.io/kubernetes/pkg/apis/core" k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" k8scorevalid "k8s.io/kubernetes/pkg/apis/core/validation" @@ -43,7 +42,7 @@ var VolcanoClientSet vcclientset.Interface // AdmitJobs is to admit jobs and return response func AdmitJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation) + klog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation) job, err := DecodeJob(ar.Request.Object, ar.Request.Resource) if err != nil { @@ -147,8 +146,8 @@ func validateJob(job v1alpha1.Job, reviewResponse *v1beta1.AdmissionResponse) st } } - if validateInfo, ok := ValidateIO(job.Spec.Volumes); ok { - msg = msg + validateInfo + if err := validateIO(job.Spec.Volumes); err != nil { + msg = msg + err.Error() } // Check whether Queue already present or not diff --git a/pkg/admission/admit_job_test.go b/pkg/admission/admit_job_test.go index 88896d8ff3..f1e8243925 100644 --- a/pkg/admission/admit_job_test.go +++ b/pkg/admission/admit_job_test.go @@ -820,6 +820,57 @@ func TestValidateExecution(t *testing.T) { Name: "duplicate-mount-volume", Namespace: namespace, }, + Spec: v1alpha1.JobSpec{ + MinAvailable: 1, + Queue: "default", + Tasks: []v1alpha1.TaskSpec{ + { + Name: "task-1", + Replicas: 1, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"name": "test"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fake-name", + Image: "busybox:1.24", + }, + }, + }, + }, + }, + }, + Policies: []v1alpha1.LifecyclePolicy{ + { + Event: v1alpha1.AnyEvent, + Action: v1alpha1.AbortJobAction, + }, + }, + Volumes: []v1alpha1.VolumeSpec{ + { + MountPath: "/var", + VolumeClaimName: "pvc1", + }, + { + MountPath: "/var", + VolumeClaimName: "pvc2", + }, + }, + }, + }, + reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, + ret: " duplicated mountPath: /var;", + ExpectErr: true, + }, + { + Name: "volume without VolumeClaimName and VolumeClaim", + Job: v1alpha1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-volume", + Namespace: namespace, + }, Spec: v1alpha1.JobSpec{ MinAvailable: 1, Queue: "default", @@ -859,7 +910,7 @@ func TestValidateExecution(t *testing.T) { }, }, reviewResponse: v1beta1.AdmissionResponse{Allowed: true}, - ret: " duplicated mountPath: /var;", + ret: " either VolumeClaim or VolumeClaimName must be specified;", ExpectErr: true, }, // task Policy with any event and other events diff --git a/pkg/admission/admit_pod.go b/pkg/admission/admit_pod.go index 43395ff040..09d891f498 100644 --- a/pkg/admission/admit_pod.go +++ b/pkg/admission/admit_pod.go @@ -21,13 +21,12 @@ import ( "net/http" "strings" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/helpers" "volcano.sh/volcano/pkg/apis/scheduling/v1alpha1" @@ -42,7 +41,7 @@ func (c *Controller) ServerPods(w http.ResponseWriter, r *http.Request) { // AdmitPods is to admit pods and return response func (c *Controller) AdmitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("admitting pods -- %s", ar.Request.Operation) + klog.V(3).Infof("admitting pods -- %s", ar.Request.Operation) pod, err := decodePod(ar.Request.Object, ar.Request.Resource) if err != nil { @@ -82,7 +81,7 @@ func decodePod(object runtime.RawExtension, resource metav1.GroupVersionResource if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil { return pod, err } - glog.V(3).Infof("the pod struct is %+v", pod) + klog.V(3).Infof("the pod struct is %+v", pod) return pod, nil } diff --git a/pkg/admission/mutate_job.go b/pkg/admission/mutate_job.go index 6c08115215..8fba26f7e0 100644 --- a/pkg/admission/mutate_job.go +++ b/pkg/admission/mutate_job.go @@ -21,10 +21,9 @@ import ( "fmt" "strconv" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" ) @@ -42,7 +41,7 @@ type patchOperation struct { // MutateJobs mutate jobs func MutateJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - glog.V(3).Infof("mutating jobs") + klog.V(3).Infof("mutating jobs") job, err := DecodeJob(ar.Request.Object, ar.Request.Resource) if err != nil { @@ -66,7 +65,7 @@ func MutateJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { reviewResponse.Result = &metav1.Status{Message: err.Error()} return &reviewResponse } - glog.V(3).Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) + klog.V(3).Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) reviewResponse.Patch = patchBytes pt := v1beta1.PatchTypeJSONPatch reviewResponse.PatchType = &pt diff --git a/pkg/admission/server.go b/pkg/admission/server.go index 4c3fa15e88..f61a8d9d66 100644 --- a/pkg/admission/server.go +++ b/pkg/admission/server.go @@ -21,10 +21,9 @@ import ( "io/ioutil" "net/http" - "github.com/golang/glog" - "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" ) // Serve the http request @@ -39,7 +38,7 @@ func Serve(w http.ResponseWriter, r *http.Request, admit AdmitFunc) { // verify the content type is accurate contentType := r.Header.Get(CONTENTTYPE) if contentType != APPLICATIONJSON { - glog.Errorf("contentType=%s, expect application/json", contentType) + klog.Errorf("contentType=%s, expect application/json", contentType) return } @@ -51,15 +50,15 @@ func Serve(w http.ResponseWriter, r *http.Request, admit AdmitFunc) { } else { reviewResponse = admit(ar) } - glog.V(3).Infof("sending response: %v", reviewResponse) + klog.V(3).Infof("sending response: %v", reviewResponse) response := createResponse(reviewResponse, &ar) resp, err := json.Marshal(response) if err != nil { - glog.Error(err) + klog.Error(err) } if _, err := w.Write(resp); err != nil { - glog.Error(err) + klog.Error(err) } } diff --git a/pkg/apis/helpers/helpers.go b/pkg/apis/helpers/helpers.go index 130fcb6c1e..b5df3faf1d 100644 --- a/pkg/apis/helpers/helpers.go +++ b/pkg/apis/helpers/helpers.go @@ -26,8 +26,6 @@ import ( "syscall" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -38,9 +36,11 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" "k8s.io/client-go/kubernetes" + "k8s.io/klog" vcbatch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" vcbus "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + schedulerv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" ) // JobKind creates job GroupVersionKind @@ -49,6 +49,9 @@ var JobKind = vcbatch.SchemeGroupVersion.WithKind("Job") // CommandKind creates command GroupVersionKind var CommandKind = vcbus.SchemeGroupVersion.WithKind("Command") +// V1alpha2QueueKind is queue kind with v1alpha2 version +var V1alpha2QueueKind = schedulerv1alpha2.SchemeGroupVersion.WithKind("Queue") + // GetController returns the controller uid func GetController(obj interface{}) types.UID { accessor, err := meta.Accessor(obj) @@ -85,7 +88,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac cmOld, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Get(cmName, metav1.GetOptions{}) if err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -102,7 +105,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac } if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Create(cm); err != nil { - glog.V(3).Infof("Failed to create ConfigMap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create ConfigMap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -111,7 +114,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac cmOld.Data = data if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Update(cmOld); err != nil { - glog.V(3).Infof("Failed to update ConfigMap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to update ConfigMap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -123,7 +126,7 @@ func CreateConfigMapIfNotExist(job *vcbatch.Job, kubeClients kubernetes.Interfac func DeleteConfigmap(job *vcbatch.Job, kubeClients kubernetes.Interface, cmName string) error { if _, err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Get(cmName, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Configmap for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -133,7 +136,7 @@ func DeleteConfigmap(job *vcbatch.Job, kubeClients kubernetes.Interface, cmName if err := kubeClients.CoreV1().ConfigMaps(job.Namespace).Delete(cmName, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Configmap of Job %v/%v: %v", + klog.Errorf("Failed to delete Configmap of Job %v/%v: %v", job.Namespace, job.Name, err) return err } @@ -204,9 +207,9 @@ func runServer(server *http.Server, ln net.Listener) error { msg := fmt.Sprintf("Stopped listening on %s", listener.Addr().String()) select { case <-stopCh: - glog.Info(msg) + klog.Info(msg) default: - glog.Fatalf("%s due to error: %v", msg, err) + klog.Fatalf("%s due to error: %v", msg, err) } }() diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go index f3a738d6fe..468d5a6cc8 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go @@ -106,6 +106,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*QueueRequest)(nil), (*scheduling.QueueRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(a.(*QueueRequest), b.(*scheduling.QueueRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.QueueRequest)(nil), (*QueueRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(a.(*scheduling.QueueRequest), b.(*QueueRequest), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*QueueSpec)(nil), (*scheduling.QueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_QueueSpec_To_scheduling_QueueSpec(a.(*QueueSpec), b.(*scheduling.QueueSpec), scope) }); err != nil { @@ -321,6 +331,30 @@ func Convert_scheduling_QueueList_To_v1alpha2_QueueList(in *scheduling.QueueList return autoConvert_scheduling_QueueList_To_v1alpha2_QueueList(in, out, s) } +func autoConvert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in *QueueRequest, out *scheduling.QueueRequest, s conversion.Scope) error { + out.Name = in.Name + out.Event = scheduling.QueueEvent(in.Event) + out.Action = scheduling.QueueAction(in.Action) + return nil +} + +// Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest is an autogenerated conversion function. +func Convert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in *QueueRequest, out *scheduling.QueueRequest, s conversion.Scope) error { + return autoConvert_v1alpha2_QueueRequest_To_scheduling_QueueRequest(in, out, s) +} + +func autoConvert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in *scheduling.QueueRequest, out *QueueRequest, s conversion.Scope) error { + out.Name = in.Name + out.Event = QueueEvent(in.Event) + out.Action = QueueAction(in.Action) + return nil +} + +// Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest is an autogenerated conversion function. +func Convert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in *scheduling.QueueRequest, out *QueueRequest, s conversion.Scope) error { + return autoConvert_scheduling_QueueRequest_To_v1alpha2_QueueRequest(in, out, s) +} + func autoConvert_v1alpha2_QueueSpec_To_scheduling_QueueSpec(in *QueueSpec, out *scheduling.QueueSpec, s conversion.Scope) error { out.Weight = in.Weight out.Capability = *(*v1.ResourceList)(unsafe.Pointer(&in.Capability)) diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go index a66a4de4bc..7ae3636210 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go @@ -215,6 +215,22 @@ func (in *QueueList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueRequest) DeepCopyInto(out *QueueRequest) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueRequest. +func (in *QueueRequest) DeepCopy() *QueueRequest { + if in == nil { + return nil + } + out := new(QueueRequest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { *out = *in diff --git a/pkg/apis/scheduling/zz_generated.deepcopy.go b/pkg/apis/scheduling/zz_generated.deepcopy.go index 0410d2658e..55f65dd2cb 100644 --- a/pkg/apis/scheduling/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/zz_generated.deepcopy.go @@ -215,6 +215,22 @@ func (in *QueueList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueRequest) DeepCopyInto(out *QueueRequest) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueRequest. +func (in *QueueRequest) DeepCopy() *QueueRequest { + if in == nil { + return nil + } + out := new(QueueRequest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { *out = *in diff --git a/pkg/cli/queue/create.go b/pkg/cli/queue/create.go index 01df533243..68bac301d9 100644 --- a/pkg/cli/queue/create.go +++ b/pkg/cli/queue/create.go @@ -30,20 +30,23 @@ type createFlags struct { Name string Weight int32 + // State is state of Queue + State string } var createQueueFlags = &createFlags{} -// InitRunFlags is used to init all run flags -func InitRunFlags(cmd *cobra.Command) { +// InitCreateFlags is used to init all flags during queue creating +func InitCreateFlags(cmd *cobra.Command) { initFlags(cmd, &createQueueFlags.commonFlags) cmd.Flags().StringVarP(&createQueueFlags.Name, "name", "n", "test", "the name of queue") cmd.Flags().Int32VarP(&createQueueFlags.Weight, "weight", "w", 1, "the weight of the queue") + cmd.Flags().StringVarP(&createQueueFlags.State, "state", "S", "Open", "the state of queue") } -// CreateQueue creates queue +// CreateQueue create queue func CreateQueue() error { config, err := buildConfig(createQueueFlags.Master, createQueueFlags.Kubeconfig) if err != nil { @@ -56,6 +59,7 @@ func CreateQueue() error { }, Spec: schedulingV1alpha2.QueueSpec{ Weight: int32(createQueueFlags.Weight), + State: schedulingV1alpha2.QueueState(createQueueFlags.State), }, } diff --git a/pkg/cli/queue/delete.go b/pkg/cli/queue/delete.go new file mode 100644 index 0000000000..1ec9366325 --- /dev/null +++ b/pkg/cli/queue/delete.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + + "volcano.sh/volcano/pkg/client/clientset/versioned" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type deleteFlags struct { + commonFlags + + // Name is name of queue + Name string +} + +var deleteQueueFlags = &deleteFlags{} + +// InitDeleteFlags is used to init all flags during queue deleting +func InitDeleteFlags(cmd *cobra.Command) { + initFlags(cmd, &deleteQueueFlags.commonFlags) + + cmd.Flags().StringVarP(&deleteQueueFlags.Name, "name", "n", "", "the name of queue") +} + +// DeleteQueue delete queue +func DeleteQueue() error { + config, err := buildConfig(deleteQueueFlags.Master, deleteQueueFlags.Kubeconfig) + if err != nil { + return err + } + + if len(deleteQueueFlags.Name) == 0 { + return fmt.Errorf("Queue name must be specified") + } + + queueClient := versioned.NewForConfigOrDie(config) + return queueClient.SchedulingV1alpha2().Queues().Delete(deleteQueueFlags.Name, &metav1.DeleteOptions{}) +} diff --git a/pkg/cli/queue/delete_test.go b/pkg/cli/queue/delete_test.go new file mode 100644 index 0000000000..5a37394310 --- /dev/null +++ b/pkg/cli/queue/delete_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDeleteQueue(t *testing.T) { + response := v1alpha2.Queue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-queue", + }, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + val, err := json.Marshal(response) + if err == nil { + w.Write(val) + } + }) + + server := httptest.NewServer(handler) + defer server.Close() + + deleteQueueFlags.Master = server.URL + testCases := []struct { + Name string + QueueName string + ExpectValue error + }{ + { + Name: "Normal Case Delete Queue Succeed", + QueueName: "normal-case", + ExpectValue: nil, + }, + { + Name: "Abnormal Case Delete Queue Failed For Name Not Specified", + QueueName: "", + ExpectValue: fmt.Errorf("Queue name must be specified"), + }, + } + + for _, testCase := range testCases { + deleteQueueFlags.Name = testCase.QueueName + + err := DeleteQueue() + if false == reflect.DeepEqual(err, testCase.ExpectValue) { + t.Errorf("Case '%s' failed, expected: '%v', got '%v'", testCase.Name, testCase.ExpectValue, err) + } + } +} + +func TestInitDeleteFlags(t *testing.T) { + var cmd cobra.Command + InitDeleteFlags(&cmd) + + if cmd.Flag("name") == nil { + t.Errorf("Could not find the flag name") + } +} diff --git a/pkg/cli/queue/get.go b/pkg/cli/queue/get.go index 919df10c6f..da46d1909d 100644 --- a/pkg/cli/queue/get.go +++ b/pkg/cli/queue/get.go @@ -70,13 +70,14 @@ func GetQueue() error { // PrintQueue prints queue information func PrintQueue(queue *v1alpha2.Queue, writer io.Writer) { - _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s\n", - Name, Weight, Pending, Running, Unknown) + _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n", + Name, Weight, State, Inqueue, Pending, Running, Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } - _, err = fmt.Fprintf(writer, "%-25s%-8d%-8d%-8d%-8d\n", - queue.Name, queue.Spec.Weight, queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) + _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n", + queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue, + queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } diff --git a/pkg/cli/queue/list.go b/pkg/cli/queue/list.go index 33bd1b7d8a..b961320d2b 100644 --- a/pkg/cli/queue/list.go +++ b/pkg/cli/queue/list.go @@ -48,6 +48,12 @@ const ( // Unknown status of the queue Unknown string = "Unknown" + + // Inqueue status of queue + Inqueue string = "Inqueue" + + // State is state of queue + State string = "State" ) var listQueueFlags = &listFlags{} @@ -81,14 +87,15 @@ func ListQueue() error { // PrintQueues prints queue information func PrintQueues(queues *v1alpha2.QueueList, writer io.Writer) { - _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s\n", - Name, Weight, Pending, Running, Unknown) + _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n", + Name, Weight, State, Inqueue, Pending, Running, Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } for _, queue := range queues.Items { - _, err = fmt.Fprintf(writer, "%-25s%-8d%-8d%-8d%-8d\n", - queue.Name, queue.Spec.Weight, queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) + _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n", + queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue, + queue.Status.Pending, queue.Status.Running, queue.Status.Unknown) if err != nil { fmt.Printf("Failed to print queue command result: %s.\n", err) } diff --git a/pkg/cli/queue/operate.go b/pkg/cli/queue/operate.go new file mode 100644 index 0000000000..ac37f979b2 --- /dev/null +++ b/pkg/cli/queue/operate.go @@ -0,0 +1,105 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/client/clientset/versioned" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // ActionOpen is `open` action + ActionOpen = "open" + // ActionClose is `close` action + ActionClose = "close" + // ActionUpdate is `update` action + ActionUpdate = "update" +) + +type operateFlags struct { + commonFlags + + // Name is name of queue + Name string + // Weight is weight of queue + Weight int32 + // Action is operation action of queue + Action string +} + +var operateQueueFlags = &operateFlags{} + +// InitOperateFlags is used to init all flags during queue operating +func InitOperateFlags(cmd *cobra.Command) { + initFlags(cmd, &operateQueueFlags.commonFlags) + + cmd.Flags().StringVarP(&operateQueueFlags.Name, "name", "n", "", "the name of queue") + cmd.Flags().Int32VarP(&operateQueueFlags.Weight, "weight", "w", 0, "the weight of the queue") + cmd.Flags().StringVarP(&operateQueueFlags.Action, "action", "a", "", + "operate action to queue, valid actions are open, close, update") +} + +// OperateQueue operates queue +func OperateQueue() error { + config, err := buildConfig(operateQueueFlags.Master, operateQueueFlags.Kubeconfig) + if err != nil { + return err + } + + if len(operateQueueFlags.Name) == 0 { + return fmt.Errorf("Queue name must be specified") + } + + var action schedulingv1alpha2.QueueAction + + switch operateQueueFlags.Action { + case ActionOpen: + action = schedulingv1alpha2.OpenQueueAction + case ActionClose: + action = schedulingv1alpha2.CloseQueueAction + case ActionUpdate: + if operateQueueFlags.Weight == 0 { + return fmt.Errorf("When %s queue %s, weight must be specified, "+ + "the value must be greater than 0", ActionUpdate, operateQueueFlags.Name) + } + + queueClient := versioned.NewForConfigOrDie(config) + queue, err := queueClient.SchedulingV1alpha2().Queues().Get(operateQueueFlags.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + queue.Spec.Weight = int32(operateQueueFlags.Weight) + + _, err = queueClient.SchedulingV1alpha2().Queues().Update(queue) + + return err + case "": + return fmt.Errorf("Action can not be null") + default: + return fmt.Errorf("Action %s invalid, valid actions are %s, %s and %s", + operateQueueFlags.Action, ActionOpen, ActionClose, ActionUpdate) + } + + return createQueueCommand(config, action) +} diff --git a/pkg/cli/queue/operate_test.go b/pkg/cli/queue/operate_test.go new file mode 100644 index 0000000000..a3265ce7be --- /dev/null +++ b/pkg/cli/queue/operate_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestOperateQueue(t *testing.T) { + response := v1alpha2.Queue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-queue", + }, + } + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + val, err := json.Marshal(response) + if err == nil { + w.Write(val) + } + }) + + server := httptest.NewServer(handler) + defer server.Close() + + operateQueueFlags.Master = server.URL + testCases := []struct { + Name string + QueueName string + Weight int32 + Action string + ExpectValue error + }{ + { + Name: "Normal Case Operate Queue Succeed, Action close", + QueueName: "normal-case-action-close", + Action: ActionClose, + ExpectValue: nil, + }, + { + Name: "Normal Case Operate Queue Succeed, Action open", + QueueName: "normal-case-action-open", + Action: ActionOpen, + ExpectValue: nil, + }, + { + Name: "Normal Case Operate Queue Succeed, Update Weight", + QueueName: "normal-case-update-weight", + Action: ActionUpdate, + Weight: 3, + ExpectValue: nil, + }, + { + Name: "Abnormal Case Update Queue Failed For Invalid Weight", + QueueName: "abnormal-case-invalid-weight", + Action: ActionUpdate, + ExpectValue: fmt.Errorf("When %s queue %s, weight must be specified, "+ + "the value must be greater than 0", ActionUpdate, "abnormal-case-invalid-weight"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Name Not Specified", + QueueName: "", + ExpectValue: fmt.Errorf("Queue name must be specified"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Action null", + QueueName: "abnormal-case-null-action", + Action: "", + ExpectValue: fmt.Errorf("Action can not be null"), + }, + { + Name: "Abnormal Case Operate Queue Failed For Action Invalid", + QueueName: "abnormal-case-invalid-action", + Action: "invalid", + ExpectValue: fmt.Errorf("Action %s invalid, valid actions are %s, %s and %s", + "invalid", ActionOpen, ActionClose, ActionUpdate), + }, + } + + for _, testCase := range testCases { + operateQueueFlags.Name = testCase.QueueName + operateQueueFlags.Action = testCase.Action + operateQueueFlags.Weight = testCase.Weight + + err := OperateQueue() + if false == reflect.DeepEqual(err, testCase.ExpectValue) { + t.Errorf("Case '%s' failed, expected: '%v', got '%v'", testCase.Name, testCase.ExpectValue, err) + } + } +} + +func TestInitOperateFlags(t *testing.T) { + var cmd cobra.Command + InitOperateFlags(&cmd) + + if cmd.Flag("name") == nil { + t.Errorf("Could not find the flag name") + } + if cmd.Flag("weight") == nil { + t.Errorf("Could not find the flag weight") + } + if cmd.Flag("action") == nil { + t.Errorf("Could not find the flag action") + } +} diff --git a/pkg/cli/queue/queue_test.go b/pkg/cli/queue/queue_test.go index 126487eca2..c49e3ad48d 100644 --- a/pkg/cli/queue/queue_test.go +++ b/pkg/cli/queue/queue_test.go @@ -80,7 +80,7 @@ func getCommonFlags(master string) commonFlags { } func TestCreateQueue(t *testing.T) { - InitRunFlags(&cobra.Command{}) + InitCreateFlags(&cobra.Command{}) server := getTestQueueHTTPServer(t) defer server.Close() diff --git a/pkg/cli/queue/util.go b/pkg/cli/queue/util.go index f6617f287d..f862ac642d 100644 --- a/pkg/cli/queue/util.go +++ b/pkg/cli/queue/util.go @@ -17,8 +17,16 @@ limitations under the License. package queue import ( + "fmt" "os" + "strings" + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + "volcano.sh/volcano/pkg/apis/helpers" + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/client/clientset/versioned" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" // Initialize client auth plugin. @@ -35,3 +43,30 @@ func homeDir() string { func buildConfig(master, kubeconfig string) (*rest.Config, error) { return clientcmd.BuildConfigFromFlags(master, kubeconfig) } + +func createQueueCommand(config *rest.Config, action schedulingv1alpha2.QueueAction) error { + queueClient := versioned.NewForConfigOrDie(config) + queue, err := queueClient.SchedulingV1alpha2().Queues().Get(operateQueueFlags.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ctrlRef := metav1.NewControllerRef(queue, helpers.V1alpha2QueueKind) + cmd := &busv1alpha1.Command{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-%s-", + queue.Name, strings.ToLower(string(action))), + OwnerReferences: []metav1.OwnerReference{ + *ctrlRef, + }, + }, + TargetObject: ctrlRef, + Action: string(action), + } + + if _, err := queueClient.BusV1alpha1().Commands("default").Create(cmd); err != nil { + return err + } + + return nil +} diff --git a/pkg/controllers/OWNERS b/pkg/controllers/OWNERS index 16643b106d..f206ebbb72 100644 --- a/pkg/controllers/OWNERS +++ b/pkg/controllers/OWNERS @@ -1,4 +1,6 @@ reviewers: + - hzxuzhonghu - TommyLike approvers: + - hzxuzhonghu - TommyLike diff --git a/pkg/controllers/cache/cache.go b/pkg/controllers/cache/cache.go index cb76ce88fc..c2d45eaa76 100644 --- a/pkg/controllers/cache/cache.go +++ b/pkg/controllers/cache/cache.go @@ -21,13 +21,12 @@ import ( "sync" "time" - "github.com/golang/glog" - "golang.org/x/time/rate" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/controllers/apis" @@ -295,7 +294,7 @@ func (jc *jobCache) processCleanupJob() bool { job, ok := obj.(*apis.JobInfo) if !ok { - glog.Errorf("failed to convert %v to *apis.JobInfo", obj) + klog.Errorf("failed to convert %v to *apis.JobInfo", obj) return true } @@ -306,7 +305,7 @@ func (jc *jobCache) processCleanupJob() bool { jc.deletedJobs.Forget(obj) key := keyFn(job.Namespace, job.Name) delete(jc.jobs, key) - glog.V(3).Infof("Job <%s> was deleted.", key) + klog.V(3).Infof("Job <%s> was deleted.", key) } else { // Retry jc.deleteJob(job) @@ -315,7 +314,7 @@ func (jc *jobCache) processCleanupJob() bool { } func (jc *jobCache) deleteJob(job *apis.JobInfo) { - glog.V(3).Infof("Try to delete Job <%v/%v>", + klog.V(3).Infof("Try to delete Job <%v/%v>", job.Namespace, job.Name) jc.deletedJobs.AddRateLimited(job) diff --git a/pkg/controllers/garbagecollector/garbagecollector.go b/pkg/controllers/garbagecollector/garbagecollector.go index 160bd9cc4a..5bdd8fc2ea 100644 --- a/pkg/controllers/garbagecollector/garbagecollector.go +++ b/pkg/controllers/garbagecollector/garbagecollector.go @@ -20,13 +20,12 @@ import ( "fmt" "time" - "github.com/golang/glog" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "k8s.io/kubernetes/pkg/controller" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -80,8 +79,8 @@ func NewGarbageCollector(vkClient vcclientset.Interface) *GarbageCollector { func (gb *GarbageCollector) Run(stopCh <-chan struct{}) { defer gb.queue.ShutDown() - glog.Infof("Starting garbage collector") - defer glog.Infof("Shutting down garbage collector") + klog.Infof("Starting garbage collector") + defer klog.Infof("Shutting down garbage collector") go gb.jobInformer.Informer().Run(stopCh) if !controller.WaitForCacheSync("garbage collector", stopCh, gb.jobSynced) { @@ -95,7 +94,7 @@ func (gb *GarbageCollector) Run(stopCh <-chan struct{}) { func (gb *GarbageCollector) addJob(obj interface{}) { job := obj.(*v1alpha1.Job) - glog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { gb.enqueue(job) @@ -104,7 +103,7 @@ func (gb *GarbageCollector) addJob(obj interface{}) { func (gb *GarbageCollector) updateJob(old, cur interface{}) { job := cur.(*v1alpha1.Job) - glog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) + klog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name) if job.DeletionTimestamp == nil && needsCleanup(job) { gb.enqueue(job) @@ -112,10 +111,10 @@ func (gb *GarbageCollector) updateJob(old, cur interface{}) { } func (gb *GarbageCollector) enqueue(job *v1alpha1.Job) { - glog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) + klog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name) key, err := controller.KeyFunc(job) if err != nil { - glog.Errorf("couldn't get key for object %#v: %v", job, err) + klog.Errorf("couldn't get key for object %#v: %v", job, err) return } @@ -125,7 +124,7 @@ func (gb *GarbageCollector) enqueue(job *v1alpha1.Job) { func (gb *GarbageCollector) enqueueAfter(job *v1alpha1.Job, after time.Duration) { key, err := controller.KeyFunc(job) if err != nil { - glog.Errorf("couldn't get key for object %#v: %v", job, err) + klog.Errorf("couldn't get key for object %#v: %v", job, err) return } @@ -156,7 +155,7 @@ func (gb *GarbageCollector) handleErr(err error, key interface{}) { return } - glog.Errorf("error cleaning up Job %v, will retry: %v", key, err) + klog.Errorf("error cleaning up Job %v, will retry: %v", key, err) gb.queue.AddRateLimited(key) } @@ -171,7 +170,7 @@ func (gb *GarbageCollector) processJob(key string) error { return err } - glog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) + klog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name) // Ignore the Jobs that are already deleted or being deleted, or the ones that don't need clean up. job, err := gb.jobLister.Jobs(namespace).Get(name) if errors.IsNotFound(err) { @@ -210,7 +209,7 @@ func (gb *GarbageCollector) processJob(key string) error { PropagationPolicy: &policy, Preconditions: &metav1.Preconditions{UID: &fresh.UID}, } - glog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) + klog.V(4).Infof("Cleaning up Job %s/%s", namespace, name) return gb.vcClient.BatchV1alpha1().Jobs(fresh.Namespace).Delete(fresh.Name, options) } @@ -267,10 +266,10 @@ func timeLeft(j *v1alpha1.Job, since *time.Time) (*time.Duration, error) { return nil, err } if finishAt.UTC().After(since.UTC()) { - glog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) + klog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name) } remaining := expireAt.UTC().Sub(since.UTC()) - glog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) + klog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC()) return &remaining, nil } diff --git a/pkg/controllers/job/helpers/helpers.go b/pkg/controllers/job/helpers/helpers.go index 86583214e4..207c164c45 100644 --- a/pkg/controllers/job/helpers/helpers.go +++ b/pkg/controllers/job/helpers/helpers.go @@ -30,8 +30,8 @@ import ( const ( // PodNameFmt pod name format PodNameFmt = "%s-%s-%d" - // VolumeClaimFmt volume claim name format - VolumeClaimFmt = "%s-volume-%s" + // persistentVolumeClaimFmt represents persistent volume claim name format + persistentVolumeClaimFmt = "%s-pvc-%s" ) // GetTaskIndex returns task Index @@ -61,9 +61,9 @@ func GenRandomStr(l int) string { return string(result) } -// MakeVolumeClaimName creates volume claim name -func MakeVolumeClaimName(jobName string) string { - return fmt.Sprintf(VolumeClaimFmt, jobName, GenRandomStr(12)) +// GenPVCName generates pvc name with job name +func GenPVCName(jobName string) string { + return fmt.Sprintf(persistentVolumeClaimFmt, jobName, GenRandomStr(12)) } // GetJobKeyByReq gets the key for the job request diff --git a/pkg/controllers/job/job_controller.go b/pkg/controllers/job/job_controller.go index 058512136e..2c3cdec1a3 100644 --- a/pkg/controllers/job/job_controller.go +++ b/pkg/controllers/job/job_controller.go @@ -23,8 +23,6 @@ import ( "sync" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" "k8s.io/apimachinery/pkg/util/wait" @@ -38,6 +36,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" batchv1alpha1 "volcano.sh/volcano/pkg/apis/batch/v1alpha1" busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" @@ -125,7 +124,7 @@ func NewJobController( //Initialize event client eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(vcscheme.Scheme, v1.EventSource{Component: "vc-controllers"}) @@ -253,11 +252,11 @@ func (cc *Controller) Run(stopCh <-chan struct{}) { // Re-sync error tasks. go wait.Until(cc.processResyncTask, 0, stopCh) - glog.Infof("JobController is running ...... ") + klog.Infof("JobController is running ...... ") } func (cc *Controller) worker(i uint32) { - glog.Infof("worker %d start ...... ", i) + klog.Infof("worker %d start ...... ", i) for cc.processNextReq(i) { } @@ -297,7 +296,7 @@ func (cc *Controller) processNextReq(count uint32) bool { queue := cc.queueList[count] obj, shutdown := queue.Get() if shutdown { - glog.Errorf("Fail to pop item from queue") + klog.Errorf("Fail to pop item from queue") return false } @@ -306,30 +305,30 @@ func (cc *Controller) processNextReq(count uint32) bool { key := jobcache.JobKeyByReq(&req) if !cc.belongsToThisRoutine(key, count) { - glog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count) + klog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count) queueLocal := cc.getWorkerQueue(key) queueLocal.Add(req) return true } - glog.V(3).Infof("Try to handle request <%v>", req) + klog.V(3).Infof("Try to handle request <%v>", req) jobInfo, err := cc.cache.Get(jobcache.JobKeyByReq(&req)) if err != nil { // TODO(k82cn): ignore not-ready error. - glog.Errorf("Failed to get job by <%v> from cache: %v", req, err) + klog.Errorf("Failed to get job by <%v> from cache: %v", req, err) return true } st := state.NewState(jobInfo) if st == nil { - glog.Errorf("Invalid state <%s> of Job <%v/%v>", + klog.Errorf("Invalid state <%s> of Job <%v/%v>", jobInfo.Job.Status.State, jobInfo.Job.Namespace, jobInfo.Job.Name) return true } action := applyPolicies(jobInfo.Job, &req) - glog.V(3).Infof("Execute <%v> on Job <%s/%s> in <%s> by <%T>.", + klog.V(3).Infof("Execute <%v> on Job <%s/%s> in <%s> by <%T>.", action, req.Namespace, req.JobName, jobInfo.Job.Status.State.Phase, st) if action != batchv1alpha1.SyncJobAction { @@ -339,7 +338,7 @@ func (cc *Controller) processNextReq(count uint32) bool { if err := st.Execute(action); err != nil { if queue.NumRequeues(req) < maxRetries { - glog.V(2).Infof("Failed to handle Job <%s/%s>: %v", + klog.V(2).Infof("Failed to handle Job <%s/%s>: %v", jobInfo.Job.Namespace, jobInfo.Job.Name, err) // If any error, requeue it. queue.AddRateLimited(req) @@ -347,7 +346,7 @@ func (cc *Controller) processNextReq(count uint32) bool { } cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf( "Job failed on action %s for retry limit reached", action)) - glog.Warningf("Dropping job<%s/%s> out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err) + klog.Warningf("Dropping job<%s/%s> out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err) } // If no error, forget it. diff --git a/pkg/controllers/job/job_controller_actions.go b/pkg/controllers/job/job_controller_actions.go index 0b3f7abe87..cf95229018 100644 --- a/pkg/controllers/job/job_controller_actions.go +++ b/pkg/controllers/job/job_controller_actions.go @@ -17,17 +17,15 @@ limitations under the License. package job import ( - "errors" "fmt" "sort" "sync" "sync/atomic" - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" k8scontroller "k8s.io/kubernetes/pkg/controller" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -39,13 +37,13 @@ import ( ) func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseMap, updateStatus state.UpdateStatusFn) error { - glog.V(3).Infof("Killing Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) - defer glog.V(3).Infof("Finished Job <%s/%s> killing", jobInfo.Job.Namespace, jobInfo.Job.Name) + klog.V(3).Infof("Killing Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) + defer klog.V(3).Infof("Finished Job <%s/%s> killing", jobInfo.Job.Namespace, jobInfo.Job.Name) job := jobInfo.Job - glog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) + klog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) if job.DeletionTimestamp != nil { - glog.Infof("Job <%s/%s> is terminating, skip management process.", + klog.Infof("Job <%s/%s> is terminating, skip management process.", job.Namespace, job.Name) return nil } @@ -60,7 +58,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM total++ if pod.DeletionTimestamp != nil { - glog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) + klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) terminating++ continue } @@ -83,7 +81,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM } if len(errs) != 0 { - glog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs) + klog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs) cc.recorder.Event(job, v1.EventTypeWarning, k8scontroller.FailedDeletePodReason, fmt.Sprintf("Error deleting pods: %+v", errs)) return fmt.Errorf("failed to kill %d pods of %d", len(errs), total) @@ -116,12 +114,12 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM // Update Job status newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return err } if e := cc.cache.Update(newJob); e != nil { - glog.Errorf("KillJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("KillJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, e) return e } @@ -129,7 +127,7 @@ func (cc *Controller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseM // Delete PodGroup if err := cc.vcClient.SchedulingV1alpha2().PodGroups(job.Namespace).Delete(job.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete PodGroup of Job %v/%v: %v", + klog.Errorf("Failed to delete PodGroup of Job %v/%v: %v", job.Namespace, job.Name, err) return err } @@ -175,14 +173,14 @@ func (cc *Controller) createJob(job *batch.Job) (*batch.Job, error) { } func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateStatusFn) error { - glog.V(3).Infof("Starting to sync up Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) - defer glog.V(3).Infof("Finished Job <%s/%s> sync up", jobInfo.Job.Namespace, jobInfo.Job.Name) + klog.V(3).Infof("Starting to sync up Job <%s/%s>", jobInfo.Job.Namespace, jobInfo.Job.Name) + defer klog.V(3).Infof("Finished Job <%s/%s> sync up", jobInfo.Job.Namespace, jobInfo.Job.Name) job := jobInfo.Job.DeepCopy() - glog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) + klog.Infof("Current Version is: %d of job: %s/%s", job.Status.Version, job.Namespace, job.Name) if job.DeletionTimestamp != nil { - glog.Infof("Job <%s/%s> is terminating, skip management process.", + klog.Infof("Job <%s/%s> is terminating, skip management process.", job.Namespace, job.Name) return nil } @@ -227,7 +225,7 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt } else { delete(pods, podName) if pod.DeletionTimestamp != nil { - glog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) + klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name) atomic.AddInt32(&terminating, 1) continue } @@ -251,12 +249,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt // Failed to create Pod, waitCreationGroup a moment and then create it again // This is to ensure all podsMap under the same Job created // So gang-scheduling could schedule the Job successfully - glog.Errorf("Failed to create pod %s for Job %s, err %#v", + klog.Errorf("Failed to create pod %s for Job %s, err %#v", pod.Name, job.Name, err) appendError(&creationErrs, fmt.Errorf("failed to create pod %s, err: %#v", pod.Name, err)) } else { classifyAndAddUpPodBaseOnPhase(newPod, &pending, &running, &succeeded, &failed, &unknown) - glog.V(3).Infof("Created Task <%s> of Job <%s/%s>", + klog.V(3).Infof("Created Task <%s> of Job <%s/%s>", pod.Name, job.Namespace, job.Name) } }(pod) @@ -281,12 +279,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt // Failed to delete Pod, waitCreationGroup a moment and then create it again // This is to ensure all podsMap under the same Job created // So gang-scheduling could schedule the Job successfully - glog.Errorf("Failed to delete pod %s for Job %s, err %#v", + klog.Errorf("Failed to delete pod %s for Job %s, err %#v", pod.Name, job.Name, err) appendError(&deletionErrs, err) cc.resyncTask(pod) } else { - glog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>", + klog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>", pod.Name, job.Namespace, job.Name) atomic.AddInt32(&terminating, 1) } @@ -322,12 +320,12 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt } newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return err } if e := cc.cache.Update(newJob); e != nil { - glog.Errorf("SyncJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("SyncJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, e) return e } @@ -338,16 +336,15 @@ func (cc *Controller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateSt func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) { // If PVC does not exist, create them for Job. var needUpdate bool - volumes := job.Spec.Volumes if job.Status.ControlledResources == nil { job.Status.ControlledResources = make(map[string]string) } - for index, volume := range volumes { + for index, volume := range job.Spec.Volumes { vcName := volume.VolumeClaimName if len(vcName) == 0 { - //NOTE(k82cn): Ensure never have duplicated generated names. + // NOTE(k82cn): Ensure never have duplicated generated names. for { - vcName = jobhelpers.MakeVolumeClaimName(job.Name) + vcName = jobhelpers.GenPVCName(job.Name) exist, err := cc.checkPVCExist(job, vcName) if err != nil { return job, err @@ -363,31 +360,22 @@ func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) if err := cc.createPVC(job, vcName, volume.VolumeClaim); err != nil { return job, err } - job.Status.ControlledResources["volume-pvc-"+vcName] = vcName - } else { - job.Status.ControlledResources["volume-emptyDir-"+vcName] = vcName } } else { - if job.Status.ControlledResources["volume-emptyDir-"+vcName] == vcName || job.Status.ControlledResources["volume-pvc-"+vcName] == vcName { - continue - } exist, err := cc.checkPVCExist(job, vcName) if err != nil { return job, err } - if exist { - job.Status.ControlledResources["volume-pvc-"+vcName] = vcName - } else { - msg := fmt.Sprintf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName) - glog.Error(msg) - return job, errors.New(msg) + if !exist { + return job, fmt.Errorf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName) } } + job.Status.ControlledResources["volume-pvc-"+vcName] = vcName } if needUpdate { newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(job) if err != nil { - glog.Errorf("Failed to update Job %v/%v for volume claim name: %v ", + klog.Errorf("Failed to update Job %v/%v for volume claim name: %v ", job.Namespace, job.Name, err) return job, err } @@ -398,13 +386,13 @@ func (cc *Controller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) return job, nil } -func (cc *Controller) checkPVCExist(job *batch.Job, vcName string) (bool, error) { - if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(vcName); err != nil { +func (cc *Controller) checkPVCExist(job *batch.Job, pvc string) (bool, error) { + if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(pvc); err != nil { if apierrors.IsNotFound(err) { return false, nil } - glog.V(3).Infof("Failed to get PVC for job <%s/%s>: %v", - job.Namespace, job.Name, err) + klog.V(3).Infof("Failed to get PVC %s for job <%s/%s>: %v", + pvc, job.Namespace, job.Name, err) return false, err } return true, nil @@ -422,10 +410,10 @@ func (cc *Controller) createPVC(job *batch.Job, vcName string, volumeClaim *v1.P Spec: *volumeClaim, } - glog.V(3).Infof("Try to create PVC: %v", pvc) + klog.V(3).Infof("Try to create PVC: %v", pvc) if _, e := cc.kubeClient.CoreV1().PersistentVolumeClaims(job.Namespace).Create(pvc); e != nil { - glog.V(3).Infof("Failed to create PVC for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create PVC for Job <%s/%s>: %v", job.Namespace, job.Name, e) return e } @@ -436,7 +424,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { // If PodGroup does not exist, create one for Job. if _, err := cc.pgLister.PodGroups(job.Namespace).Get(job.Name); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get PodGroup for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get PodGroup for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -459,7 +447,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { if _, err = cc.vcClient.SchedulingV1alpha2().PodGroups(job.Namespace).Create(pg); err != nil { if !apierrors.IsAlreadyExists(err) { - glog.V(3).Infof("Failed to create PodGroup for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to create PodGroup for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -472,7 +460,7 @@ func (cc *Controller) createPodGroupIfNotExist(job *batch.Job) error { func (cc *Controller) deleteJobPod(jobName string, pod *v1.Pod) error { err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Delete(pod.Name, nil) if err != nil && !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v", + klog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v", pod.Namespace, pod.Name, jobName, err) return fmt.Errorf("failed to delete pod %s, err %#v", pod.Name, err) @@ -524,12 +512,12 @@ func (cc *Controller) initJobStatus(job *batch.Job) (*batch.Job, error) { job.Status.MinAvailable = int32(job.Spec.MinAvailable) newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(job) if err != nil { - glog.Errorf("Failed to update status of Job %v/%v: %v", + klog.Errorf("Failed to update status of Job %v/%v: %v", job.Namespace, job.Name, err) return nil, err } if err := cc.cache.Update(newJob); err != nil { - glog.Errorf("CreateJob - Failed to update Job %v/%v in cache: %v", + klog.Errorf("CreateJob - Failed to update Job %v/%v in cache: %v", newJob.Namespace, newJob.Name, err) return nil, err } diff --git a/pkg/controllers/job/job_controller_handler.go b/pkg/controllers/job/job_controller_handler.go index 749620113d..335f842a08 100644 --- a/pkg/controllers/job/job_controller_handler.go +++ b/pkg/controllers/job/job_controller_handler.go @@ -21,12 +21,11 @@ import ( "reflect" "strconv" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" bus "volcano.sh/volcano/pkg/apis/bus/v1alpha1" @@ -40,7 +39,7 @@ import ( func (cc *Controller) addCommand(obj interface{}) { cmd, ok := obj.(*bus.Command) if !ok { - glog.Errorf("obj is not Command") + klog.Errorf("obj is not Command") return } @@ -50,7 +49,7 @@ func (cc *Controller) addCommand(obj interface{}) { func (cc *Controller) addJob(obj interface{}) { job, ok := obj.(*batch.Job) if !ok { - glog.Errorf("obj is not Job") + klog.Errorf("obj is not Job") return } @@ -63,7 +62,7 @@ func (cc *Controller) addJob(obj interface{}) { // TODO(k82cn): if failed to add job, the cache should be refresh if err := cc.cache.Add(job); err != nil { - glog.Errorf("Failed to add job <%s/%s>: %v in cache", + klog.Errorf("Failed to add job <%s/%s>: %v in cache", job.Namespace, job.Name, err) } key := jobhelpers.GetJobKeyByReq(&req) @@ -74,25 +73,25 @@ func (cc *Controller) addJob(obj interface{}) { func (cc *Controller) updateJob(oldObj, newObj interface{}) { newJob, ok := newObj.(*batch.Job) if !ok { - glog.Errorf("newObj is not Job") + klog.Errorf("newObj is not Job") return } oldJob, ok := oldObj.(*batch.Job) if !ok { - glog.Errorf("oldJob is not Job") + klog.Errorf("oldJob is not Job") return } // NOTE: Since we only reconcile job based on Spec, we will ignore other attributes // For Job status, it's used internally and always been updated via our controller. if reflect.DeepEqual(newJob.Spec, oldJob.Spec) && newJob.Status.State.Phase == oldJob.Status.State.Phase { - glog.Infof("Job update event is ignored since no update in 'Spec'.") + klog.Infof("Job update event is ignored since no update in 'Spec'.") return } if err := cc.cache.Update(newJob); err != nil { - glog.Errorf("UpdateJob - Failed to update job <%s/%s>: %v in cache", + klog.Errorf("UpdateJob - Failed to update job <%s/%s>: %v in cache", newJob.Namespace, newJob.Name, err) } @@ -114,18 +113,18 @@ func (cc *Controller) deleteJob(obj interface{}) { // If we reached here it means the Job was deleted but its final state is unrecorded. tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } job, ok = tombstone.Obj.(*batch.Job) if !ok { - glog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj) + klog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj) return } } if err := cc.cache.Delete(job); err != nil { - glog.Errorf("Failed to delete job <%s/%s>: %v in cache", + klog.Errorf("Failed to delete job <%s/%s>: %v in cache", job.Namespace, job.Name, err) } } @@ -133,7 +132,7 @@ func (cc *Controller) deleteJob(obj interface{}) { func (cc *Controller) addPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", obj) + klog.Errorf("Failed to convert %v to v1.Pod", obj) return } // Filter out pods that are not created from volcano job @@ -143,21 +142,21 @@ func (cc *Controller) addPod(obj interface{}) { jobName, found := pod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } version, found := pod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", + klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", pod.Namespace, pod.Name) return } @@ -176,7 +175,7 @@ func (cc *Controller) addPod(obj interface{}) { } if err := cc.cache.AddPod(pod); err != nil { - glog.Errorf("Failed to add Pod <%s/%s>: %v to cache", + klog.Errorf("Failed to add Pod <%s/%s>: %v to cache", pod.Namespace, pod.Name, err) } key := jobhelpers.GetJobKeyByReq(&req) @@ -187,13 +186,13 @@ func (cc *Controller) addPod(obj interface{}) { func (cc *Controller) updatePod(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", oldObj) + klog.Errorf("Failed to convert %v to v1.Pod", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", newObj) + klog.Errorf("Failed to convert %v to v1.Pod", newObj) return } @@ -213,34 +212,34 @@ func (cc *Controller) updatePod(oldObj, newObj interface{}) { taskName, found := newPod.Annotations[batch.TaskSpecKey] if !found { - glog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } jobName, found := newPod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } version, found := newPod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod into number <%s/%s>, skipping", + klog.Infof("Failed to convert jobVersion of Pod into number <%s/%s>, skipping", newPod.Namespace, newPod.Name) return } if err := cc.cache.UpdatePod(newPod); err != nil { - glog.Errorf("Failed to update Pod <%s/%s>: %v in cache", + klog.Errorf("Failed to update Pod <%s/%s>: %v in cache", newPod.Namespace, newPod.Name, err) } @@ -284,12 +283,12 @@ func (cc *Controller) deletePod(obj interface{}) { // If we reached here it means the pod was deleted but its final state is unrecorded. tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) + klog.Errorf("Couldn't get object from tombstone %#v", obj) return } pod, ok = tombstone.Obj.(*v1.Pod) if !ok { - glog.Errorf("Tombstone contained object that is not a Pod: %#v", obj) + klog.Errorf("Tombstone contained object that is not a Pod: %#v", obj) return } } @@ -301,28 +300,28 @@ func (cc *Controller) deletePod(obj interface{}) { taskName, found := pod.Annotations[batch.TaskSpecKey] if !found { - glog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } jobName, found := pod.Annotations[batch.JobNameKey] if !found { - glog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } version, found := pod.Annotations[batch.JobVersion] if !found { - glog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", + klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping", pod.Namespace, pod.Name) return } dVersion, err := strconv.Atoi(version) if err != nil { - glog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", + klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping", pod.Namespace, pod.Name) return } @@ -337,7 +336,7 @@ func (cc *Controller) deletePod(obj interface{}) { } if err := cc.cache.DeletePod(pod); err != nil { - glog.Errorf("Failed to delete Pod <%s/%s>: %v in cache", + klog.Errorf("Failed to delete Pod <%s/%s>: %v in cache", pod.Namespace, pod.Name, err) } @@ -349,7 +348,7 @@ func (cc *Controller) deletePod(obj interface{}) { func (cc *Controller) recordJobEvent(namespace, name string, event batch.JobEvent, message string) { job, err := cc.cache.Get(jobcache.JobKeyByName(namespace, name)) if err != nil { - glog.Warningf("Failed to find job in cache when reporting job event <%s/%s>: %v", + klog.Warningf("Failed to find job in cache when reporting job event <%s/%s>: %v", namespace, name, err) return } @@ -372,7 +371,7 @@ func (cc *Controller) processNextCommand() bool { if err := cc.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(cmd.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Command <%s/%s>.", cmd.Namespace, cmd.Name) + klog.Errorf("Failed to delete Command <%s/%s>.", cmd.Namespace, cmd.Name) cc.commandQueue.AddRateLimited(cmd) } return true @@ -398,19 +397,19 @@ func (cc *Controller) processNextCommand() bool { func (cc *Controller) updatePodGroup(oldObj, newObj interface{}) { oldPG, ok := oldObj.(*scheduling.PodGroup) if !ok { - glog.Errorf("Failed to convert %v to PodGroup", newObj) + klog.Errorf("Failed to convert %v to PodGroup", newObj) return } newPG, ok := newObj.(*scheduling.PodGroup) if !ok { - glog.Errorf("Failed to convert %v to PodGroup", newObj) + klog.Errorf("Failed to convert %v to PodGroup", newObj) return } _, err := cc.cache.Get(jobcache.JobKeyByName(newPG.Namespace, newPG.Name)) if err != nil && newPG.Annotations != nil { - glog.Warningf( + klog.Warningf( "Failed to find job in cache by PodGroup, this may not be a PodGroup for volcano job.") } @@ -466,11 +465,11 @@ func convert2PriorityClass(obj interface{}) *v1beta1.PriorityClass { var ok bool pc, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return nil } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return nil } diff --git a/pkg/controllers/job/job_controller_plugins.go b/pkg/controllers/job/job_controller_plugins.go index 861a1e829b..b410fc9d08 100644 --- a/pkg/controllers/job/job_controller_plugins.go +++ b/pkg/controllers/job/job_controller_plugins.go @@ -19,9 +19,8 @@ package job import ( "fmt" - "github.com/golang/glog" - "k8s.io/api/core/v1" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/controllers/job/plugins" @@ -34,12 +33,12 @@ func (cc *Controller) pluginOnPodCreate(job *batch.Job, pod *v1.Pod) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnPodCreate(pod, job); err != nil { - glog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err) + klog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err) return err } @@ -56,12 +55,12 @@ func (cc *Controller) pluginOnJobAdd(job *batch.Job) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnJobAdd(job); err != nil { - glog.Errorf("Failed to process on job add plugin %s, err %v.", name, err) + klog.Errorf("Failed to process on job add plugin %s, err %v.", name, err) return err } @@ -76,12 +75,12 @@ func (cc *Controller) pluginOnJobDelete(job *batch.Job) error { pb, found := plugins.GetPluginBuilder(name) if !found { err := fmt.Errorf("failed to get plugin %s", name) - glog.Error(err) + klog.Error(err) return err } - glog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) + klog.Infof("Starting to execute plugin at : %s on job: <%s/%s>", name, job.Namespace, job.Name) if err := pb(client, args).OnJobDelete(job); err != nil { - glog.Errorf("failed to process on job delete plugin %s, err %v.", name, err) + klog.Errorf("failed to process on job delete plugin %s, err %v.", name, err) return err } diff --git a/pkg/controllers/job/job_controller_resync.go b/pkg/controllers/job/job_controller_resync.go index 888cf250a7..66e28d9ed5 100644 --- a/pkg/controllers/job/job_controller_resync.go +++ b/pkg/controllers/job/job_controller_resync.go @@ -20,13 +20,13 @@ import ( "fmt" "time" - "github.com/golang/glog" "golang.org/x/time/rate" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" ) func newRateLimitingQueue() workqueue.RateLimitingInterface { @@ -53,12 +53,12 @@ func (cc *Controller) processResyncTask() { task, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("failed to convert %v to *v1.Pod", obj) + klog.Errorf("failed to convert %v to *v1.Pod", obj) return } if err := cc.syncTask(task); err != nil { - glog.Errorf("Failed to sync pod <%v/%v>, retry it, err %v", task.Namespace, task.Name, err) + klog.Errorf("Failed to sync pod <%v/%v>, retry it, err %v", task.Namespace, task.Name, err) cc.resyncTask(task) } } @@ -71,10 +71,10 @@ func (cc *Controller) syncTask(oldTask *v1.Pod) error { if err != nil { if errors.IsNotFound(err) { if err := cc.cache.DeletePod(oldTask); err != nil { - glog.Errorf("failed to delete cache pod <%v/%v>, err %v.", oldTask.Namespace, oldTask.Name, err) + klog.Errorf("failed to delete cache pod <%v/%v>, err %v.", oldTask.Namespace, oldTask.Name, err) return err } - glog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) + klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) return nil } diff --git a/pkg/controllers/job/job_controller_util.go b/pkg/controllers/job/job_controller_util.go index df37c349d4..39d8c06f00 100644 --- a/pkg/controllers/job/job_controller_util.go +++ b/pkg/controllers/job/job_controller_util.go @@ -19,11 +19,10 @@ package job import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/apis/helpers" @@ -63,24 +62,19 @@ func createJobPod(job *batch.Job, template *v1.PodTemplateSpec, ix int) *v1.Pod vcName := volume.VolumeClaimName name := fmt.Sprintf("%s-%s", job.Name, jobhelpers.GenRandomStr(12)) if _, ok := volumeMap[vcName]; !ok { - if _, ok := job.Status.ControlledResources["volume-emptyDir-"+vcName]; ok { - volume := v1.Volume{ - Name: name, - } - volume.EmptyDir = &v1.EmptyDirVolumeSource{} - pod.Spec.Volumes = append(pod.Spec.Volumes, volume) - } else { - volume := v1.Volume{ - Name: name, - } - volume.PersistentVolumeClaim = &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: vcName, - } - pod.Spec.Volumes = append(pod.Spec.Volumes, volume) + volume := v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: vcName, + }, + }, } + pod.Spec.Volumes = append(pod.Spec.Volumes, volume) volumeMap[vcName] = name } else { - name = volumeMap[vcName] + // duplicate volumes, should be prevented + continue } for i, c := range pod.Spec.Containers { @@ -137,7 +131,7 @@ func applyPolicies(job *batch.Job, req *apis.Request) batch.Action { // For all the requests triggered from discarded job resources will perform sync action instead if req.JobVersion < job.Status.Version { - glog.Infof("Request %s is outdated, will perform sync instead.", req) + klog.Infof("Request %s is outdated, will perform sync instead.", req) return batch.SyncJobAction } diff --git a/pkg/controllers/job/job_controller_util_test.go b/pkg/controllers/job/job_controller_util_test.go index 624e081fff..da5cf11576 100644 --- a/pkg/controllers/job/job_controller_util_test.go +++ b/pkg/controllers/job/job_controller_util_test.go @@ -213,11 +213,6 @@ func TestCreateJobPod(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, PodTemplate: &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -292,11 +287,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Action: v1alpha1.EnqueueAction, @@ -332,11 +322,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Event: v1alpha1.OutOfSyncEvent, @@ -372,12 +357,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - Version: 2, - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ JobVersion: 1, @@ -420,11 +399,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -466,11 +440,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -507,11 +476,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ TaskName: "task1", @@ -554,11 +518,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{ Event: v1alpha1.CommandIssuedEvent, @@ -601,11 +560,6 @@ func TestApplyPolicies(t *testing.T) { }, }, }, - Status: v1alpha1.JobStatus{ - ControlledResources: map[string]string{ - "volume-emptyDir-vc1": "vc1", - }, - }, }, Request: &apis.Request{}, ReturnVal: v1alpha1.SyncJobAction, diff --git a/pkg/controllers/job/plugins/ssh/ssh.go b/pkg/controllers/job/plugins/ssh/ssh.go index 1c7e7ff023..bef4c57fcf 100644 --- a/pkg/controllers/job/plugins/ssh/ssh.go +++ b/pkg/controllers/job/plugins/ssh/ssh.go @@ -26,9 +26,8 @@ import ( "golang.org/x/crypto/ssh" - "github.com/golang/glog" - "k8s.io/api/core/v1" + "k8s.io/klog" batch "volcano.sh/volcano/pkg/apis/batch/v1alpha1" "volcano.sh/volcano/pkg/apis/helpers" @@ -44,14 +43,23 @@ type sshPlugin struct { Clientset pluginsinterface.PluginClientset // flag parse args - noRoot bool + noRoot bool + sshKeyFilePath string } // New creates ssh plugin func New(client pluginsinterface.PluginClientset, arguments []string) pluginsinterface.PluginInterface { - sshPlugin := sshPlugin{pluginArguments: arguments, Clientset: client} + sshPlugin := sshPlugin{ + pluginArguments: arguments, + Clientset: client, + sshKeyFilePath: SSHAbsolutePath, + } sshPlugin.addFlags() + // if not set ssh key files path, use the default. + if sshPlugin.noRoot && sshPlugin.sshKeyFilePath == SSHAbsolutePath { + sshPlugin.sshKeyFilePath = env.ConfigMapMountPath + "/" + SSHRelativePath + } return &sshPlugin } @@ -94,10 +102,6 @@ func (sp *sshPlugin) OnJobDelete(job *batch.Job) error { } func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { - sshPath := SSHAbsolutePath - if sp.noRoot { - sshPath = env.ConfigMapMountPath + "/" + SSHRelativePath - } cmName := sp.cmName(job) sshVolume := v1.Volume{ @@ -129,7 +133,7 @@ func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { DefaultMode: &mode, } - if sshPath != SSHAbsolutePath { + if sp.sshKeyFilePath != SSHAbsolutePath { var noRootMode int32 = 0755 sshVolume.ConfigMap.DefaultMode = &noRootMode } @@ -138,7 +142,7 @@ func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) { for i, c := range pod.Spec.Containers { vm := v1.VolumeMount{ - MountPath: sshPath, + MountPath: sp.sshKeyFilePath, SubPath: SSHRelativePath, Name: cmName, } @@ -154,7 +158,7 @@ func generateRsaKey(job *batch.Job) (map[string]string, error) { privateKey, err := rsa.GenerateKey(rand.Reader, bitSize) if err != nil { - glog.Errorf("rsa generateKey err: %v", err) + klog.Errorf("rsa generateKey err: %v", err) return nil, err } @@ -168,7 +172,7 @@ func generateRsaKey(job *batch.Job) (map[string]string, error) { // id_rsa.pub publicRsaKey, err := ssh.NewPublicKey(&privateKey.PublicKey) if err != nil { - glog.Errorf("ssh newPublicKey err: %v", err) + klog.Errorf("ssh newPublicKey err: %v", err) return nil, err } publicKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey) @@ -187,10 +191,13 @@ func (sp *sshPlugin) cmName(job *batch.Job) string { func (sp *sshPlugin) addFlags() { flagSet := flag.NewFlagSet(sp.Name(), flag.ContinueOnError) + // TODO: deprecate no-root flagSet.BoolVar(&sp.noRoot, "no-root", sp.noRoot, "The ssh user, --no-root is common user") + flagSet.StringVar(&sp.sshKeyFilePath, "ssh-key-file-path", sp.sshKeyFilePath, "The path used to store "+ + "ssh private and public keys, it is `/root/.ssh` by default.") if err := flagSet.Parse(sp.pluginArguments); err != nil { - glog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) + klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) } return } diff --git a/pkg/controllers/job/plugins/ssh/ssh_test.go b/pkg/controllers/job/plugins/ssh/ssh_test.go new file mode 100644 index 0000000000..4017c81fd1 --- /dev/null +++ b/pkg/controllers/job/plugins/ssh/ssh_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "testing" + + "volcano.sh/volcano/pkg/controllers/job/plugins/env" + "volcano.sh/volcano/pkg/controllers/job/plugins/interface" +) + +func TestSSHPlugin(t *testing.T) { + + tests := []struct { + name string + params []string + noRoot bool + sshKeyFilePath string + }{ + { + name: "no params specified", + noRoot: false, + sshKeyFilePath: SSHAbsolutePath, + }, + { + name: "--no-root=true, ssh-key-file-path empty", + params: []string{"--no-root=true"}, + noRoot: true, + sshKeyFilePath: env.ConfigMapMountPath + "/" + SSHRelativePath, + }, + { + name: "--no-root=true, --ssh-key-file-path=/a/b", + params: []string{"--no-root=true", "--ssh-key-file-path=/a/b"}, + noRoot: true, + sshKeyFilePath: "/a/b", + }, + { + name: "--ssh-key-file-path=/a/b", + params: []string{"--ssh-key-file-path=/a/b"}, + noRoot: false, + sshKeyFilePath: "/a/b", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pluginInterface := New(pluginsinterface.PluginClientset{}, test.params) + plugin := pluginInterface.(*sshPlugin) + if plugin.noRoot != test.noRoot { + t.Errorf("Expected noRoot=%v, got %v", test.noRoot, plugin.noRoot) + } + + if plugin.sshKeyFilePath != test.sshKeyFilePath { + t.Errorf("Expected sshKeyFilePath=%s, got %s", test.sshKeyFilePath, plugin.sshKeyFilePath) + } + }) + } +} diff --git a/pkg/controllers/job/plugins/svc/svc.go b/pkg/controllers/job/plugins/svc/svc.go index c80070deee..54c79a47a8 100644 --- a/pkg/controllers/job/plugins/svc/svc.go +++ b/pkg/controllers/job/plugins/svc/svc.go @@ -21,9 +21,10 @@ import ( "fmt" "strings" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -63,13 +64,23 @@ func (sp *servicePlugin) addFlags() { "set publishNotReadyAddresses of svc to true") if err := flagSet.Parse(sp.pluginArguments); err != nil { - glog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) + klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err) } return } func (sp *servicePlugin) OnPodCreate(pod *v1.Pod, job *batch.Job) error { - // use podName.serviceName as default pod DNS domain + // Add `hostname` and `subdomain` for pod, mount service config for pod. + // A pod with `hostname` and `subdomain` will have the fully qualified domain name(FQDN) + // `hostname.subdomain.namespace.svc.cluster-domain.example`. + // If there exists a headless service in the same namespace as the pod and with the + // same name as the `subdomain`, the cluster's KubeDNS Server will returns an A record for + // the Pods's fully qualified hostname, pointing to the Pod’s IP. + // `hostname.subdomain` will be used as address of the pod. + // By default, a client Pod’s DNS search list will include the Pod’s own namespace and + // the cluster’s default domain, so the pod can be accessed by pods in the same namespace + // through the address of pod. + // More info: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service if len(pod.Spec.Hostname) == 0 { pod.Spec.Hostname = pod.Name } @@ -97,6 +108,11 @@ func (sp *servicePlugin) OnJobAdd(job *batch.Job) error { return err } + // TODO: maybe add a flag + if err := sp.createNetworkPolicyIfNotExist(job); err != nil { + return err + } + job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() return nil @@ -109,7 +125,7 @@ func (sp *servicePlugin) OnJobDelete(job *batch.Job) error { if err := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Delete(job.Name, nil); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to delete Service of Job %v/%v: %v", job.Namespace, job.Name, err) + klog.Errorf("Failed to delete Service of Job %v/%v: %v", job.Namespace, job.Name, err) return err } } @@ -146,7 +162,7 @@ func (sp *servicePlugin) createServiceIfNotExist(job *batch.Job) error { // If Service does not exist, create one for Job. if _, err := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Get(job.Name, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { - glog.V(3).Infof("Failed to get Service for Job <%s/%s>: %v", + klog.V(3).Infof("Failed to get Service for Job <%s/%s>: %v", job.Namespace, job.Name, err) return err } @@ -178,7 +194,57 @@ func (sp *servicePlugin) createServiceIfNotExist(job *batch.Job) error { } if _, e := sp.Clientset.KubeClients.CoreV1().Services(job.Namespace).Create(svc); e != nil { - glog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) + klog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) + return e + } + job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() + + } + + return nil +} + +// Limit pods can be accessible only by pods belong to the job. +func (sp *servicePlugin) createNetworkPolicyIfNotExist(job *batch.Job) error { + // If network policy does not exist, create one for Job. + if _, err := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Get(job.Name, metav1.GetOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + klog.V(3).Infof("Failed to get NetworkPolicy for Job <%s/%s>: %v", + job.Namespace, job.Name, err) + return err + } + + networkpolicy := &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: job.Namespace, + Name: job.Name, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(job, helpers.JobKind), + }, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + batch.JobNameKey: job.Name, + batch.JobNamespaceKey: job.Namespace, + }, + }, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + batch.JobNameKey: job.Name, + batch.JobNamespaceKey: job.Namespace, + }, + }, + }}, + }}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } + + if _, e := sp.Clientset.KubeClients.NetworkingV1().NetworkPolicies(job.Namespace).Create(networkpolicy); e != nil { + klog.V(3).Infof("Failed to create Service for Job <%s/%s>: %v", job.Namespace, job.Name, e) return e } job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name() diff --git a/pkg/controllers/podgroup/pg_controller.go b/pkg/controllers/podgroup/pg_controller.go index 82159b5c4e..dc5a031031 100644 --- a/pkg/controllers/podgroup/pg_controller.go +++ b/pkg/controllers/podgroup/pg_controller.go @@ -17,8 +17,6 @@ limitations under the License. package podgroup import ( - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" @@ -27,6 +25,7 @@ import ( corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" scheduling "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" vcclientset "volcano.sh/volcano/pkg/client/clientset/versioned" @@ -107,7 +106,7 @@ func (cc *Controller) Run(stopCh <-chan struct{}) { go wait.Until(cc.worker, 0, stopCh) - glog.Infof("PodgroupController is running ...... ") + klog.Infof("PodgroupController is running ...... ") } func (cc *Controller) worker() { @@ -118,7 +117,7 @@ func (cc *Controller) worker() { func (cc *Controller) processNextReq() bool { obj, shutdown := cc.queue.Get() if shutdown { - glog.Errorf("Fail to pop item from queue") + klog.Errorf("Fail to pop item from queue") return false } @@ -127,13 +126,13 @@ func (cc *Controller) processNextReq() bool { pod, err := cc.podLister.Pods(req.podNamespace).Get(req.podName) if err != nil { - glog.Errorf("Failed to get pod by <%v> from cache: %v", req, err) + klog.Errorf("Failed to get pod by <%v> from cache: %v", req, err) return true } // normal pod use volcano if err := cc.createNormalPodPGIfNotExist(pod); err != nil { - glog.Errorf("Failed to handle Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Failed to handle Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) cc.queue.AddRateLimited(req) return true } diff --git a/pkg/controllers/podgroup/pg_controller_handler.go b/pkg/controllers/podgroup/pg_controller_handler.go index 799e9719f2..781559f4b3 100644 --- a/pkg/controllers/podgroup/pg_controller_handler.go +++ b/pkg/controllers/podgroup/pg_controller_handler.go @@ -17,11 +17,10 @@ limitations under the License. package podgroup import ( - "github.com/golang/glog" - "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/helpers" scheduling "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" @@ -35,7 +34,7 @@ type podRequest struct { func (cc *Controller) addPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Failed to convert %v to v1.Pod", obj) + klog.Errorf("Failed to convert %v to v1.Pod", obj) return } @@ -55,14 +54,14 @@ func (cc *Controller) updatePodAnnotations(pod *v1.Pod, pgName string) error { pod.Annotations[scheduling.GroupNameAnnotationKey] = pgName } else { if pod.Annotations[scheduling.GroupNameAnnotationKey] != pgName { - glog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name, + klog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name, scheduling.GroupNameAnnotationKey, pgName, pod.Annotations[scheduling.GroupNameAnnotationKey]) } return nil } if _, err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Update(pod); err != nil { - glog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err) + klog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } @@ -74,7 +73,7 @@ func (cc *Controller) createNormalPodPGIfNotExist(pod *v1.Pod) error { if _, err := cc.pgLister.PodGroups(pod.Namespace).Get(pgName); err != nil { if !apierrors.IsNotFound(err) { - glog.Errorf("Failed to get normal PodGroup for Pod <%s/%s>: %v", + klog.Errorf("Failed to get normal PodGroup for Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } @@ -92,7 +91,7 @@ func (cc *Controller) createNormalPodPGIfNotExist(pod *v1.Pod) error { } if _, err := cc.vcClient.SchedulingV1alpha2().PodGroups(pod.Namespace).Create(pg); err != nil { - glog.Errorf("Failed to create normal PodGroup for Pod <%s/%s>: %v", + klog.Errorf("Failed to create normal PodGroup for Pod <%s/%s>: %v", pod.Namespace, pod.Name, err) return err } diff --git a/pkg/controllers/queue/queue_controller.go b/pkg/controllers/queue/queue_controller.go index e0b9007706..bcf839e71a 100644 --- a/pkg/controllers/queue/queue_controller.go +++ b/pkg/controllers/queue/queue_controller.go @@ -18,21 +18,38 @@ package queue import ( "fmt" - "reflect" "sync" + "time" - "github.com/golang/glog" - "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" vcclientset "volcano.sh/volcano/pkg/client/clientset/versioned" + versionedscheme "volcano.sh/volcano/pkg/client/clientset/versioned/scheme" informerfactory "volcano.sh/volcano/pkg/client/informers/externalversions" + busv1alpha1informer "volcano.sh/volcano/pkg/client/informers/externalversions/bus/v1alpha1" schedulinginformer "volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1alpha2" + busv1alpha1lister "volcano.sh/volcano/pkg/client/listers/bus/v1alpha1" schedulinglister "volcano.sh/volcano/pkg/client/listers/scheduling/v1alpha2" + queuestate "volcano.sh/volcano/pkg/controllers/queue/state" +) + +const ( + // maxRetries is the number of times a queue or command will be retried before it is dropped out of the queue. + // With the current rate-limiter in use (5ms*2^(maxRetries-1)) the following numbers represent the times + // a queue or command is going to be requeued: + // 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s + maxRetries = 15 ) // Controller manages queue status. @@ -52,12 +69,24 @@ type Controller struct { pgLister schedulinglister.PodGroupLister pgSynced cache.InformerSynced + cmdInformer busv1alpha1informer.CommandInformer + cmdLister busv1alpha1lister.CommandLister + cmdSynced cache.InformerSynced + // queues that need to be updated. - queue workqueue.RateLimitingInterface + queue workqueue.RateLimitingInterface + commandQueue workqueue.RateLimitingInterface pgMutex sync.RWMutex // queue name -> podgroup namespace/name podGroups map[string]map[string]struct{} + + syncHandler func(req *schedulingv1alpha2.QueueRequest) error + syncCommandHandler func(cmd *busv1alpha1.Command) error + + enqueueQueue func(req *schedulingv1alpha2.QueueRequest) + + recorder record.EventRecorder } // NewQueueController creates a QueueController @@ -68,6 +97,11 @@ func NewQueueController( factory := informerfactory.NewSharedInformerFactory(vcClient, 0) queueInformer := factory.Scheduling().V1alpha2().Queues() pgInformer := factory.Scheduling().V1alpha2().PodGroups() + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + c := &Controller{ kubeClient: kubeClient, vcClient: vcClient, @@ -81,12 +115,17 @@ func NewQueueController( pgLister: pgInformer.Lister(), pgSynced: pgInformer.Informer().HasSynced, - queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + commandQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + podGroups: make(map[string]map[string]struct{}), + + recorder: eventBroadcaster.NewRecorder(versionedscheme.Scheme, v1.EventSource{Component: "vc-controllers"}), } queueInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addQueue, + UpdateFunc: c.updateQueue, DeleteFunc: c.deleteQueue, }) @@ -96,22 +135,58 @@ func NewQueueController( DeleteFunc: c.deletePodGroup, }) + c.cmdInformer = informerfactory.NewSharedInformerFactory(c.vcClient, 0).Bus().V1alpha1().Commands() + c.cmdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + switch obj.(type) { + case *busv1alpha1.Command: + cmd := obj.(*busv1alpha1.Command) + return IsQueueReference(cmd.TargetObject) + default: + return false + } + }, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: c.addCommand, + }, + }) + c.cmdLister = c.cmdInformer.Lister() + c.cmdSynced = c.cmdInformer.Informer().HasSynced + + queuestate.SyncQueue = c.syncQueue + queuestate.OpenQueue = c.openQueue + queuestate.CloseQueue = c.closeQueue + + c.syncHandler = c.handleQueue + c.syncCommandHandler = c.handleCommand + + c.enqueueQueue = c.enqueue + return c } // Run starts QueueController func (c *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + defer c.commandQueue.ShutDown() + + klog.Infof("Starting queue controller.") + defer klog.Infof("Shutting down queue controller.") go c.queueInformer.Informer().Run(stopCh) go c.pgInformer.Informer().Run(stopCh) + go c.cmdInformer.Informer().Run(stopCh) - if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced) { - glog.Errorf("unable to sync caches for queue controller") + if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced, c.cmdSynced) { + klog.Errorf("unable to sync caches for queue controller.") return } go wait.Until(c.worker, 0, stopCh) - glog.Infof("QueueController is running ...... ") + go wait.Until(c.commandWorker, 0, stopCh) + + <-stopCh } // worker runs a worker thread that just dequeues items, processes them, and @@ -124,168 +199,134 @@ func (c *Controller) worker() { } func (c *Controller) processNextWorkItem() bool { - eKey, quit := c.queue.Get() - if quit { + obj, shutdown := c.queue.Get() + if shutdown { return false } - defer c.queue.Done(eKey) + defer c.queue.Done(obj) - if err := c.syncQueue(eKey.(string)); err != nil { - glog.V(2).Infof("Error syncing queues %q, retrying. Error: %v", eKey, err) - c.queue.AddRateLimited(eKey) + req, ok := obj.(*schedulingv1alpha2.QueueRequest) + if !ok { + klog.Errorf("%v is not a valid queue request struct.", obj) return true } - c.queue.Forget(eKey) - return true -} - -func (c *Controller) getPodGroups(key string) ([]string, error) { - c.pgMutex.RLock() - defer c.pgMutex.RUnlock() + err := c.syncHandler(req) + c.handleQueueErr(err, obj) - if c.podGroups[key] == nil { - return nil, fmt.Errorf("queue %s has not been seen or deleted", key) - } - podGroups := make([]string, 0, len(c.podGroups[key])) - for pgKey := range c.podGroups[key] { - podGroups = append(podGroups, pgKey) - } - - return podGroups, nil + return true } -func (c *Controller) syncQueue(key string) error { - glog.V(4).Infof("Begin sync queue %s", key) +func (c *Controller) handleQueue(req *schedulingv1alpha2.QueueRequest) error { + startTime := time.Now() + defer func() { + klog.V(4).Infof("Finished syncing queue %s (%v).", req.Name, time.Since(startTime)) + }() - podGroups, err := c.getPodGroups(key) + queue, err := c.queueLister.Get(req.Name) if err != nil { - return err - } - - queueStatus := schedulingv1alpha2.QueueStatus{} - - for _, pgKey := range podGroups { - // Ignore error here, tt can not occur. - ns, name, _ := cache.SplitMetaNamespaceKey(pgKey) - - // TODO: check NotFound error and sync local cache. - pg, err := c.pgLister.PodGroups(ns).Get(name) - if err != nil { - return err + if apierrors.IsNotFound(err) { + klog.V(4).Infof("Queue %s has been deleted.", req.Name) + return nil } - switch pg.Status.Phase { - case schedulingv1alpha2.PodGroupPending: - queueStatus.Pending++ - case schedulingv1alpha2.PodGroupRunning: - queueStatus.Running++ - case schedulingv1alpha2.PodGroupUnknown: - queueStatus.Unknown++ - case schedulingv1alpha2.PodGroupInqueue: - queueStatus.Inqueue++ - } + return fmt.Errorf("get queue %s failed for %v", req.Name, err) } - queue, err := c.queueLister.Get(key) - if err != nil { - if errors.IsNotFound(err) { - glog.V(2).Infof("queue %s has been deleted", key) - return nil - } - // TODO: do not retry to syncQueue for this error - return err + queueState := queuestate.NewState(queue) + if queueState == nil { + return fmt.Errorf("queue %s state %s is invalid", queue.Name, queue.Status.State) } - // ignore update when status does not change - if reflect.DeepEqual(queueStatus, queue.Status) { - return nil + if err := queueState.Execute(req.Action); err != nil { + return fmt.Errorf("sync queue %s failed for %v, event is %v, action is %s", + req.Name, err, req.Event, req.Action) } - newQueue := queue.DeepCopy() - newQueue.Status = queueStatus + return nil +} + +func (c *Controller) handleQueueErr(err error, obj interface{}) { + if err == nil { + c.queue.Forget(obj) + return + } - if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { - glog.Errorf("Failed to update status of Queue %s: %v", newQueue.Name, err) - return err + if c.queue.NumRequeues(obj) < maxRetries { + klog.V(4).Infof("Error syncing queue request %v for %v.", obj, err) + c.queue.AddRateLimited(obj) + return } - return nil + req, _ := obj.(*schedulingv1alpha2.QueueRequest) + c.recordEventsForQueue(req.Name, v1.EventTypeWarning, string(req.Action), + fmt.Sprintf("%v queue failed for %v", req.Action, err)) + klog.V(2).Infof("Dropping queue request %v out of the queue for %v.", obj, err) + c.queue.Forget(obj) } -func (c *Controller) addQueue(obj interface{}) { - queue := obj.(*schedulingv1alpha2.Queue) - c.queue.Add(queue.Name) +func (c *Controller) commandWorker() { + for c.processNextCommand() { + } } -func (c *Controller) deleteQueue(obj interface{}) { - queue, ok := obj.(*schedulingv1alpha2.Queue) +func (c *Controller) processNextCommand() bool { + obj, shutdown := c.commandQueue.Get() + if shutdown { + return false + } + defer c.commandQueue.Done(obj) + + cmd, ok := obj.(*busv1alpha1.Command) if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) - return - } - queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue) - if !ok { - glog.Errorf("Tombstone contained object that is not a Queue: %#v", obj) - return - } + klog.Errorf("%v is not a valid Command struct.", obj) + return true } - c.pgMutex.Lock() - defer c.pgMutex.Unlock() - delete(c.podGroups, queue.Name) + err := c.syncCommandHandler(cmd) + c.handleCommandErr(err, obj) + + return true } -func (c *Controller) addPodGroup(obj interface{}) { - pg := obj.(*schedulingv1alpha2.PodGroup) - key, _ := cache.MetaNamespaceKeyFunc(obj) +func (c *Controller) handleCommand(cmd *busv1alpha1.Command) error { + startTime := time.Now() + defer func() { + klog.V(4).Infof("Finished syncing command %s/%s (%v).", cmd.Namespace, cmd.Name, time.Since(startTime)) + }() - c.pgMutex.Lock() - defer c.pgMutex.Unlock() + err := c.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(cmd.Name, nil) + if err != nil { + if true == apierrors.IsNotFound(err) { + return nil + } - if c.podGroups[pg.Spec.Queue] == nil { - c.podGroups[pg.Spec.Queue] = make(map[string]struct{}) + return fmt.Errorf("failed to delete command <%s/%s> for %v", cmd.Namespace, cmd.Name, err) } - c.podGroups[pg.Spec.Queue][key] = struct{}{} - // enqueue - c.queue.Add(pg.Spec.Queue) -} + req := &schedulingv1alpha2.QueueRequest{ + Name: cmd.TargetObject.Name, + Event: schedulingv1alpha2.QueueCommandIssuedEvent, + Action: schedulingv1alpha2.QueueAction(cmd.Action), + } -func (c *Controller) updatePodGroup(old, new interface{}) { - oldPG := old.(*schedulingv1alpha2.PodGroup) - newPG := new.(*schedulingv1alpha2.PodGroup) + c.enqueueQueue(req) - // Note: we have no use case update PodGroup.Spec.Queue - // So do not consider it here. - if oldPG.Status.Phase != newPG.Status.Phase { - c.queue.Add(newPG.Spec.Queue) - } + return nil } -func (c *Controller) deletePodGroup(obj interface{}) { - pg, ok := obj.(*schedulingv1alpha2.PodGroup) - if !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - glog.Errorf("Couldn't get object from tombstone %#v", obj) - return - } - pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup) - if !ok { - glog.Errorf("Tombstone contained object that is not a PodGroup: %#v", obj) - return - } +func (c *Controller) handleCommandErr(err error, obj interface{}) { + if err == nil { + c.commandQueue.Forget(obj) + return } - key, _ := cache.MetaNamespaceKeyFunc(obj) - - c.pgMutex.Lock() - defer c.pgMutex.Unlock() - - delete(c.podGroups[pg.Spec.Queue], key) + if c.commandQueue.NumRequeues(obj) < maxRetries { + klog.V(4).Infof("Error syncing command %v for %v.", obj, err) + c.commandQueue.AddRateLimited(obj) + return + } - c.queue.Add(pg.Spec.Queue) + klog.V(2).Infof("Dropping command %v out of the queue for %v.", obj, err) + c.commandQueue.Forget(obj) } diff --git a/pkg/controllers/queue/queue_controller_action.go b/pkg/controllers/queue/queue_controller_action.go new file mode 100644 index 0000000000..c405f58f48 --- /dev/null +++ b/pkg/controllers/queue/queue_controller_action.go @@ -0,0 +1,167 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "fmt" + "reflect" + + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + "volcano.sh/volcano/pkg/controllers/queue/state" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + "k8s.io/klog" +) + +func (c *Controller) syncQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to sync queue %s.", queue.Name) + + podGroups := c.getPodGroups(queue.Name) + queueStatus := schedulingv1alpha2.QueueStatus{} + + for _, pgKey := range podGroups { + // Ignore error here, tt can not occur. + ns, name, _ := cache.SplitMetaNamespaceKey(pgKey) + + // TODO: check NotFound error and sync local cache. + pg, err := c.pgLister.PodGroups(ns).Get(name) + if err != nil { + return err + } + + switch pg.Status.Phase { + case schedulingv1alpha2.PodGroupPending: + queueStatus.Pending++ + case schedulingv1alpha2.PodGroupRunning: + queueStatus.Running++ + case schedulingv1alpha2.PodGroupUnknown: + queueStatus.Unknown++ + case schedulingv1alpha2.PodGroupInqueue: + queueStatus.Inqueue++ + } + } + + if updateStateFn != nil { + updateStateFn(&queueStatus, podGroups) + } else { + queueStatus.State = queue.Status.State + } + + // ignore update when status does not change + if reflect.DeepEqual(queueStatus, queue.Status) { + return nil + } + + newQueue := queue.DeepCopy() + newQueue.Status = queueStatus + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + klog.Errorf("Failed to update status of Queue %s: %v.", newQueue.Name, err) + return err + } + + return nil +} + +func (c *Controller) openQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to open queue %s.", queue.Name) + + newQueue := queue.DeepCopy() + newQueue.Spec.State = schedulingv1alpha2.QueueStateOpen + + if queue.Spec.State != newQueue.Spec.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Open queue failed for %v", err)) + return err + } + + c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Open queue succeed")) + } else { + return nil + } + + q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + newQueue = q.DeepCopy() + if updateStateFn != nil { + updateStateFn(&newQueue.Status, nil) + } else { + return fmt.Errorf("internal error, update state function should be provided") + } + + if queue.Status.State != newQueue.Status.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction), + fmt.Sprintf("Update queue status from %s to %s failed for %v", + queue.Status.State, newQueue.Status.State, err)) + return err + } + } + + return nil +} + +func (c *Controller) closeQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error { + klog.V(4).Infof("Begin to close queue %s.", queue.Name) + + newQueue := queue.DeepCopy() + newQueue.Spec.State = schedulingv1alpha2.QueueStateClosed + + if queue.Spec.State != newQueue.Spec.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Close queue failed for %v", err)) + return err + } + + c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Close queue succeed")) + } else { + return nil + } + + q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + newQueue = q.DeepCopy() + podGroups := c.getPodGroups(newQueue.Name) + if updateStateFn != nil { + updateStateFn(&newQueue.Status, podGroups) + } else { + return fmt.Errorf("internal error, update state function should be provided") + } + + if queue.Status.State != newQueue.Status.State { + if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil { + c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction), + fmt.Sprintf("Update queue status from %s to %s failed for %v", + queue.Status.State, newQueue.Status.State, err)) + return err + } + } + + return nil +} diff --git a/pkg/controllers/queue/queue_controller_handler.go b/pkg/controllers/queue/queue_controller_handler.go new file mode 100644 index 0000000000..7165f8a215 --- /dev/null +++ b/pkg/controllers/queue/queue_controller_handler.go @@ -0,0 +1,186 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1" + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + "k8s.io/client-go/tools/cache" + + "k8s.io/klog" +) + +func (c *Controller) enqueue(req *schedulingv1alpha2.QueueRequest) { + c.queue.Add(req) +} + +func (c *Controller) addQueue(obj interface{}) { + queue := obj.(*schedulingv1alpha2.Queue) + + req := &schedulingv1alpha2.QueueRequest{ + Name: queue.Name, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) deleteQueue(obj interface{}) { + queue, ok := obj.(*schedulingv1alpha2.Queue) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v.", obj) + return + } + queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Tombstone contained object that is not a Queue: %#v.", obj) + return + } + } + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + delete(c.podGroups, queue.Name) +} + +func (c *Controller) updateQueue(old, new interface{}) { + oldQueue, ok := old.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Can not covert old object %v to queues.scheduling.sigs.dev.", old) + return + } + + newQueue, ok := new.(*schedulingv1alpha2.Queue) + if !ok { + klog.Errorf("Can not covert new object %v to queues.scheduling.sigs.dev.", old) + return + } + + if oldQueue.ResourceVersion == newQueue.ResourceVersion { + return + } + + c.addQueue(newQueue) + + return +} + +func (c *Controller) addPodGroup(obj interface{}) { + pg := obj.(*schedulingv1alpha2.PodGroup) + key, _ := cache.MetaNamespaceKeyFunc(obj) + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + + if c.podGroups[pg.Spec.Queue] == nil { + c.podGroups[pg.Spec.Queue] = make(map[string]struct{}) + } + c.podGroups[pg.Spec.Queue][key] = struct{}{} + + req := &schedulingv1alpha2.QueueRequest{ + Name: pg.Spec.Queue, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) updatePodGroup(old, new interface{}) { + oldPG := old.(*schedulingv1alpha2.PodGroup) + newPG := new.(*schedulingv1alpha2.PodGroup) + + // Note: we have no use case update PodGroup.Spec.Queue + // So do not consider it here. + if oldPG.Status.Phase != newPG.Status.Phase { + c.addPodGroup(newPG) + } +} + +func (c *Controller) deletePodGroup(obj interface{}) { + pg, ok := obj.(*schedulingv1alpha2.PodGroup) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v.", obj) + return + } + pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup) + if !ok { + klog.Errorf("Tombstone contained object that is not a PodGroup: %#v.", obj) + return + } + } + + key, _ := cache.MetaNamespaceKeyFunc(obj) + + c.pgMutex.Lock() + defer c.pgMutex.Unlock() + + delete(c.podGroups[pg.Spec.Queue], key) + + req := &schedulingv1alpha2.QueueRequest{ + Name: pg.Spec.Queue, + + Event: schedulingv1alpha2.QueueOutOfSyncEvent, + Action: schedulingv1alpha2.SyncQueueAction, + } + + c.enqueue(req) +} + +func (c *Controller) addCommand(obj interface{}) { + cmd, ok := obj.(*busv1alpha1.Command) + if !ok { + klog.Errorf("Obj %v is not command.", obj) + return + } + + c.commandQueue.Add(cmd) +} + +func (c *Controller) getPodGroups(key string) []string { + c.pgMutex.RLock() + defer c.pgMutex.RUnlock() + + if c.podGroups[key] == nil { + return nil + } + podGroups := make([]string, 0, len(c.podGroups[key])) + for pgKey := range c.podGroups[key] { + podGroups = append(podGroups, pgKey) + } + + return podGroups +} + +func (c *Controller) recordEventsForQueue(name, eventType, reason, message string) { + queue, err := c.queueLister.Get(name) + if err != nil { + klog.Errorf("Get queue %s failed for %v.", name, err) + return + } + + c.recorder.Event(queue, eventType, reason, message) + return +} diff --git a/pkg/controllers/queue/queue_controller_test.go b/pkg/controllers/queue/queue_controller_test.go index e1e7806a61..9c4a068456 100644 --- a/pkg/controllers/queue/queue_controller_test.go +++ b/pkg/controllers/queue/queue_controller_test.go @@ -271,7 +271,7 @@ func TestSyncQueue(t *testing.T) { c.queueInformer.Informer().GetIndexer().Add(testcase.queue) c.vcClient.SchedulingV1alpha2().Queues().Create(testcase.queue) - err := c.syncQueue(testcase.queue.Name) + err := c.syncQueue(testcase.queue, nil) item, _ := c.vcClient.SchedulingV1alpha2().Queues().Get(testcase.queue.Name, metav1.GetOptions{}) if err != nil && testcase.ExpectValue != item.Status.Pending { t.Errorf("case %d (%s): expected: %v, got %v ", i, testcase.Name, testcase.ExpectValue, c.queue.Len()) diff --git a/pkg/controllers/queue/queue_controller_util.go b/pkg/controllers/queue/queue_controller_util.go new file mode 100644 index 0000000000..28ce7a261b --- /dev/null +++ b/pkg/controllers/queue/queue_controller_util.go @@ -0,0 +1,40 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// IsQueueReference return if ownerReference is Queue Kind +func IsQueueReference(ref *metav1.OwnerReference) bool { + if ref == nil { + return false + } + + if ref.APIVersion != schedulingv1alpha2.SchemeGroupVersion.String() { + return false + } + + if ref.Kind != "Queue" { + return false + } + + return true +} diff --git a/pkg/controllers/queue/state/closed.go b/pkg/controllers/queue/state/closed.go new file mode 100644 index 0000000000..91e402d87c --- /dev/null +++ b/pkg/controllers/queue/state/closed.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type closedState struct { + queue *v1alpha2.Queue +} + +func (cs *closedState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateClosed + return + }) + default: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := cs.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + status.State = v1alpha2.QueueStateClosed + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/closing.go b/pkg/controllers/queue/state/closing.go new file mode 100644 index 0000000000..afcbdd6b37 --- /dev/null +++ b/pkg/controllers/queue/state/closing.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type closingState struct { + queue *v1alpha2.Queue +} + +func (cs *closingState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(cs.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := cs.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + + status.State = v1alpha2.QueueStateClosing + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/factory.go b/pkg/controllers/queue/state/factory.go new file mode 100644 index 0000000000..07212a075e --- /dev/null +++ b/pkg/controllers/queue/state/factory.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +// State interface +type State interface { + // Execute executes the actions based on current state. + Execute(action v1alpha2.QueueAction) error +} + +// UpdateQueueStatusFn updates the queue status +type UpdateQueueStatusFn func(status *v1alpha2.QueueStatus, podGroupList []string) + +// QueueActionFn will open, close or sync queue. +type QueueActionFn func(queue *v1alpha2.Queue, fn UpdateQueueStatusFn) error + +var ( + // SyncQueue will sync queue status. + SyncQueue QueueActionFn + // OpenQueue will set state of queue to open + OpenQueue QueueActionFn + // CloseQueue will set state of queue to close + CloseQueue QueueActionFn +) + +// NewState gets the state from queue status +func NewState(queue *v1alpha2.Queue) State { + switch queue.Status.State { + case "", v1alpha2.QueueStateOpen: + return &openState{queue: queue} + case v1alpha2.QueueStateClosed: + return &closedState{queue: queue} + case v1alpha2.QueueStateClosing: + return &closingState{queue: queue} + case v1alpha2.QueueStateUnknown: + return &unknownState{queue: queue} + } + + return nil +} diff --git a/pkg/controllers/queue/state/open.go b/pkg/controllers/queue/state/open.go new file mode 100644 index 0000000000..4d7ad898f9 --- /dev/null +++ b/pkg/controllers/queue/state/open.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type openState struct { + queue *v1alpha2.Queue +} + +func (os *openState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return SyncQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return CloseQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(os.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := os.queue.Spec.State + if len(specState) == 0 || specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } + + return nil +} diff --git a/pkg/controllers/queue/state/unknown.go b/pkg/controllers/queue/state/unknown.go new file mode 100644 index 0000000000..89bf6bebd9 --- /dev/null +++ b/pkg/controllers/queue/state/unknown.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2" +) + +type unknownState struct { + queue *v1alpha2.Queue +} + +func (us *unknownState) Execute(action v1alpha2.QueueAction) error { + switch action { + case v1alpha2.OpenQueueAction: + return OpenQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + status.State = v1alpha2.QueueStateOpen + return + }) + case v1alpha2.CloseQueueAction: + return CloseQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + }) + default: + return SyncQueue(us.queue, func(status *v1alpha2.QueueStatus, podGroupList []string) { + specState := us.queue.Spec.State + if specState == v1alpha2.QueueStateOpen { + status.State = v1alpha2.QueueStateOpen + return + } + + if specState == v1alpha2.QueueStateClosed { + if len(podGroupList) == 0 { + status.State = v1alpha2.QueueStateClosed + return + } + status.State = v1alpha2.QueueStateClosing + + return + } + + status.State = v1alpha2.QueueStateUnknown + return + }) + } +} diff --git a/pkg/scheduler/OWNERS b/pkg/scheduler/OWNERS index 896e4cc053..caf70c72d5 100644 --- a/pkg/scheduler/OWNERS +++ b/pkg/scheduler/OWNERS @@ -2,9 +2,7 @@ approvers: - k82cn - animeshsingh - hex108 - - hzxuzhonghu reviewers: - k82cn - animeshsingh - - hex108 - - hzxuzhonghu + - hex108 \ No newline at end of file diff --git a/pkg/scheduler/actions/allocate/allocate.go b/pkg/scheduler/actions/allocate/allocate.go index 80a1ee13e2..3e2fe6a4ec 100644 --- a/pkg/scheduler/actions/allocate/allocate.go +++ b/pkg/scheduler/actions/allocate/allocate.go @@ -17,7 +17,7 @@ limitations under the License. package allocate import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -40,8 +40,8 @@ func (alloc *allocateAction) Name() string { func (alloc *allocateAction) Initialize() {} func (alloc *allocateAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Allocate ...") - defer glog.V(3).Infof("Leaving Allocate ...") + klog.V(3).Infof("Enter Allocate ...") + defer klog.V(3).Infof("Leaving Allocate ...") // the allocation for pod may have many stages // 1. pick a namespace named N (using ssn.NamespaceOrderFn) @@ -62,12 +62,12 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if _, found := ssn.Queues[job.Queue]; !found { - glog.Warningf("Skip adding Job <%s/%s> because its queue %s is not found", + klog.Warningf("Skip adding Job <%s/%s> because its queue %s is not found", job.Namespace, job.Name, job.Queue) continue } @@ -87,11 +87,11 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { queueMap[job.Queue] = jobs } - glog.V(4).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) + klog.V(4).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) jobs.Push(job) } - glog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap)) + klog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap)) pendingTasks := map[api.JobID]*util.PriorityQueue{} @@ -99,13 +99,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error { // Check for Resource Predicate - // TODO: We could not allocate resource to task from both node.Idle and node.Releasing now, - // after it is done, we could change the following compare to: - // clonedNode := node.Idle.Clone() - // if !task.InitResreq.LessEqual(clonedNode.Add(node.Releasing)) { - // ... - // } - if !task.InitResreq.LessEqual(node.Idle) && !task.InitResreq.LessEqual(node.Releasing) { + if !task.InitResreq.LessEqual(node.FutureIdle()) { return api.NewFitError(task, node, api.NodeResourceFitFailed) } @@ -135,7 +129,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for queueId := range queueInNamespace { currentQueue := ssn.Queues[queueId] if ssn.Overused(currentQueue) { - glog.V(3).Infof("Namespace <%s> Queue <%s> is overused, ignore it.", namespace, currentQueue.Name) + klog.V(3).Infof("Namespace <%s> Queue <%s> is overused, ignore it.", namespace, currentQueue.Name) delete(queueInNamespace, queueId) continue } @@ -146,15 +140,15 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { } if queue == nil { - glog.V(3).Infof("Namespace <%s> have no queue, skip it", namespace) + klog.V(3).Infof("Namespace <%s> have no queue, skip it", namespace) continue } - glog.V(3).Infof("Try to allocate resource to Jobs in Namespace <%s> Queue <%v>", namespace, queue.Name) + klog.V(3).Infof("Try to allocate resource to Jobs in Namespace <%s> Queue <%v>", namespace, queue.Name) jobs, found := queueInNamespace[queue.UID] if !found || jobs.Empty() { - glog.V(4).Infof("Can not find jobs for queue %s.", queue.Name) + klog.V(4).Infof("Can not find jobs for queue %s.", queue.Name) continue } @@ -164,7 +158,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for _, task := range job.TaskStatusIndex[api.Pending] { // Skip BestEffort task in 'allocate' action. if task.Resreq.IsEmpty() { - glog.V(4).Infof("Task <%v/%v> is BestEffort task, skip it.", + klog.V(4).Infof("Task <%v/%v> is BestEffort task, skip it.", task.Namespace, task.Name) continue } @@ -175,7 +169,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { } tasks := pendingTasks[job.UID] - glog.V(3).Infof("Try to allocate resource to %d tasks of Job <%v/%v>", + klog.V(3).Infof("Try to allocate resource to %d tasks of Job <%v/%v>", tasks.Len(), job.Namespace, job.Name) stmt := ssn.Statement() @@ -183,7 +177,7 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { for !tasks.Empty() { task := tasks.Pop().(*api.TaskInfo) - glog.V(3).Infof("There are <%d> nodes for Job <%v/%v>", + klog.V(3).Infof("There are <%d> nodes for Job <%v/%v>", len(ssn.Nodes), job.Namespace, job.Name) //any task that doesn't fit will be the last processed @@ -205,25 +199,25 @@ func (alloc *allocateAction) Execute(ssn *framework.Session) { node := util.SelectBestNode(nodeScores) // Allocate idle resource to the task. if task.InitResreq.LessEqual(node.Idle) { - glog.V(3).Infof("Binding Task <%v/%v> to node <%v>", + klog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) if err := stmt.Allocate(task, node.Name); err != nil { - glog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v", + klog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v", task.UID, node.Name, ssn.UID, err) } } else { //store information about missing resources job.NodesFitDelta[node.Name] = node.Idle.Clone() job.NodesFitDelta[node.Name].FitDelta(task.InitResreq) - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s> with limited resources", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s> with limited resources", task.Namespace, task.Name, node.Name) // Allocate releasing resource to the task if any. - if task.InitResreq.LessEqual(node.Releasing) { - glog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>", + if task.InitResreq.LessEqual(node.FutureIdle()) { + klog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>", task.Namespace, task.Name, node.Name, task.InitResreq, node.Releasing) if err := stmt.Pipeline(task, node.Name); err != nil { - glog.Errorf("Failed to pipeline Task %v on %v", + klog.Errorf("Failed to pipeline Task %v on %v", task.UID, node.Name) } } diff --git a/pkg/scheduler/actions/allocate/allocate_test.go b/pkg/scheduler/actions/allocate/allocate_test.go index 63c173ef55..0a74be5e5e 100644 --- a/pkg/scheduler/actions/allocate/allocate_test.go +++ b/pkg/scheduler/actions/allocate/allocate_test.go @@ -202,7 +202,7 @@ func TestAllocate(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) allocate.Execute(ssn) diff --git a/pkg/scheduler/actions/backfill/backfill.go b/pkg/scheduler/actions/backfill/backfill.go index c4e96364b9..20fdfdf47e 100644 --- a/pkg/scheduler/actions/backfill/backfill.go +++ b/pkg/scheduler/actions/backfill/backfill.go @@ -17,7 +17,7 @@ limitations under the License. package backfill import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -39,8 +39,8 @@ func (alloc *backfillAction) Name() string { func (alloc *backfillAction) Initialize() {} func (alloc *backfillAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Backfill ...") - defer glog.V(3).Infof("Leaving Backfill ...") + klog.V(3).Infof("Enter Backfill ...") + defer klog.V(3).Infof("Leaving Backfill ...") // TODO (k82cn): When backfill, it's also need to balance between Queues. for _, job := range ssn.Jobs { @@ -48,7 +48,7 @@ func (alloc *backfillAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip backfill, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip backfill, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } @@ -63,15 +63,15 @@ func (alloc *backfillAction) Execute(ssn *framework.Session) { // TODO (k82cn): predicates did not consider pod number for now, there'll // be ping-pong case here. if err := ssn.PredicateFn(task, node); err != nil { - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", task.Namespace, task.Name, node.Name, err) fe.SetNodeError(node.Name, err) continue } - glog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) + klog.V(3).Infof("Binding Task <%v/%v> to node <%v>", task.Namespace, task.Name, node.Name) if err := ssn.Allocate(task, node.Name); err != nil { - glog.Errorf("Failed to bind Task %v on %v in Session %v", task.UID, node.Name, ssn.UID) + klog.Errorf("Failed to bind Task %v on %v in Session %v", task.UID, node.Name, ssn.UID) fe.SetNodeError(node.Name, err) continue } diff --git a/pkg/scheduler/actions/enqueue/enqueue.go b/pkg/scheduler/actions/enqueue/enqueue.go index 2c01e7ec37..8008f5ba47 100644 --- a/pkg/scheduler/actions/enqueue/enqueue.go +++ b/pkg/scheduler/actions/enqueue/enqueue.go @@ -17,7 +17,7 @@ limitations under the License. package enqueue import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -25,6 +25,18 @@ import ( "volcano.sh/volcano/pkg/scheduler/util" ) +const ( + // overCommitFactor is resource overCommit factor for enqueue action + // It determines the number of `pending` pods that the scheduler will tolerate + // when the resources of the cluster is insufficient + overCommitFactor = "overcommit-factor" +) + +var ( + // defaultOverCommitFactor defines the default overCommit resource factor for enqueue action + defaultOverCommitFactor = 1.2 +) + type enqueueAction struct { ssn *framework.Session } @@ -40,8 +52,8 @@ func (enqueue *enqueueAction) Name() string { func (enqueue *enqueueAction) Initialize() {} func (enqueue *enqueueAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Enqueue ...") - defer glog.V(3).Infof("Leaving Enqueue ...") + klog.V(3).Infof("Enter Enqueue ...") + defer klog.V(3).Infof("Leaving Enqueue ...") queues := util.NewPriorityQueue(ssn.QueueOrderFn) queueMap := map[api.QueueID]*api.QueueInfo{} @@ -50,12 +62,12 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { for _, job := range ssn.Jobs { if queue, found := ssn.Queues[job.Queue]; !found { - glog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", + klog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", job.Queue, job.Namespace, job.Name) continue } else { if _, existed := queueMap[queue.UID]; !existed { - glog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queueMap[queue.UID] = queue @@ -67,17 +79,17 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { if _, found := jobsMap[job.Queue]; !found { jobsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn) } - glog.V(3).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) + klog.V(3).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue) jobsMap[job.Queue].Push(job) } } - glog.V(3).Infof("Try to enqueue PodGroup to %d Queues", len(jobsMap)) + klog.V(3).Infof("Try to enqueue PodGroup to %d Queues", len(jobsMap)) emptyRes := api.EmptyResource() nodesIdleRes := api.EmptyResource() for _, node := range ssn.Nodes { - nodesIdleRes.Add(node.Allocatable.Clone().Multi(1.2).Sub(node.Used)) + nodesIdleRes.Add(node.Allocatable.Clone().Multi(enqueue.getOverCommitFactor(ssn)).Sub(node.Used)) } for { @@ -86,7 +98,7 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { } if nodesIdleRes.Less(emptyRes) { - glog.V(3).Infof("Node idle resource is overused, ignore it.") + klog.V(3).Infof("Node idle resource is overused, ignore it.") break } @@ -122,3 +134,13 @@ func (enqueue *enqueueAction) Execute(ssn *framework.Session) { } func (enqueue *enqueueAction) UnInitialize() {} + +func (enqueue *enqueueAction) getOverCommitFactor(ssn *framework.Session) float64 { + factor := defaultOverCommitFactor + arg := framework.GetArgOfActionFromConf(ssn.Configurations, enqueue.Name()) + if arg != nil { + arg.GetFloat64(&factor, overCommitFactor) + } + + return factor +} diff --git a/pkg/scheduler/actions/enqueue/enqueue_test.go b/pkg/scheduler/actions/enqueue/enqueue_test.go new file mode 100644 index 0000000000..5c3787e192 --- /dev/null +++ b/pkg/scheduler/actions/enqueue/enqueue_test.go @@ -0,0 +1,53 @@ +package enqueue + +import ( + "testing" + + "volcano.sh/volcano/pkg/scheduler/conf" + "volcano.sh/volcano/pkg/scheduler/framework" +) + +func TestGetOverCommitFactor(t *testing.T) { + cases := []struct { + name string + ssn *framework.Session + expectedValue float64 + }{ + { + name: "arguments of action not exist", + ssn: &framework.Session{ + Configurations: []conf.Configuration{ + { + Name: "allocate", + Arguments: map[string]string{ + "placeholder": "placeholder", + }, + }, + }, + }, + expectedValue: 1.2, + }, + { + name: "arguments of action exist", + ssn: &framework.Session{ + Configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + overCommitFactor: "2", + }, + }, + }, + }, + expectedValue: 2, + }, + } + + enqueue := New() + for index, c := range cases { + factor := enqueue.getOverCommitFactor(c.ssn) + if factor != c.expectedValue { + t.Errorf("index %d, case %s, expected %v, but got %v", index, c.name, c.expectedValue, factor) + } + } +} diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 923d5f0574..e61ee6b48f 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -19,7 +19,7 @@ package preempt import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -43,8 +43,8 @@ func (alloc *preemptAction) Name() string { func (alloc *preemptAction) Initialize() {} func (alloc *preemptAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Preempt ...") - defer glog.V(3).Infof("Leaving Preempt ...") + klog.V(3).Infof("Enter Preempt ...") + defer klog.V(3).Infof("Leaving Preempt ...") preemptorsMap := map[api.QueueID]*util.PriorityQueue{} preemptorTasks := map[api.JobID]*util.PriorityQueue{} @@ -57,19 +57,19 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if queue, found := ssn.Queues[job.Queue]; !found { continue } else if _, existed := queues[queue.UID]; !existed { - glog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queues[queue.UID] = queue } - if len(job.TaskStatusIndex[api.Pending]) != 0 { + if len(job.TaskStatusIndex[api.Pending]) != 0 && !ssn.JobPipelined(job) { if _, found := preemptorsMap[job.Queue]; !found { preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn) } @@ -89,7 +89,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { // If no preemptors, no preemption. if preemptors == nil || preemptors.Empty() { - glog.V(4).Infof("No preemptors in Queue <%s>, break.", queue.Name) + klog.V(4).Infof("No preemptors in Queue <%s>, break.", queue.Name) break } @@ -98,16 +98,21 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { stmt := ssn.Statement() assigned := false for { + // If job is pipelined, then stop preempting. + if ssn.JobPipelined(preemptorJob) { + break + } + // If not preemptor tasks, next job. if preemptorTasks[preemptorJob.UID].Empty() { - glog.V(3).Infof("No preemptor task in job <%s/%s>.", + klog.V(3).Infof("No preemptor task in job <%s/%s>.", preemptorJob.Namespace, preemptorJob.Name) break } preemptor := preemptorTasks[preemptorJob.UID].Pop().(*api.TaskInfo) - if preempted, _ := preempt(ssn, stmt, preemptor, ssn.Nodes, func(task *api.TaskInfo) bool { + if preempted, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool { // Ignore non running task. if task.Status != api.Running { return false @@ -122,16 +127,12 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { }); preempted { assigned = true } - - // If job is not pipelined, keep preempting - if ssn.JobPipelined(preemptorJob) { - stmt.Commit() - break - } } - // If job is not pipelined after try all tasks, next job. - if !ssn.JobPipelined(preemptorJob) { + // Commit changes only if job is pipelined, otherwise try next job. + if ssn.JobPipelined(preemptorJob) { + stmt.Commit() + } else { stmt.Discard() continue } @@ -155,7 +156,7 @@ func (alloc *preemptAction) Execute(ssn *framework.Session) { preemptor := preemptorTasks[job.UID].Pop().(*api.TaskInfo) stmt := ssn.Statement() - assigned, _ := preempt(ssn, stmt, preemptor, ssn.Nodes, func(task *api.TaskInfo) bool { + assigned, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool { // Ignore non running task. if task.Status != api.Running { return false @@ -181,12 +182,11 @@ func preempt( ssn *framework.Session, stmt *framework.Statement, preemptor *api.TaskInfo, - nodes map[string]*api.NodeInfo, filter func(*api.TaskInfo) bool, ) (bool, error) { assigned := false - allNodes := util.GetNodeList(nodes) + allNodes := util.GetNodeList(ssn.Nodes) predicateNodes, _ := util.PredicateNodes(preemptor, allNodes, ssn.PredicateFn) @@ -194,13 +194,10 @@ func preempt( selectedNodes := util.SortNodes(nodeScores) for _, node := range selectedNodes { - glog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", + klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", preemptor.Namespace, preemptor.Name, node.Name) var preemptees []*api.TaskInfo - preempted := api.EmptyResource() - resreq := preemptor.InitResreq.Clone() - for _, task := range node.Tasks { if filter == nil { preemptees = append(preemptees, task.Clone()) @@ -211,8 +208,8 @@ func preempt( victims := ssn.Preemptable(preemptor, preemptees) metrics.UpdatePreemptionVictimsCount(len(victims)) - if err := validateVictims(victims, resreq); err != nil { - glog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err) + if err := validateVictims(preemptor, node, victims); err != nil { + klog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err) continue } @@ -223,29 +220,31 @@ func preempt( victimsQueue.Push(victim) } // Preempt victims for tasks, pick lowest priority task first. + preempted := api.EmptyResource() + for !victimsQueue.Empty() { + // If reclaimed enough resources, break loop to avoid Sub panic. + if preemptor.InitResreq.LessEqual(node.FutureIdle()) { + break + } preemptee := victimsQueue.Pop().(*api.TaskInfo) - glog.Errorf("Try to preempt Task <%s/%s> for Tasks <%s/%s>", + klog.V(3).Infof("Try to preempt Task <%s/%s> for Tasks <%s/%s>", preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name) if err := stmt.Evict(preemptee, "preempt"); err != nil { - glog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v", + klog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v", preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name, err) continue } preempted.Add(preemptee.Resreq) - // If reclaimed enough resources, break loop to avoid Sub panic. - if resreq.LessEqual(preempted) { - break - } } metrics.RegisterPreemptionAttempts() - glog.V(3).Infof("Preempted <%v> for task <%s/%s> requested <%v>.", + klog.V(3).Infof("Preempted <%v> for Task <%s/%s> requested <%v>.", preempted, preemptor.Namespace, preemptor.Name, preemptor.InitResreq) - if preemptor.InitResreq.LessEqual(preempted) { + if preemptor.InitResreq.LessEqual(node.FutureIdle()) { if err := stmt.Pipeline(preemptor, node.Name); err != nil { - glog.Errorf("Failed to pipline Task <%s/%s> on Node <%s>", + klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", preemptor.Namespace, preemptor.Name, node.Name) } @@ -259,19 +258,19 @@ func preempt( return assigned, nil } -func validateVictims(victims []*api.TaskInfo, resreq *api.Resource) error { +func validateVictims(preemptor *api.TaskInfo, node *api.NodeInfo, victims []*api.TaskInfo) error { if len(victims) == 0 { return fmt.Errorf("no victims") } - - // If not enough resource, continue - allRes := api.EmptyResource() - for _, v := range victims { - allRes.Add(v.Resreq) + futureIdle := node.FutureIdle() + for _, victim := range victims { + futureIdle.Add(victim.Resreq) } - if allRes.Less(resreq) { - return fmt.Errorf("not enough resources") + // Every resource of the preemptor needs to be less or equal than corresponding + // idle resource after preemption. + if !preemptor.InitResreq.LessEqual(futureIdle) { + return fmt.Errorf("not enough resources: requested <%v>, but future idle <%v>", + preemptor.InitResreq, futureIdle) } - return nil } diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 9a46c675f8..f8be6703d8 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -54,7 +54,7 @@ func TestPreempt(t *testing.T) { expected int }{ { - name: "one Job with two Pods on one node", + name: "do not preempt if there are enough idle resources", podGroups: []*schedulingv2.PodGroup{ { ObjectMeta: metav1.ObjectMeta{ @@ -62,7 +62,8 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 3, + Queue: "q1", }, }, }, @@ -70,10 +71,10 @@ func TestPreempt(t *testing.T) { util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), }, + // If there are enough idle resources on the node, then there is no need to preempt anything. nodes: []*v1.Node{ - util.BuildNode("n1", util.BuildResourceList("3", "3Gi"), make(map[string]string)), + util.BuildNode("n1", util.BuildResourceList("10", "10G"), make(map[string]string)), }, queues: []*schedulingv2.Queue{ { @@ -85,10 +86,10 @@ func TestPreempt(t *testing.T) { }, }, }, - expected: 1, + expected: 0, }, { - name: "two Jobs on one node", + name: "do not preempt if job is pipelined", podGroups: []*schedulingv2.PodGroup{ { ObjectMeta: metav1.ObjectMeta{ @@ -96,7 +97,8 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 1, + Queue: "q1", }, }, { @@ -105,19 +107,63 @@ func TestPreempt(t *testing.T) { Namespace: "c1", }, Spec: schedulingv2.PodGroupSpec{ - Queue: "q1", + MinMember: 1, + Queue: "q1", + }, + }, + }, + // Both pg1 and pg2 jobs are pipelined, because enough pods are already running. + pods: []*v1.Pod{ + util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee3", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), + }, + // All resources on the node will be in use. + nodes: []*v1.Node{ + util.BuildNode("n1", util.BuildResourceList("3", "3G"), make(map[string]string)), + }, + queues: []*schedulingv2.Queue{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "q1", + }, + Spec: schedulingv2.QueueSpec{ + Weight: 1, + }, + }, + }, + expected: 0, + }, + { + name: "preempt one task of different job to fit both jobs on one node", + podGroups: []*schedulingv2.PodGroup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg2", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", }, }, }, pods: []*v1.Pod{ - // running pod with pg1, under c1 util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - // running pod with pg1, under c1 util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), - // pending pod with pg2, under c1 util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), - // pending pod with pg2, under c1 util.BuildPod("c1", "preemptor2", "", v1.PodPending, util.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string)), }, nodes: []*v1.Node{ @@ -133,76 +179,130 @@ func TestPreempt(t *testing.T) { }, }, }, + expected: 1, + }, + { + name: "preempt enough tasks to fit large task of different job", + podGroups: []*schedulingv2.PodGroup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pg2", + Namespace: "c1", + }, + Spec: schedulingv2.PodGroupSpec{ + MinMember: 1, + Queue: "q1", + }, + }, + }, + // There are 3 cpus and 3G of memory idle and 3 tasks running each consuming 1 cpu and 1G of memory. + // Big task requiring 5 cpus and 5G of memory should preempt 2 of 3 running tasks to fit into the node. + pods: []*v1.Pod{ + util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptee3", "n1", v1.PodRunning, util.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "preemptor1", "", v1.PodPending, util.BuildResourceList("5", "5G"), "pg2", make(map[string]string), make(map[string]string)), + }, + nodes: []*v1.Node{ + util.BuildNode("n1", util.BuildResourceList("6", "6G"), make(map[string]string)), + }, + queues: []*schedulingv2.Queue{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "q1", + }, + Spec: schedulingv2.QueueSpec{ + Weight: 1, + }, + }, + }, expected: 2, }, } - allocate := New() - - for i, test := range tests { - binder := &util.FakeBinder{ - Binds: map[string]string{}, - Channel: make(chan string), - } - evictor := &util.FakeEvictor{ - Evicts: make([]string, 0), - Channel: make(chan string), - } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - Evictor: evictor, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - - Recorder: record.NewFakeRecorder(100), - } - for _, node := range test.nodes { - schedulerCache.AddNode(node) - } - for _, pod := range test.pods { - schedulerCache.AddPod(pod) - } - - for _, ss := range test.podGroups { - schedulerCache.AddPodGroupV1alpha2(ss) - } - - for _, q := range test.queues { - schedulerCache.AddQueueV1alpha2(q) - } - - trueValue := true - ssn := framework.OpenSession(schedulerCache, []conf.Tier{ - { - Plugins: []conf.PluginOption{ - { - Name: "conformance", - EnabledPreemptable: &trueValue, - }, - { - Name: "gang", - EnabledPreemptable: &trueValue, + preempt := New() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binder := &util.FakeBinder{ + Binds: map[string]string{}, + Channel: make(chan string), + } + evictor := &util.FakeEvictor{ + Channel: make(chan string), + } + schedulerCache := &cache.SchedulerCache{ + Nodes: make(map[string]*api.NodeInfo), + Jobs: make(map[api.JobID]*api.JobInfo), + Queues: make(map[api.QueueID]*api.QueueInfo), + Binder: binder, + Evictor: evictor, + StatusUpdater: &util.FakeStatusUpdater{}, + VolumeBinder: &util.FakeVolumeBinder{}, + + Recorder: record.NewFakeRecorder(100), + } + for _, node := range test.nodes { + schedulerCache.AddNode(node) + } + for _, pod := range test.pods { + schedulerCache.AddPod(pod) + } + + for _, ss := range test.podGroups { + schedulerCache.AddPodGroupV1alpha2(ss) + } + + for _, q := range test.queues { + schedulerCache.AddQueueV1alpha2(q) + } + + trueValue := true + ssn := framework.OpenSession(schedulerCache, []conf.Tier{ + { + Plugins: []conf.PluginOption{ + { + Name: "conformance", + EnabledPreemptable: &trueValue, + }, + { + Name: "gang", + EnabledPreemptable: &trueValue, + EnabledJobPipelined: &trueValue, + }, }, }, - }, - }) - defer framework.CloseSession(ssn) + }, nil) + defer framework.CloseSession(ssn) - allocate.Execute(ssn) + preempt.Execute(ssn) - for i := 0; i < test.expected; i++ { + for i := 0; i < test.expected; i++ { + select { + case <-evictor.Channel: + case <-time.After(time.Second): + t.Errorf("not enough evictions") + } + } select { - case <-evictor.Channel: - case <-time.After(3 * time.Second): - t.Errorf("Failed to get evicting request.") + case key, opened := <-evictor.Channel: + if opened { + t.Errorf("unexpected eviction: %s", key) + } + case <-time.After(50 * time.Millisecond): + // TODO: Active waiting here is not optimal, but there is no better way currently. + // Ideally we would like to wait for evict and bind request goroutines to finish first. } - } - - if test.expected != len(evictor.Evicts) { - t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts)) - } + }) } } diff --git a/pkg/scheduler/actions/reclaim/reclaim.go b/pkg/scheduler/actions/reclaim/reclaim.go index ebc041804c..9891497657 100644 --- a/pkg/scheduler/actions/reclaim/reclaim.go +++ b/pkg/scheduler/actions/reclaim/reclaim.go @@ -17,7 +17,7 @@ limitations under the License. package reclaim import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -40,8 +40,8 @@ func (alloc *reclaimAction) Name() string { func (alloc *reclaimAction) Initialize() {} func (alloc *reclaimAction) Execute(ssn *framework.Session) { - glog.V(3).Infof("Enter Reclaim ...") - defer glog.V(3).Infof("Leaving Reclaim ...") + klog.V(3).Infof("Enter Reclaim ...") + defer klog.V(3).Infof("Leaving Reclaim ...") queues := util.NewPriorityQueue(ssn.QueueOrderFn) queueMap := map[api.QueueID]*api.QueueInfo{} @@ -49,7 +49,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { preemptorsMap := map[api.QueueID]*util.PriorityQueue{} preemptorTasks := map[api.JobID]*util.PriorityQueue{} - glog.V(3).Infof("There are <%d> Jobs and <%d> Queues in total for scheduling.", + klog.V(3).Infof("There are <%d> Jobs and <%d> Queues in total for scheduling.", len(ssn.Jobs), len(ssn.Queues)) var underRequest []*api.JobInfo @@ -58,17 +58,17 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { continue } if vr := ssn.JobValid(job); vr != nil && !vr.Pass { - glog.V(4).Infof("Job <%s/%s> Queue <%s> skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) + klog.V(4).Infof("Job <%s/%s> Queue <%s> skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message) continue } if queue, found := ssn.Queues[job.Queue]; !found { - glog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", + klog.Errorf("Failed to find Queue <%s> for Job <%s/%s>", job.Queue, job.Namespace, job.Name) continue } else { if _, existed := queueMap[queue.UID]; !existed { - glog.V(4).Infof("Added Queue <%s> for Job <%s/%s>", + klog.V(4).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name) queueMap[queue.UID] = queue @@ -100,7 +100,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { queue := queues.Pop().(*api.QueueInfo) if ssn.Overused(queue) { - glog.V(3).Infof("Queue <%s> is overused, ignore it.", queue.Name) + klog.V(3).Infof("Queue <%s> is overused, ignore it.", queue.Name) continue } @@ -128,7 +128,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { resreq := task.InitResreq.Clone() reclaimed := api.EmptyResource() - glog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", + klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", task.Namespace, task.Name, n.Name) var reclaimees []*api.TaskInfo @@ -148,7 +148,7 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { victims := ssn.Reclaimable(task, reclaimees) if len(victims) == 0 { - glog.V(3).Infof("No victims on Node <%s>.", n.Name) + klog.V(3).Infof("No victims on Node <%s>.", n.Name) continue } @@ -157,17 +157,17 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { for _, v := range victims { allRes.Add(v.Resreq) } - if allRes.Less(resreq) { - glog.V(3).Infof("Not enough resource from victims on Node <%s>.", n.Name) + if !resreq.LessEqual(allRes) { + klog.V(3).Infof("Not enough resource from victims on Node <%s>.", n.Name) continue } // Reclaim victims for tasks. for _, reclaimee := range victims { - glog.Errorf("Try to reclaim Task <%s/%s> for Tasks <%s/%s>", + klog.Errorf("Try to reclaim Task <%s/%s> for Tasks <%s/%s>", reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name) if err := ssn.Evict(reclaimee, "reclaim"); err != nil { - glog.Errorf("Failed to reclaim Task <%s/%s> for Tasks <%s/%s>: %v", + klog.Errorf("Failed to reclaim Task <%s/%s> for Tasks <%s/%s>: %v", reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name, err) continue } @@ -178,12 +178,12 @@ func (alloc *reclaimAction) Execute(ssn *framework.Session) { } } - glog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>.", + klog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>.", reclaimed, task.Namespace, task.Name, task.InitResreq) if task.InitResreq.LessEqual(reclaimed) { if err := ssn.Pipeline(task, n.Name); err != nil { - glog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", + klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", task.Namespace, task.Name, n.Name) } diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index 784526173a..1d32721f05 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -108,7 +108,6 @@ func TestReclaim(t *testing.T) { Channel: make(chan string), } evictor := &util.FakeEvictor{ - Evicts: make([]string, 0), Channel: make(chan string), } schedulerCache := &cache.SchedulerCache{ @@ -151,7 +150,7 @@ func TestReclaim(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) reclaim.Execute(ssn) @@ -164,8 +163,8 @@ func TestReclaim(t *testing.T) { } } - if test.expected != len(evictor.Evicts) { - t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts)) + if test.expected != len(evictor.Evicts()) { + t.Errorf("case %d (%s): expected: %v, got %v ", i, test.name, test.expected, len(evictor.Evicts())) } } } diff --git a/pkg/scheduler/api/namespace_info.go b/pkg/scheduler/api/namespace_info.go index 80ce552a19..2642b4fb27 100644 --- a/pkg/scheduler/api/namespace_info.go +++ b/pkg/scheduler/api/namespace_info.go @@ -19,10 +19,9 @@ package api import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog" ) // NamespaceName is name of namespace @@ -132,7 +131,7 @@ func (n *NamespaceCollection) Snapshot() *NamespaceInfo { obj, err := n.quotaWeight.Pop() if err != nil { - glog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err) + klog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err) } else { item := obj.(*quotaItem) weight = item.weight diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 9bb8924288..df97598154 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -19,9 +19,8 @@ package api import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" + "k8s.io/klog" ) // NodeInfo is node level aggregated information. @@ -34,6 +33,8 @@ type NodeInfo struct { // The releasing resource on that node Releasing *Resource + // The pipelined resource on that node + Pipelined *Resource // The idle resource on that node Idle *Resource // The used resource on that node, including running and terminating @@ -49,6 +50,13 @@ type NodeInfo struct { Others map[string]interface{} } +// FutureIdle returns resources that will be idle in the future: +// +// That is current idle resources plus released resources minus pipelined resources. +func (ni *NodeInfo) FutureIdle() *Resource { + return ni.Idle.Clone().Add(ni.Releasing).Sub(ni.Pipelined) +} + // NodeState defines the current state of node. type NodeState struct { Phase NodePhase @@ -62,6 +70,7 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { if node == nil { ni = &NodeInfo{ Releasing: EmptyResource(), + Pipelined: EmptyResource(), Idle: EmptyResource(), Used: EmptyResource(), @@ -76,6 +85,7 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { Node: node, Releasing: EmptyResource(), + Pipelined: EmptyResource(), Idle: NewResource(node.Status.Allocatable), Used: EmptyResource(), @@ -149,7 +159,7 @@ func (ni *NodeInfo) SetNode(node *v1.Node) { ni.setNodeState(node) if !ni.Ready() { - glog.Warningf("Failed to set node info, phase: %s, reason: %s", + klog.Warningf("Failed to set node info, phase: %s, reason: %s", ni.State.Phase, ni.State.Reason) return } @@ -159,16 +169,23 @@ func (ni *NodeInfo) SetNode(node *v1.Node) { ni.Allocatable = NewResource(node.Status.Allocatable) ni.Capability = NewResource(node.Status.Capacity) + ni.Releasing = EmptyResource() + ni.Pipelined = EmptyResource() ni.Idle = NewResource(node.Status.Allocatable) ni.Used = EmptyResource() - for _, task := range ni.Tasks { - if task.Status == Releasing { - ni.Releasing.Add(task.Resreq) + for _, ti := range ni.Tasks { + switch ti.Status { + case Releasing: + ni.Idle.Sub(ti.Resreq) + ni.Releasing.Add(ti.Resreq) + ni.Used.Add(ti.Resreq) + case Pipelined: + ni.Pipelined.Add(ti.Resreq) + default: + ni.Idle.Sub(ti.Resreq) + ni.Used.Add(ti.Resreq) } - - ni.Idle.Sub(task.Resreq) - ni.Used.Add(task.Resreq) } } @@ -203,15 +220,15 @@ func (ni *NodeInfo) AddTask(task *TaskInfo) error { return err } ni.Releasing.Add(ti.Resreq) + ni.Used.Add(ti.Resreq) case Pipelined: - ni.Releasing.Sub(ti.Resreq) + ni.Pipelined.Add(ti.Resreq) default: if err := ni.allocateIdleResource(ti); err != nil { return err } + ni.Used.Add(ti.Resreq) } - - ni.Used.Add(ti.Resreq) } ni.Tasks[key] = ti @@ -234,13 +251,13 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error { case Releasing: ni.Releasing.Sub(task.Resreq) ni.Idle.Add(task.Resreq) + ni.Used.Sub(task.Resreq) case Pipelined: - ni.Releasing.Add(task.Resreq) + ni.Pipelined.Sub(task.Resreq) default: ni.Idle.Add(task.Resreq) + ni.Used.Sub(task.Resreq) } - - ni.Used.Sub(task.Resreq) } delete(ni.Tasks, key) diff --git a/pkg/scheduler/api/node_info_test.go b/pkg/scheduler/api/node_info_test.go index 4ccbfb9985..6de73c8b32 100644 --- a/pkg/scheduler/api/node_info_test.go +++ b/pkg/scheduler/api/node_info_test.go @@ -57,6 +57,7 @@ func TestNodeInfo_AddPod(t *testing.T) { Idle: buildResource("5000m", "7G"), Used: buildResource("3000m", "3G"), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("8000m", "10G"), Capability: buildResource("8000m", "10G"), State: NodeState{Phase: Ready}, @@ -76,6 +77,7 @@ func TestNodeInfo_AddPod(t *testing.T) { Idle: buildResource("2000m", "1G"), Used: EmptyResource(), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("2000m", "1G"), Capability: buildResource("2000m", "1G"), State: NodeState{Phase: NotReady, Reason: "OutOfSync"}, @@ -124,6 +126,7 @@ func TestNodeInfo_RemovePod(t *testing.T) { Idle: buildResource("4000m", "6G"), Used: buildResource("4000m", "4G"), Releasing: EmptyResource(), + Pipelined: EmptyResource(), Allocatable: buildResource("8000m", "10G"), Capability: buildResource("8000m", "10G"), State: NodeState{Phase: Ready}, diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go index 37ca774182..cd813303a3 100644 --- a/pkg/scheduler/cache/cache.go +++ b/pkg/scheduler/cache/cache.go @@ -21,8 +21,6 @@ import ( "sync" "time" - "github.com/golang/glog" - "k8s.io/api/core/v1" "k8s.io/api/scheduling/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +39,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + "k8s.io/klog" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler/volumebinder" @@ -128,7 +127,7 @@ func (db *defaultBinder) Bind(p *v1.Pod, hostname string) error { Name: hostname, }, }); err != nil { - glog.Errorf("Failed to bind pod <%v/%v>: %#v", p.Namespace, p.Name, err) + klog.Errorf("Failed to bind pod <%v/%v>: %#v", p.Namespace, p.Name, err) return err } return nil @@ -140,10 +139,10 @@ type defaultEvictor struct { //Evict will send delete pod request to api server func (de *defaultEvictor) Evict(p *v1.Pod) error { - glog.V(3).Infof("Evicting pod %v/%v", p.Namespace, p.Name) + klog.V(3).Infof("Evicting pod %v/%v", p.Namespace, p.Name) if err := de.kubeclient.CoreV1().Pods(p.Namespace).Delete(p.Name, nil); err != nil { - glog.Errorf("Failed to evict pod <%v/%v>: %#v", p.Namespace, p.Name, err) + klog.Errorf("Failed to evict pod <%v/%v>: %#v", p.Namespace, p.Name, err) return err } return nil @@ -182,7 +181,7 @@ func podConditionHaveUpdate(status *v1.PodStatus, condition *v1.PodCondition) bo // UpdatePodCondition will Update pod with podCondition func (su *defaultStatusUpdater) UpdatePodCondition(pod *v1.Pod, condition *v1.PodCondition) (*v1.Pod, error) { - glog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) + klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status) if podutil.UpdatePodCondition(&pod.Status, condition) { return su.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(pod) } @@ -194,19 +193,19 @@ func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*sch if pg.Version == schedulingapi.PodGroupVersionV1Alpha1 { podgroup := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return nil, err } updated, err := su.vcclient.SchedulingV1alpha1().PodGroups(podgroup.Namespace).Update(podgroup) if err != nil { - glog.Errorf("Error while updating PodGroup with error: %v", err) + klog.Errorf("Error while updating PodGroup with error: %v", err) return nil, err } podGroupInfo := &schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Alpha1} if err := schedulingscheme.Scheme.Convert(updated, &podGroupInfo.PodGroup, nil); err != nil { - glog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err) + klog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err) return nil, err } @@ -216,18 +215,18 @@ func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*sch if pg.Version == schedulingapi.PodGroupVersionV1Alpha2 { podgroup := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return nil, err } updated, err := su.vcclient.SchedulingV1alpha2().PodGroups(podgroup.Namespace).Update(podgroup) if err != nil { - glog.Errorf("Error while updating PodGroup with error: %v", err) + klog.Errorf("Error while updating PodGroup with error: %v", err) } podGroupInfo := &schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Alpha2} if err := schedulingscheme.Scheme.Convert(updated, &podGroupInfo.PodGroup, nil); err != nil { - glog.Errorf("Error While converting v2alpha.PodGroup to api.PodGroup with error: %v", err) + klog.Errorf("Error While converting v2alpha.PodGroup to api.PodGroup with error: %v", err) return nil, err } @@ -534,7 +533,7 @@ func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) if job.PodGroup.Version == schedulingapi.PodGroupVersionV1Alpha1 { podgroup := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&job.PodGroup.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return err } sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason) @@ -544,7 +543,7 @@ func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) if job.PodGroup.Version == schedulingapi.PodGroupVersionV1Alpha2 { podgroup := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&job.PodGroup.PodGroup, podgroup, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return err } sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason) @@ -647,14 +646,14 @@ func (sc *SchedulerCache) taskUnschedulable(task *schedulingapi.TaskInfo, messag return err } } else { - glog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message) + klog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message) } return nil } func (sc *SchedulerCache) deleteJob(job *schedulingapi.JobInfo) { - glog.V(3).Infof("Try to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name) + klog.V(3).Infof("Try to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name) sc.deletedJobs.AddRateLimited(job) } @@ -669,7 +668,7 @@ func (sc *SchedulerCache) processCleanupJob() { job, found := obj.(*schedulingapi.JobInfo) if !found { - glog.Errorf("Failed to convert <%v> to *JobInfo", obj) + klog.Errorf("Failed to convert <%v> to *JobInfo", obj) return } @@ -678,7 +677,7 @@ func (sc *SchedulerCache) processCleanupJob() { if schedulingapi.JobTerminated(job) { delete(sc.Jobs, job.UID) - glog.V(3).Infof("Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name) + klog.V(3).Infof("Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name) } else { // Retry sc.deleteJob(job) @@ -699,12 +698,12 @@ func (sc *SchedulerCache) processResyncTask() { task, ok := obj.(*schedulingapi.TaskInfo) if !ok { - glog.Errorf("failed to convert %v to *v1.Pod", obj) + klog.Errorf("failed to convert %v to *v1.Pod", obj) return } if err := sc.syncTask(task); err != nil { - glog.Errorf("Failed to sync pod <%v/%v>, retry it.", task.Namespace, task.Name) + klog.Errorf("Failed to sync pod <%v/%v>, retry it.", task.Namespace, task.Name) sc.resyncTask(task) } } @@ -745,7 +744,7 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { value.Priority = priorityClass.Value } - glog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>", + klog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>", value.Namespace, value.Name, priName, value.Priority) } @@ -760,21 +759,21 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { for _, value := range sc.NamespaceCollection { info := value.Snapshot() snapshot.NamespaceInfo[info.Name] = info - glog.V(4).Infof("Namespace %s has weight %v", + klog.V(4).Infof("Namespace %s has weight %v", value.Name, info.GetWeight()) } for _, value := range sc.Jobs { // If no scheduling spec, does not handle it. if value.PodGroup == nil && value.PDB == nil { - glog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.", + klog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.", value.UID, value.Namespace, value.Name) continue } if _, found := snapshot.Queues[value.Queue]; !found { - glog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.", + klog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.", value.Queue, value.Namespace, value.Name) continue } @@ -784,7 +783,7 @@ func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo { } wg.Wait() - glog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.", + klog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.", len(snapshot.Jobs), len(snapshot.Queues), len(snapshot.Nodes)) return snapshot @@ -861,7 +860,7 @@ func (sc *SchedulerCache) RecordJobStatusEvent(job *schedulingapi.JobInfo) { msg = fitError.Error() } if err := sc.taskUnschedulable(taskInfo, msg); err != nil { - glog.Errorf("Failed to update unschedulable task status <%s/%s>: %v", + klog.Errorf("Failed to update unschedulable task status <%s/%s>: %v", taskInfo.Namespace, taskInfo.Name, err) } } @@ -891,7 +890,7 @@ func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, if podGroup.Version == schedulingapi.PodGroupVersionV1Alpha1 { pg := &v1alpha1.PodGroup{} if err := schedulingscheme.Scheme.Convert(&podGroup.PodGroup, pg, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err) return } @@ -901,7 +900,7 @@ func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, if podGroup.Version == schedulingapi.PodGroupVersionV1Alpha2 { pg := &v1alpha2.PodGroup{} if err := schedulingscheme.Scheme.Convert(&podGroup.PodGroup, pg, nil); err != nil { - glog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) + klog.Errorf("Error while converting PodGroup to v1alpha2.PodGroup with error: %v", err) return } diff --git a/pkg/scheduler/cache/event_handlers.go b/pkg/scheduler/cache/event_handlers.go index 03d65d84d0..dfe0c62f91 100644 --- a/pkg/scheduler/cache/event_handlers.go +++ b/pkg/scheduler/cache/event_handlers.go @@ -19,14 +19,13 @@ package cache import ( "fmt" - "github.com/golang/glog" - "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1beta1" "k8s.io/api/scheduling/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/apis/scheduling/scheme" @@ -45,7 +44,7 @@ func isTerminated(status schedulingapi.TaskStatus) bool { func (sc *SchedulerCache) getOrCreateJob(pi *schedulingapi.TaskInfo) *schedulingapi.JobInfo { if len(pi.Job) == 0 { if pi.Pod.Spec.SchedulerName != sc.schedulerName { - glog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it", + klog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it", pi.Pod.Namespace, pi.Pod.Name, sc.schedulerName) } return nil @@ -90,7 +89,7 @@ func (sc *SchedulerCache) syncTask(oldTask *schedulingapi.TaskInfo) error { if err != nil { if errors.IsNotFound(err) { sc.deleteTask(oldTask) - glog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) + klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name) return nil } @@ -173,7 +172,7 @@ func (sc *SchedulerCache) deletePod(pod *v1.Pod) error { func (sc *SchedulerCache) AddPod(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert to *v1.Pod: %v", obj) + klog.Errorf("Cannot convert to *v1.Pod: %v", obj) return } @@ -182,11 +181,11 @@ func (sc *SchedulerCache) AddPod(obj interface{}) { err := sc.addPod(pod) if err != nil { - glog.Errorf("Failed to add pod <%s/%s> into cache: %v", + klog.Errorf("Failed to add pod <%s/%s> into cache: %v", pod.Namespace, pod.Name, err) return } - glog.V(3).Infof("Added pod <%s/%v> into cache.", pod.Namespace, pod.Name) + klog.V(3).Infof("Added pod <%s/%v> into cache.", pod.Namespace, pod.Name) return } @@ -194,12 +193,12 @@ func (sc *SchedulerCache) AddPod(obj interface{}) { func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj) return } newPod, ok := newObj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj) return } @@ -208,11 +207,11 @@ func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) { err := sc.updatePod(oldPod, newPod) if err != nil { - glog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err) + klog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err) return } - glog.V(3).Infof("Updated pod <%s/%v> in cache.", oldPod.Namespace, oldPod.Name) + klog.V(3).Infof("Updated pod <%s/%v> in cache.", oldPod.Namespace, oldPod.Name) return } @@ -227,11 +226,11 @@ func (sc *SchedulerCache) DeletePod(obj interface{}) { var ok bool pod, ok = t.Obj.(*v1.Pod) if !ok { - glog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.Pod: %v", t) + klog.Errorf("Cannot convert to *v1.Pod: %v", t) return } @@ -240,11 +239,11 @@ func (sc *SchedulerCache) DeletePod(obj interface{}) { err := sc.deletePod(pod) if err != nil { - glog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err) + klog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err) return } - glog.V(3).Infof("Deleted pod <%s/%v> from cache.", pod.Namespace, pod.Name) + klog.V(3).Infof("Deleted pod <%s/%v> from cache.", pod.Namespace, pod.Name) return } @@ -282,7 +281,7 @@ func (sc *SchedulerCache) deleteNode(node *v1.Node) error { func (sc *SchedulerCache) AddNode(obj interface{}) { node, ok := obj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert to *v1.Node: %v", obj) + klog.Errorf("Cannot convert to *v1.Node: %v", obj) return } @@ -291,7 +290,7 @@ func (sc *SchedulerCache) AddNode(obj interface{}) { err := sc.addNode(node) if err != nil { - glog.Errorf("Failed to add node %s into cache: %v", node.Name, err) + klog.Errorf("Failed to add node %s into cache: %v", node.Name, err) return } return @@ -301,12 +300,12 @@ func (sc *SchedulerCache) AddNode(obj interface{}) { func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) { oldNode, ok := oldObj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj) return } newNode, ok := newObj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj) return } @@ -315,7 +314,7 @@ func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) { err := sc.updateNode(oldNode, newNode) if err != nil { - glog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err) + klog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err) return } return @@ -331,11 +330,11 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) { var ok bool node, ok = t.Obj.(*v1.Node) if !ok { - glog.Errorf("Cannot convert to *v1.Node: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.Node: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.Node: %v", t) + klog.Errorf("Cannot convert to *v1.Node: %v", t) return } @@ -344,7 +343,7 @@ func (sc *SchedulerCache) DeleteNode(obj interface{}) { err := sc.deleteNode(node) if err != nil { - glog.Errorf("Failed to delete node %s from cache: %v", node.Name, err) + klog.Errorf("Failed to delete node %s from cache: %v", node.Name, err) return } return @@ -395,24 +394,24 @@ func (sc *SchedulerCache) deletePodGroup(id schedulingapi.JobID) error { func (sc *SchedulerCache) AddPodGroupV1alpha1(obj interface{}) { ss, ok := obj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj) + klog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj) return } podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(ss, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) return } pg := &schedulingapi.PodGroup{podgroup, schedulingapi.PodGroupVersionV1Alpha1} - glog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() if err := sc.setPodGroup(pg); err != nil { - glog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) + klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) return } @@ -423,24 +422,24 @@ func (sc *SchedulerCache) AddPodGroupV1alpha1(obj interface{}) { func (sc *SchedulerCache) AddPodGroupV1alpha2(obj interface{}) { ss, ok := obj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", obj) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", obj) return } podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(ss, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup) return } pg := &schedulingapi.PodGroup{podgroup, schedulingapi.PodGroupVersionV1Alpha2} - glog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() if err := sc.setPodGroup(pg); err != nil { - glog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) + klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err) return } return @@ -450,12 +449,12 @@ func (sc *SchedulerCache) AddPodGroupV1alpha2(obj interface{}) { func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj) return } newSS, ok := newObj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj) return } @@ -465,7 +464,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(newSS, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) return } @@ -475,7 +474,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { defer sc.Mutex.Unlock() if err := sc.updatePodGroup(pg); err != nil { - glog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err) + klog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err) return } return @@ -485,12 +484,12 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha1(oldObj, newObj interface{}) { func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv2.SchedulingSpec: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv2.SchedulingSpec: %v", oldObj) return } newSS, ok := newObj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv2.SchedulingSpec: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv2.SchedulingSpec: %v", newObj) return } @@ -500,7 +499,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { podgroup := scheduling.PodGroup{} if err := scheme.Scheme.Convert(newSS, &podgroup, nil); err != nil { - glog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) + klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup) return } @@ -510,7 +509,7 @@ func (sc *SchedulerCache) UpdatePodGroupV1alpha2(oldObj, newObj interface{}) { defer sc.Mutex.Unlock() if err := sc.updatePodGroup(pg); err != nil { - glog.Errorf("Failed to update podgroup %s into cache: %v", pg.Name, err) + klog.Errorf("Failed to update podgroup %s into cache: %v", pg.Name, err) return } } @@ -525,11 +524,11 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha1(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv1.PodGroup) if !ok { - glog.Errorf("Cannot convert to podgroup: %v", t.Obj) + klog.Errorf("Cannot convert to podgroup: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to podgroup: %v", t) + klog.Errorf("Cannot convert to podgroup: %v", t) return } @@ -539,7 +538,7 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha1(obj interface{}) { defer sc.Mutex.Unlock() if err := sc.deletePodGroup(jobID); err != nil { - glog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) + klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) return } return @@ -555,11 +554,11 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha2(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv2.PodGroup) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t) + klog.Errorf("Cannot convert to *schedulingv2.PodGroup: %v", t) return } @@ -569,7 +568,7 @@ func (sc *SchedulerCache) DeletePodGroupV1alpha2(obj interface{}) { defer sc.Mutex.Unlock() if err := sc.deletePodGroup(jobID); err != nil { - glog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) + klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err) return } @@ -621,7 +620,7 @@ func (sc *SchedulerCache) deletePDB(pdb *policyv1.PodDisruptionBudget) error { func (sc *SchedulerCache) AddPDB(obj interface{}) { pdb, ok := obj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", obj) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", obj) return } @@ -630,7 +629,7 @@ func (sc *SchedulerCache) AddPDB(obj interface{}) { err := sc.setPDB(pdb) if err != nil { - glog.Errorf("Failed to add PodDisruptionBudget %s into cache: %v", pdb.Name, err) + klog.Errorf("Failed to add PodDisruptionBudget %s into cache: %v", pdb.Name, err) return } return @@ -640,12 +639,12 @@ func (sc *SchedulerCache) AddPDB(obj interface{}) { func (sc *SchedulerCache) UpdatePDB(oldObj, newObj interface{}) { oldPDB, ok := oldObj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert oldObj to *policyv1.PodDisruptionBudget: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *policyv1.PodDisruptionBudget: %v", oldObj) return } newPDB, ok := newObj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert newObj to *policyv1.PodDisruptionBudget: %v", newObj) + klog.Errorf("Cannot convert newObj to *policyv1.PodDisruptionBudget: %v", newObj) return } @@ -654,7 +653,7 @@ func (sc *SchedulerCache) UpdatePDB(oldObj, newObj interface{}) { err := sc.updatePDB(oldPDB, newPDB) if err != nil { - glog.Errorf("Failed to update PodDisruptionBudget %s into cache: %v", oldPDB.Name, err) + klog.Errorf("Failed to update PodDisruptionBudget %s into cache: %v", oldPDB.Name, err) return } return @@ -670,11 +669,11 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { var ok bool pdb, ok = t.Obj.(*policyv1.PodDisruptionBudget) if !ok { - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t.Obj) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t) + klog.Errorf("Cannot convert to *policyv1.PodDisruptionBudget: %v", t) return } @@ -683,7 +682,7 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { err := sc.deletePDB(pdb) if err != nil { - glog.Errorf("Failed to delete PodDisruptionBudget %s from cache: %v", pdb.Name, err) + klog.Errorf("Failed to delete PodDisruptionBudget %s from cache: %v", pdb.Name, err) return } return @@ -693,20 +692,20 @@ func (sc *SchedulerCache) DeletePDB(obj interface{}) { func (sc *SchedulerCache) AddQueueV1alpha1(obj interface{}) { ss, ok := obj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj) return } queue := &scheduling.Queue{} if err := scheme.Scheme.Convert(ss, queue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", ss, queue) + klog.Errorf("Failed to convert queue from %T to %T", ss, queue) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.addQueue(queue) return @@ -716,17 +715,17 @@ func (sc *SchedulerCache) AddQueueV1alpha1(obj interface{}) { func (sc *SchedulerCache) AddQueueV1alpha2(obj interface{}) { ss, ok := obj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.Queue: %v", obj) + klog.Errorf("Cannot convert to *schedulingv2.Queue: %v", obj) return } queue := &scheduling.Queue{} if err := scheme.Scheme.Convert(ss, queue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", ss, queue) + klog.Errorf("Failed to convert queue from %T to %T", ss, queue) return } - glog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) + klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec) sc.Mutex.Lock() defer sc.Mutex.Unlock() @@ -739,12 +738,12 @@ func (sc *SchedulerCache) AddQueueV1alpha2(obj interface{}) { func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj) return } newSS, ok := newObj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj) return } @@ -754,7 +753,7 @@ func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { newQueue := &scheduling.Queue{} if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) + klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) return } @@ -769,12 +768,12 @@ func (sc *SchedulerCache) UpdateQueueV1alpha1(oldObj, newObj interface{}) { func (sc *SchedulerCache) UpdateQueueV1alpha2(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert oldObj to *schedulingv2.Queue: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *schedulingv2.Queue: %v", oldObj) return } newSS, ok := newObj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert newObj to *schedulingv2.Queue: %v", newObj) + klog.Errorf("Cannot convert newObj to *schedulingv2.Queue: %v", newObj) return } @@ -784,7 +783,7 @@ func (sc *SchedulerCache) UpdateQueueV1alpha2(oldObj, newObj interface{}) { newQueue := &scheduling.Queue{} if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil { - glog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) + klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue) return } @@ -805,11 +804,11 @@ func (sc *SchedulerCache) DeleteQueueV1alpha1(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv1.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) return } @@ -830,11 +829,11 @@ func (sc *SchedulerCache) DeleteQueueV1alpha2(obj interface{}) { var ok bool ss, ok = t.Obj.(*schedulingv2.Queue) if !ok { - glog.Errorf("Cannot convert to *schedulingv2.Queue: %v", t.Obj) + klog.Errorf("Cannot convert to *schedulingv2.Queue: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) + klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t) return } @@ -868,11 +867,11 @@ func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) { var ok bool ss, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return } @@ -886,7 +885,7 @@ func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) { func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) { oldSS, ok := oldObj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj) + klog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj) return @@ -894,7 +893,7 @@ func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) { newSS, ok := newObj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj) return @@ -917,11 +916,11 @@ func (sc *SchedulerCache) AddPriorityClass(obj interface{}) { var ok bool ss, ok = t.Obj.(*v1beta1.PriorityClass) if !ok { - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) + klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t) return } @@ -944,7 +943,7 @@ func (sc *SchedulerCache) deletePriorityClass(pc *v1beta1.PriorityClass) { func (sc *SchedulerCache) addPriorityClass(pc *v1beta1.PriorityClass) { if pc.GlobalDefault { if sc.defaultPriorityClass != nil { - glog.Errorf("Updated default priority class from <%s> to <%s> forcefully.", + klog.Errorf("Updated default priority class from <%s> to <%s> forcefully.", sc.defaultPriorityClass.Name, pc.Name) } @@ -984,18 +983,18 @@ func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) { var ok bool r, ok = t.Obj.(*v1.ResourceQuota) if !ok { - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj) return } default: - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Delete ResourceQuota <%s/%v> in cache", r.Namespace, r.Name) + klog.V(3).Infof("Delete ResourceQuota <%s/%v> in cache", r.Namespace, r.Name) sc.deleteResourceQuota(r) } @@ -1003,14 +1002,14 @@ func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) { func (sc *SchedulerCache) UpdateResourceQuota(oldObj, newObj interface{}) { newR, ok := newObj.(*v1.ResourceQuota) if !ok { - glog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj) + klog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Update ResourceQuota <%s/%v> in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard) + klog.V(3).Infof("Update ResourceQuota <%s/%v> in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard) sc.updateResourceQuota(newR) } @@ -1021,13 +1020,13 @@ func (sc *SchedulerCache) AddResourceQuota(obj interface{}) { case *v1.ResourceQuota: r = t default: - glog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) + klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t) return } sc.Mutex.Lock() defer sc.Mutex.Unlock() - glog.V(3).Infof("Add ResourceQuota <%s/%v> in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard) + klog.V(3).Infof("Add ResourceQuota <%s/%v> in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard) sc.updateResourceQuota(r) } diff --git a/pkg/scheduler/conf/scheduler_conf.go b/pkg/scheduler/conf/scheduler_conf.go index a45c6e4697..55378fd77f 100644 --- a/pkg/scheduler/conf/scheduler_conf.go +++ b/pkg/scheduler/conf/scheduler_conf.go @@ -22,6 +22,8 @@ type SchedulerConfiguration struct { Actions string `yaml:"actions"` // Tiers defines plugins in different tiers Tiers []Tier `yaml:"tiers"` + // Configurations is configuration for actions + Configurations []Configuration `yaml:"configurations"` } // Tier defines plugin tier @@ -29,6 +31,14 @@ type Tier struct { Plugins []PluginOption `yaml:"plugins"` } +// Configuration is configuration of action +type Configuration struct { + // Name is name of action + Name string `yaml:"name"` + // Arguments defines the different arguments that can be given to specified action + Arguments map[string]string `yaml:"arguments"` +} + // PluginOption defines the options of plugin type PluginOption struct { // The name of Plugin diff --git a/pkg/scheduler/framework/arguments.go b/pkg/scheduler/framework/arguments.go index 5d803b67e7..be31cc7805 100644 --- a/pkg/scheduler/framework/arguments.go +++ b/pkg/scheduler/framework/arguments.go @@ -19,13 +19,15 @@ package framework import ( "strconv" - "github.com/golang/glog" + "volcano.sh/volcano/pkg/scheduler/conf" + + "k8s.io/klog" ) // Arguments map type Arguments map[string]string -//GetInt get the integer value from string +// GetInt get the integer value from string func (a Arguments) GetInt(ptr *int, key string) { if ptr == nil { return @@ -38,14 +40,34 @@ func (a Arguments) GetInt(ptr *int, key string) { value, err := strconv.Atoi(argv) if err != nil { - glog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + return + } + + *ptr = value +} + +// GetFloat64 get the float64 value from string +func (a Arguments) GetFloat64(ptr *float64, key string) { + if ptr == nil { + return + } + + argv, ok := a[key] + if !ok || len(argv) == 0 { + return + } + + value, err := strconv.ParseFloat(argv, 64) + if err != nil { + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) return } *ptr = value } -//GetBool get the bool value from string +// GetBool get the bool value from string func (a Arguments) GetBool(ptr *bool, key string) { if ptr == nil { return @@ -58,9 +80,20 @@ func (a Arguments) GetBool(ptr *bool, key string) { value, err := strconv.ParseBool(argv) if err != nil { - glog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) + klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err) return } *ptr = value } + +// GetArgOfActionFromConf return argument of action reading from configuration of schedule +func GetArgOfActionFromConf(configurations []conf.Configuration, actionName string) Arguments { + for _, c := range configurations { + if c.Name == actionName { + return c.Arguments + } + } + + return nil +} diff --git a/pkg/scheduler/framework/arguments_test.go b/pkg/scheduler/framework/arguments_test.go index f9d5f30a03..44559363df 100644 --- a/pkg/scheduler/framework/arguments_test.go +++ b/pkg/scheduler/framework/arguments_test.go @@ -17,7 +17,10 @@ limitations under the License. package framework import ( + "reflect" "testing" + + "volcano.sh/volcano/pkg/scheduler/conf" ) type GetIntTestCases struct { @@ -74,3 +77,111 @@ func TestArgumentsGetInt(t *testing.T) { } } } + +func TestArgumentsGetFloat64(t *testing.T) { + key1 := "float64key" + + cases := []struct { + name string + arg Arguments + key string + baseValue float64 + expectValue float64 + }{ + { + name: "key not exist", + arg: Arguments{ + "anotherKey": "12", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + { + name: "key exist", + arg: Arguments{ + key1: "1.5", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.5, + }, + { + name: "value of key invalid", + arg: Arguments{ + key1: "errorValue", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + { + name: "value of key null", + arg: Arguments{ + key1: "", + }, + key: key1, + baseValue: 1.2, + expectValue: 1.2, + }, + } + + for index, c := range cases { + baseValue := c.baseValue + c.arg.GetFloat64(&baseValue, c.key) + if baseValue != c.expectValue { + t.Errorf("index %d, case %s, value should be %v, but not %v", index, c.name, c.expectValue, baseValue) + } + } +} + +func TestGetArgOfActionFromConf(t *testing.T) { + cases := []struct { + name string + configurations []conf.Configuration + action string + expectedArguments Arguments + }{ + { + name: "action exist in configurations", + configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + { + Name: "allocate", + Arguments: map[string]string{ + "placeholde": "placeholde", + }, + }, + }, + action: "enqueue", + expectedArguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + { + name: "action not exist in configurations", + configurations: []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overCommitFactor": "1.5", + }, + }, + }, + action: "allocate", + expectedArguments: nil, + }, + } + + for index, c := range cases { + arg := GetArgOfActionFromConf(c.configurations, c.action) + if false == reflect.DeepEqual(arg, c.expectedArguments) { + t.Errorf("index %d, case %s,expected %v, but got %v", index, c.name, c.expectedArguments, arg) + } + } +} diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index 379b7c7b8d..f66275769e 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -19,7 +19,7 @@ package framework import ( "time" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -27,14 +27,15 @@ import ( ) // OpenSession start the session -func OpenSession(cache cache.Cache, tiers []conf.Tier) *Session { +func OpenSession(cache cache.Cache, tiers []conf.Tier, configurations []conf.Configuration) *Session { ssn := openSession(cache) ssn.Tiers = tiers + ssn.Configurations = configurations for _, tier := range tiers { for _, plugin := range tier.Plugins { if pb, found := GetPluginBuilder(plugin.Name); !found { - glog.Errorf("Failed to get plugin %s.", plugin.Name) + klog.Errorf("Failed to get plugin %s.", plugin.Name) } else { plugin := pb(plugin.Arguments) ssn.plugins[plugin.Name()] = plugin diff --git a/pkg/scheduler/framework/job_updater.go b/pkg/scheduler/framework/job_updater.go index 2c00614287..98454fbc39 100644 --- a/pkg/scheduler/framework/job_updater.go +++ b/pkg/scheduler/framework/job_updater.go @@ -6,9 +6,8 @@ import ( "reflect" "time" - "github.com/golang/glog" - "k8s.io/client-go/util/workqueue" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -116,7 +115,7 @@ func (ju *jobUpdater) updateJob(index int) { updatePG := !found || isPodGroupStatusUpdated(&job.PodGroup.Status, oldStatus) if _, err := ssn.cache.UpdateJobStatus(job, updatePG); err != nil { - glog.Errorf("Failed to update job <%s/%s>: %v", + klog.Errorf("Failed to update job <%s/%s>: %v", job.Namespace, job.Name, err) } } diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index 1397fd21c7..b74c48781a 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -19,12 +19,11 @@ package framework import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/scheduler/api" @@ -46,8 +45,9 @@ type Session struct { Queues map[api.QueueID]*api.QueueInfo NamespaceInfo map[api.NamespaceName]*api.NamespaceInfo - Backlog []*api.JobInfo - Tiers []conf.Tier + Backlog []*api.JobInfo + Tiers []conf.Tier + Configurations []conf.Configuration plugins map[string]Plugin eventHandlers []*EventHandler @@ -120,7 +120,7 @@ func openSession(cache cache.Cache) *Session { } if err := ssn.UpdateJobCondition(job, jc); err != nil { - glog.Errorf("Failed to update job condition: %v", err) + klog.Errorf("Failed to update job condition: %v", err) } } @@ -132,7 +132,7 @@ func openSession(cache cache.Cache) *Session { ssn.Queues = snapshot.Queues ssn.NamespaceInfo = snapshot.NamespaceInfo - glog.V(3).Infof("Open Session %v with <%d> Job and <%d> Queues", + klog.V(3).Infof("Open Session %v with <%d> Job and <%d> Queues", ssn.UID, len(ssn.Jobs), len(ssn.Queues)) return ssn @@ -151,7 +151,7 @@ func closeSession(ssn *Session) { ssn.namespaceOrderFns = nil ssn.queueOrderFns = nil - glog.V(3).Infof("Close Session %v", ssn.UID) + klog.V(3).Infof("Close Session %v", ssn.UID) } func jobStatus(ssn *Session, jobInfo *api.JobInfo) scheduling.PodGroupStatus { @@ -207,12 +207,12 @@ func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error { job, found := ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s when binding", task.Job) } @@ -221,14 +221,14 @@ func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error { if node, found := ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, ssn.UID, err) return err } - glog.V(3).Infof("After added Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After added Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -254,12 +254,12 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { job, found := ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Allocated); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Allocated, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -268,14 +268,14 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { if node, found := ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, ssn.UID, err) return err } - glog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -292,7 +292,7 @@ func (ssn *Session) Allocate(task *api.TaskInfo, hostname string) error { if ssn.JobReady(job) { for _, task := range job.TaskStatusIndex[api.Allocated] { if err := ssn.dispatch(task); err != nil { - glog.Errorf("Failed to dispatch task <%v/%v>: %v", + klog.Errorf("Failed to dispatch task <%v/%v>: %v", task.Namespace, task.Name, err) return err } @@ -314,12 +314,12 @@ func (ssn *Session) dispatch(task *api.TaskInfo) error { // Update status in session if job, found := ssn.Jobs[task.Job]; found { if err := job.UpdateTaskStatus(task, api.Binding); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Binding, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -338,12 +338,12 @@ func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error { job, found := ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, ssn.UID) return fmt.Errorf("failed to find job %s", reclaimee.Job) } @@ -351,7 +351,7 @@ func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error { // Update task in node. if node, found := ssn.Nodes[reclaimee.NodeName]; found { if err := node.UpdateTask(reclaimee); err != nil { - glog.Errorf("Failed to update task <%v/%v> in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, ssn.UID, err) return err } diff --git a/pkg/scheduler/framework/statement.go b/pkg/scheduler/framework/statement.go index 6fa2682dcb..37229119cb 100644 --- a/pkg/scheduler/framework/statement.go +++ b/pkg/scheduler/framework/statement.go @@ -19,7 +19,7 @@ package framework import ( "fmt" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/metrics" @@ -42,11 +42,11 @@ func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, s.ssn.UID) } @@ -74,7 +74,7 @@ func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error { func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error { if err := s.ssn.cache.Evict(reclaimee, reason); err != nil { if e := s.unevict(reclaimee, reason); err != nil { - glog.Errorf("Faled to unevict task <%v/%v>: %v.", + klog.Errorf("Faled to unevict task <%v/%v>: %v.", reclaimee.Namespace, reclaimee.Name, e) } return err @@ -88,17 +88,17 @@ func (s *Statement) unevict(reclaimee *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[reclaimee.Job] if found { if err := job.UpdateTaskStatus(reclaimee, api.Running); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", reclaimee.Job, s.ssn.UID) } // Update task in node. if node, found := s.ssn.Nodes[reclaimee.NodeName]; found { - node.AddTask(reclaimee) + node.UpdateTask(reclaimee) } for _, eh := range s.ssn.eventHandlers { @@ -117,11 +117,11 @@ func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) } @@ -129,13 +129,13 @@ func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) } - glog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) } @@ -162,11 +162,11 @@ func (s *Statement) unpipeline(task *api.TaskInfo) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pending); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err) } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) } @@ -174,13 +174,13 @@ func (s *Statement) unpipeline(task *api.TaskInfo) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.RemoveTask(task); err != nil { - glog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) } - glog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) } @@ -205,12 +205,12 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Allocated); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Allocated, s.ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -219,14 +219,14 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { if node, found := s.ssn.Nodes[hostname]; found { if err := node.AddTask(task); err != nil { - glog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", + klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v", task.Namespace, task.Name, hostname, s.ssn.UID, err) return err } - glog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", + klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>", task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing) } else { - glog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.", hostname, s.ssn.UID) return fmt.Errorf("failed to find node %s", hostname) } @@ -241,7 +241,7 @@ func (s *Statement) Allocate(task *api.TaskInfo, hostname string) error { } // Update status in session - glog.V(3).Info("Allocating operations ...") + klog.V(3).Info("Allocating operations ...") s.operations = append(s.operations, operation{ name: "allocate", args: []interface{}{task, hostname}, @@ -262,12 +262,12 @@ func (s *Statement) allocate(task *api.TaskInfo, hostname string) error { // Update status in session if job, found := s.ssn.Jobs[task.Job]; found { if err := job.UpdateTaskStatus(task, api.Binding); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Binding, s.ssn.UID, err) return err } } else { - glog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", + klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.", task.Job, s.ssn.UID) return fmt.Errorf("failed to find job %s", task.Job) } @@ -282,16 +282,16 @@ func (s *Statement) unallocate(task *api.TaskInfo, reason string) error { job, found := s.ssn.Jobs[task.Job] if found { if err := job.UpdateTaskStatus(task, api.Pending); err != nil { - glog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", + klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v", task.Namespace, task.Name, api.Pending, s.ssn.UID, err) } } else { - glog.Errorf("Failed to find Job <%s> in Session <%s> index when unallocating.", + klog.Errorf("Failed to find Job <%s> in Session <%s> index when unallocating.", task.Job, s.ssn.UID) } if node, found := s.ssn.Nodes[task.NodeName]; found { - glog.V(3).Infof("Remove Task <%v> on node <%v>", task.Name, task.NodeName) + klog.V(3).Infof("Remove Task <%v> on node <%v>", task.Name, task.NodeName) node.RemoveTask(task) } @@ -307,7 +307,7 @@ func (s *Statement) unallocate(task *api.TaskInfo, reason string) error { // Discard operation for evict, pipeline and allocate func (s *Statement) Discard() { - glog.V(3).Info("Discarding operations ...") + klog.V(3).Info("Discarding operations ...") for i := len(s.operations) - 1; i >= 0; i-- { op := s.operations[i] switch op.name { @@ -323,7 +323,7 @@ func (s *Statement) Discard() { // Commit operation for evict and pipeline func (s *Statement) Commit() { - glog.V(3).Info("Committing operations ...") + klog.V(3).Info("Committing operations ...") for _, op := range s.operations { switch op.name { case "evict": diff --git a/pkg/scheduler/plugins/binpack/binpack.go b/pkg/scheduler/plugins/binpack/binpack.go index 108c4d06f4..f54709c32e 100644 --- a/pkg/scheduler/plugins/binpack/binpack.go +++ b/pkg/scheduler/plugins/binpack/binpack.go @@ -20,9 +20,8 @@ import ( "fmt" "strings" - "github.com/golang/glog" - "k8s.io/api/core/v1" + "k8s.io/klog" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api" @@ -156,10 +155,10 @@ func (bp *binpackPlugin) Name() string { } func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) { - glog.V(4).Infof("Enter binpack plugin ...") - if glog.V(4) { + klog.V(4).Infof("Enter binpack plugin ...") + if klog.V(4) { defer func() { - glog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String()) + klog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String()) }() notFoundResource := []string{} @@ -175,19 +174,19 @@ func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) { notFoundResource = append(notFoundResource, string(resource)) } } - glog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", ")) + klog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", ")) } nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) { binPackingScore := BinPackingScore(task, node, bp.weight) - glog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore) + klog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore) return binPackingScore, nil } if bp.weight.BinPackingWeight != 0 { ssn.AddNodeOrderFn(bp.Name(), nodeOrderFn) } else { - glog.Infof("binpack weight is zero, skip node order function") + klog.Infof("binpack weight is zero, skip node order function") } } @@ -230,7 +229,7 @@ func BinPackingScore(task *api.TaskInfo, node *api.NodeInfo, weight priorityWeig } resourceScore := ResourceBinPackingScore(request, allocate, nodeUsed, resourceWeight) - glog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore) + klog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore) score += resourceScore weightSum += resourceWeight diff --git a/pkg/scheduler/plugins/binpack/binpack_test.go b/pkg/scheduler/plugins/binpack/binpack_test.go index 1253793c41..bcc80767ba 100644 --- a/pkg/scheduler/plugins/binpack/binpack_test.go +++ b/pkg/scheduler/plugins/binpack/binpack_test.go @@ -269,7 +269,7 @@ func TestNode(t *testing.T) { }, }, }, - }) + }, nil) defer framework.CloseSession(ssn) for _, job := range ssn.Jobs { diff --git a/pkg/scheduler/plugins/drf/drf.go b/pkg/scheduler/plugins/drf/drf.go index e7adc0d4d4..cd96157cd7 100644 --- a/pkg/scheduler/plugins/drf/drf.go +++ b/pkg/scheduler/plugins/drf/drf.go @@ -19,7 +19,7 @@ package drf import ( "math" - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/helpers" @@ -193,7 +193,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { } } - glog.V(4).Infof("Victims from DRF plugins are %+v", victims) + klog.V(4).Infof("Victims from DRF plugins are %+v", victims) return victims } @@ -204,7 +204,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.JobInfo) rv := r.(*api.JobInfo) - glog.V(4).Infof("DRF JobOrderFn: <%v/%v> share state: %v, <%v/%v> share state: %v", + klog.V(4).Infof("DRF JobOrderFn: <%v/%v> share state: %v, <%v/%v> share state: %v", lv.Namespace, lv.Name, drf.jobAttrs[lv.UID].share, rv.Namespace, rv.Name, drf.jobAttrs[rv.UID].share) if drf.jobAttrs[lv.UID].share == drf.jobAttrs[rv.UID].share { @@ -230,7 +230,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { lWeight := ssn.NamespaceInfo[lv].GetWeight() rWeight := ssn.NamespaceInfo[rv].GetWeight() - glog.V(4).Infof("DRF NamespaceOrderFn: <%v> share state: %f, weight %v, <%v> share state: %f, weight %v", + klog.V(4).Infof("DRF NamespaceOrderFn: <%v> share state: %f, weight %v, <%v> share state: %f, weight %v", lv, lOpt.share, lWeight, rv, rOpt.share, rWeight) lWeightedShare := lOpt.share / float64(lWeight) @@ -268,7 +268,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { nsShare = nsOpt.share } - glog.V(4).Infof("DRF AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", + klog.V(4).Infof("DRF AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare) }, DeallocateFunc: func(event *framework.Event) { @@ -286,7 +286,7 @@ func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) { nsShare = nsOpt.share } - glog.V(4).Infof("DRF EvictFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", + klog.V(4).Infof("DRF EvictFunc: task <%v/%v>, resreq <%v>, share <%v>, namespace share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare) }, }) diff --git a/pkg/scheduler/plugins/gang/gang.go b/pkg/scheduler/plugins/gang/gang.go index 36e180f035..d27634d9d6 100644 --- a/pkg/scheduler/plugins/gang/gang.go +++ b/pkg/scheduler/plugins/gang/gang.go @@ -19,10 +19,9 @@ package gang import ( "fmt" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "volcano.sh/volcano/pkg/apis/scheduling" "volcano.sh/volcano/pkg/apis/scheduling/v1alpha1" @@ -82,14 +81,14 @@ func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) { preemptable := job.MinAvailable <= occupid-1 || job.MinAvailable == 1 if !preemptable { - glog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling", + klog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling", preemptee.Namespace, preemptee.Name) } else { victims = append(victims, preemptee) } } - glog.V(3).Infof("Victims from Gang plugins are %+v", victims) + klog.V(3).Infof("Victims from Gang plugins are %+v", victims) return victims } @@ -105,7 +104,7 @@ func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) { lReady := lv.Ready() rReady := rv.Ready() - glog.V(4).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t", + klog.V(4).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t", lv.Namespace, lv.Name, lReady, rv.Namespace, rv.Name, rReady) if lReady && rReady { @@ -158,7 +157,7 @@ func (gp *gangPlugin) OnSessionClose(ssn *framework.Session) { } if err := ssn.UpdateJobCondition(job, jc); err != nil { - glog.Errorf("Failed to update job <%s/%s> condition: %v", + klog.Errorf("Failed to update job <%s/%s> condition: %v", job.Namespace, job.Name, err) } diff --git a/pkg/scheduler/plugins/nodeorder/nodeorder.go b/pkg/scheduler/plugins/nodeorder/nodeorder.go index 5cda0a8865..5c4d443cc8 100644 --- a/pkg/scheduler/plugins/nodeorder/nodeorder.go +++ b/pkg/scheduler/plugins/nodeorder/nodeorder.go @@ -17,11 +17,10 @@ limitations under the License. package nodeorder import ( - "fmt" - - "github.com/golang/glog" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/api/core/v1" + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -138,10 +137,10 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("node order, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("node order, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.AddPod(pod) - glog.V(4).Infof("node order, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("node order, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) } }, DeallocateFunc: func(event *framework.Event) { @@ -150,10 +149,10 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("node order, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("node order, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.RemovePod(pod) - glog.V(4).Infof("node order, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("node order, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) } }, }) @@ -163,7 +162,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { if !found { nodeInfo = cache.NewNodeInfo(node.Pods()...) nodeInfo.SetNode(node.Node) - glog.Warningf("node order, generate node info for %s at NodeOrderFn is unexpected", node.Name) + klog.Warningf("node order, generate node info for %s at NodeOrderFn is unexpected", node.Name) } var score = 0.0 @@ -172,7 +171,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err := priorities.LeastRequestedPriorityMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Least Requested Priority Failed because of Error: %v", err) + klog.Warningf("Least Requested Priority Failed because of Error: %v", err) return 0, err } // If leastReqWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. @@ -180,7 +179,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err = priorities.BalancedResourceAllocationMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Balanced Resource Allocation Priority Failed because of Error: %v", err) + klog.Warningf("Balanced Resource Allocation Priority Failed because of Error: %v", err) return 0, err } // If balancedRescourceWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. @@ -188,13 +187,13 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { host, err = priorities.CalculateNodeAffinityPriorityMap(task.Pod, nil, nodeInfo) if err != nil { - glog.Warningf("Calculate Node Affinity Priority Failed because of Error: %v", err) + klog.Warningf("Calculate Node Affinity Priority Failed because of Error: %v", err) return 0, err } // If nodeAffinityWeight in provided, host.Score is multiplied with weight, if not, host.Score is added to total score. score = score + float64(host.Score*weight.nodeAffinityWeight) - glog.V(4).Infof("Total Score for task %s/%s on node %s is: %f", task.Namespace, task.Name, node.Name, score) + klog.V(4).Infof("Total Score for task %s/%s on node %s is: %f", task.Namespace, task.Name, node.Name, score) return score, nil } ssn.AddNodeOrderFn(pp.Name(), nodeOrderFn) @@ -205,7 +204,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { mapFn := priorities.NewInterPodAffinityPriority(cn, nl, pl, v1.DefaultHardPodAffinitySymmetricWeight) interPodAffinityScore, err := mapFn(task.Pod, nodeMap, nodeSlice) if err != nil { - glog.Warningf("Calculate Inter Pod Affinity Priority Failed because of Error: %v", err) + klog.Warningf("Calculate Inter Pod Affinity Priority Failed because of Error: %v", err) return nil, err } @@ -214,7 +213,7 @@ func (pp *nodeOrderPlugin) OnSessionOpen(ssn *framework.Session) { score[host.Host] = float64(host.Score) * float64(weight.podAffinityWeight) } - glog.V(4).Infof("Batch Total Score for task %s/%s is: %v", task.Namespace, task.Name, score) + klog.V(4).Infof("Batch Total Score for task %s/%s is: %v", task.Namespace, task.Name, score) return score, nil } ssn.AddBatchNodeOrderFn(pp.Name(), batchNodeOrderFn) @@ -238,7 +237,7 @@ func (c *cachedNodeInfo) GetNodeInfo(name string) (*v1.Node, error) { } } } - return nil, fmt.Errorf("failed to find node <%s>", name) + return nil, errors.NewNotFound(v1.Resource("node"), name) } return node.Node, nil diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index 1a8503c9b6..c22a69e9ef 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -20,8 +20,7 @@ import ( "fmt" "strings" - "github.com/golang/glog" - + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -128,10 +127,10 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.AddPod(pod) - glog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) } }, DeallocateFunc: func(event *framework.Event) { @@ -140,10 +139,10 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] if !found { - glog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) } else { node.RemovePod(pod) - glog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) } }, }) @@ -159,11 +158,11 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { if !found { nodeInfo = cache.NewNodeInfo(node.Pods()...) nodeInfo.SetNode(node.Node) - glog.Warningf("predicates, generate node info for %s at PredicateFn is unexpected", node.Name) + klog.Warningf("predicates, generate node info for %s at PredicateFn is unexpected", node.Name) } if node.Allocatable.MaxTaskNum <= len(nodeInfo.Pods()) { - glog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed", + klog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed", task.Namespace, task.Name, node.Name) return api.NewFitError(task, node, api.NodePodNumberExceeded) } @@ -174,7 +173,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeCondition predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeCondition predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -187,7 +186,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeUnschedulable Predicate Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeUnschedulable Predicate Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -200,7 +199,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("NodeSelect predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("NodeSelect predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -213,7 +212,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("HostPorts predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("HostPorts predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -226,7 +225,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("Toleration/Taint predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("Toleration/Taint predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -240,7 +239,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeMemoryPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeMemoryPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -255,7 +254,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodeDiskPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodeDiskPressure predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -270,7 +269,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("CheckNodePIDPressurePredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("CheckNodePIDPressurePredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { @@ -291,7 +290,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return err } - glog.V(4).Infof("Pod Affinity/Anti-Affinity predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + klog.V(4).Infof("Pod Affinity/Anti-Affinity predicates Task <%s/%s> on Node <%s>: fit %t, err %v", task.Namespace, task.Name, node.Name, fit, err) if !fit { diff --git a/pkg/scheduler/plugins/priority/priority.go b/pkg/scheduler/plugins/priority/priority.go index f58108e906..979a4b312a 100644 --- a/pkg/scheduler/plugins/priority/priority.go +++ b/pkg/scheduler/plugins/priority/priority.go @@ -17,7 +17,7 @@ limitations under the License. package priority import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/framework" @@ -45,7 +45,7 @@ func (pp *priorityPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.TaskInfo) rv := r.(*api.TaskInfo) - glog.V(4).Infof("Priority TaskOrder: <%v/%v> priority is %v, <%v/%v> priority is %v", + klog.V(4).Infof("Priority TaskOrder: <%v/%v> priority is %v, <%v/%v> priority is %v", lv.Namespace, lv.Name, lv.Priority, rv.Namespace, rv.Name, rv.Priority) if lv.Priority == rv.Priority { @@ -66,7 +66,7 @@ func (pp *priorityPlugin) OnSessionOpen(ssn *framework.Session) { lv := l.(*api.JobInfo) rv := r.(*api.JobInfo) - glog.V(4).Infof("Priority JobOrderFn: <%v/%v> priority: %d, <%v/%v> priority: %d", + klog.V(4).Infof("Priority JobOrderFn: <%v/%v> priority: %d, <%v/%v> priority: %d", lv.Namespace, lv.Name, lv.Priority, rv.Namespace, rv.Name, rv.Priority) if lv.Priority > rv.Priority { diff --git a/pkg/scheduler/plugins/proportion/proportion.go b/pkg/scheduler/plugins/proportion/proportion.go index 78b4778f92..cca2311a9f 100644 --- a/pkg/scheduler/plugins/proportion/proportion.go +++ b/pkg/scheduler/plugins/proportion/proportion.go @@ -17,7 +17,7 @@ limitations under the License. package proportion import ( - "github.com/golang/glog" + "k8s.io/klog" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/api/helpers" @@ -64,11 +64,11 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.totalResource.Add(n.Allocatable) } - glog.V(4).Infof("The total resource is <%v>", pp.totalResource) + klog.V(4).Infof("The total resource is <%v>", pp.totalResource) // Build attributes for Queues. for _, job := range ssn.Jobs { - glog.V(4).Infof("Considering Job <%s/%s>.", job.Namespace, job.Name) + klog.V(4).Infof("Considering Job <%s/%s>.", job.Namespace, job.Name) if _, found := pp.queueOpts[job.Queue]; !found { queue := ssn.Queues[job.Queue] @@ -82,7 +82,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { request: api.EmptyResource(), } pp.queueOpts[job.Queue] = attr - glog.V(4).Infof("Added Queue <%s> attributes.", job.Queue) + klog.V(4).Infof("Added Queue <%s> attributes.", job.Queue) } for status, tasks := range job.TaskStatusIndex { @@ -114,7 +114,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { // If no queues, break if totalWeight == 0 { - glog.V(4).Infof("Exiting when total weight is 0") + klog.V(4).Infof("Exiting when total weight is 0") break } @@ -124,7 +124,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { increasedDeserved := api.EmptyResource() decreasedDeserved := api.EmptyResource() for _, attr := range pp.queueOpts { - glog.V(4).Infof("Considering Queue <%s>: weight <%d>, total weight <%d>.", + klog.V(4).Infof("Considering Queue <%s>: weight <%d>, total weight <%d>.", attr.name, attr.weight, totalWeight) if _, found := meet[attr.queueID]; found { continue @@ -136,12 +136,12 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { if attr.request.Less(attr.deserved) { attr.deserved = helpers.Min(attr.deserved, attr.request) meet[attr.queueID] = struct{}{} - glog.V(4).Infof("queue <%s> is meet", attr.name) + klog.V(4).Infof("queue <%s> is meet", attr.name) } pp.updateShare(attr) - glog.V(4).Infof("The attributes of queue <%s> in proportion: deserved <%v>, allocate <%v>, request <%v>, share <%0.2f>", + klog.V(4).Infof("The attributes of queue <%s> in proportion: deserved <%v>, allocate <%v>, request <%v>, share <%0.2f>", attr.name, attr.deserved, attr.allocated, attr.request, attr.share) increased, decreased := attr.deserved.Diff(oldDeserved) @@ -151,7 +151,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { remaining.Sub(increasedDeserved).Add(decreasedDeserved) if remaining.IsEmpty() { - glog.V(4).Infof("Exiting when remaining is empty: <%v>", remaining) + klog.V(4).Infof("Exiting when remaining is empty: <%v>", remaining) break } } @@ -184,7 +184,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { } allocated := allocations[job.Queue] if allocated.Less(reclaimee.Resreq) { - glog.V(3).Infof("Failed to allocate resource for Task <%s/%s> in Queue <%s>, not enough resource.", + klog.V(3).Infof("Failed to allocate resource for Task <%s/%s> in Queue <%s>, not enough resource.", reclaimee.Namespace, reclaimee.Name, job.Queue) continue } @@ -204,7 +204,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { overused := !attr.allocated.LessEqual(attr.deserved) if overused { - glog.V(3).Infof("Queue <%v>: deserved <%v>, allocated <%v>, share <%v>", + klog.V(3).Infof("Queue <%v>: deserved <%v>, allocated <%v>, share <%v>", queue.Name, attr.deserved, attr.allocated, attr.share) } @@ -224,7 +224,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pgResource := api.NewResource(*job.PodGroup.Spec.MinResources) if len(queue.Queue.Spec.Capability) == 0 { - glog.V(4).Infof("Capability of queue <%s> was not set, allow job <%s/%s> to Inqueue.", + klog.V(4).Infof("Capability of queue <%s> was not set, allow job <%s/%s> to Inqueue.", queue.Name, job.Namespace, job.Name) return true } @@ -244,7 +244,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.updateShare(attr) - glog.V(4).Infof("Proportion AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>", + klog.V(4).Infof("Proportion AllocateFunc: task <%v/%v>, resreq <%v>, share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share) }, DeallocateFunc: func(event *framework.Event) { @@ -254,7 +254,7 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) { pp.updateShare(attr) - glog.V(4).Infof("Proportion EvictFunc: task <%v/%v>, resreq <%v>, share <%v>", + klog.V(4).Infof("Proportion EvictFunc: task <%v/%v>, resreq <%v>, share <%v>", event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share) }, }) diff --git a/pkg/scheduler/plugins/util/util.go b/pkg/scheduler/plugins/util/util.go index e82025f923..7c2fe823c4 100644 --- a/pkg/scheduler/plugins/util/util.go +++ b/pkg/scheduler/plugins/util/util.go @@ -17,12 +17,11 @@ limitations under the License. package util import ( - "fmt" - - "github.com/golang/glog" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/cache" @@ -98,7 +97,7 @@ func (pl *PodLister) GetPod(task *api.TaskInfo) *v1.Pod { if !found { // we could not write the copied pod back into cache for read only pod = pl.copyTaskPod(task) - glog.Warningf("DeepCopy for pod %s/%s at PodLister.GetPod is unexpected", pod.Namespace, pod.Name) + klog.Warningf("DeepCopy for pod %s/%s at PodLister.GetPod is unexpected", pod.Namespace, pod.Name) } return pod } @@ -205,7 +204,8 @@ type CachedNodeInfo struct { func (c *CachedNodeInfo) GetNodeInfo(name string) (*v1.Node, error) { node, found := c.Session.Nodes[name] if !found { - return nil, fmt.Errorf("failed to find node <%s>", name) + + return nil, errors.NewNotFound(v1.Resource("node"), name) } return node.Node, nil diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index db273e2fdc..56c49b3869 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -19,10 +19,9 @@ package scheduler import ( "time" - "github.com/golang/glog" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" + "k8s.io/klog" schedcache "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -37,6 +36,7 @@ type Scheduler struct { config *rest.Config actions []framework.Action plugins []conf.Tier + configurations []conf.Configuration schedulerConf string schedulePeriod time.Duration } @@ -69,14 +69,14 @@ func (pc *Scheduler) Run(stopCh <-chan struct{}) { } func (pc *Scheduler) runOnce() { - glog.V(4).Infof("Start scheduling ...") + klog.V(4).Infof("Start scheduling ...") scheduleStartTime := time.Now() - defer glog.V(4).Infof("End scheduling ...") + defer klog.V(4).Infof("End scheduling ...") defer metrics.UpdateE2eDuration(metrics.Duration(scheduleStartTime)) pc.loadSchedulerConf() - ssn := framework.OpenSession(pc.cache, pc.plugins) + ssn := framework.OpenSession(pc.cache, pc.plugins, pc.configurations) defer framework.CloseSession(ssn) for _, action := range pc.actions { @@ -93,13 +93,13 @@ func (pc *Scheduler) loadSchedulerConf() { schedConf := defaultSchedulerConf if len(pc.schedulerConf) != 0 { if schedConf, err = readSchedulerConf(pc.schedulerConf); err != nil { - glog.Errorf("Failed to read scheduler configuration '%s', using default configuration: %v", + klog.Errorf("Failed to read scheduler configuration '%s', using default configuration: %v", pc.schedulerConf, err) schedConf = defaultSchedulerConf } } - pc.actions, pc.plugins, err = loadSchedulerConf(schedConf) + pc.actions, pc.plugins, pc.configurations, err = loadSchedulerConf(schedConf) if err != nil { panic(err) } diff --git a/pkg/scheduler/util.go b/pkg/scheduler/util.go index 5a57906606..a831414c33 100644 --- a/pkg/scheduler/util.go +++ b/pkg/scheduler/util.go @@ -41,7 +41,7 @@ tiers: - name: nodeorder ` -func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) { +func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, []conf.Configuration, error) { var actions []framework.Action schedulerConf := &conf.SchedulerConfiguration{} @@ -50,7 +50,7 @@ func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) copy(buf, confStr) if err := yaml.Unmarshal(buf, schedulerConf); err != nil { - return nil, nil, err + return nil, nil, nil, err } // Set default settings for each plugin if not set @@ -65,11 +65,11 @@ func loadSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, error) if action, found := framework.GetAction(strings.TrimSpace(actionName)); found { actions = append(actions, action) } else { - return nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName) + return nil, nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName) } } - return actions, schedulerConf.Tiers, nil + return actions, schedulerConf.Tiers, schedulerConf.Configurations, nil } func readSchedulerConf(confPath string) (string, error) { diff --git a/pkg/scheduler/util/assert/assert.go b/pkg/scheduler/util/assert/assert.go index 0ff84f0db0..f16d8078cb 100644 --- a/pkg/scheduler/util/assert/assert.go +++ b/pkg/scheduler/util/assert/assert.go @@ -5,7 +5,7 @@ import ( "os" "runtime/debug" - "github.com/golang/glog" + "k8s.io/klog" ) const ( @@ -32,7 +32,7 @@ func Assert(condition bool, message string) { if panicOnError { panic(message) } - glog.Errorf("%s, %s", message, debug.Stack()) + klog.Errorf("%s, %s", message, debug.Stack()) } // Assertf check condition, if condition is false, print message using Assert diff --git a/pkg/scheduler/util/scheduler_helper.go b/pkg/scheduler/util/scheduler_helper.go index 8d2e7122d6..5cfce5cac4 100644 --- a/pkg/scheduler/util/scheduler_helper.go +++ b/pkg/scheduler/util/scheduler_helper.go @@ -24,7 +24,7 @@ import ( "sync" "sync/atomic" - "github.com/golang/glog" + "k8s.io/klog" "k8s.io/client-go/util/workqueue" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" @@ -84,12 +84,12 @@ func PredicateNodes(task *api.TaskInfo, nodes []*api.NodeInfo, fn api.PredicateF // to make sure all nodes have the same chance of being examined across pods. node := nodes[(lastProcessedNodeIndex+index)%allNodes] atomic.AddInt32(&processedNodes, 1) - glog.V(3).Infof("Considering Task <%v/%v> on node <%v>: <%v> vs. <%v>", + klog.V(3).Infof("Considering Task <%v/%v> on node <%v>: <%v> vs. <%v>", task.Namespace, task.Name, node.Name, task.Resreq, node.Idle) // TODO (k82cn): Enable eCache for performance improvement. if err := fn(task, node); err != nil { - glog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", + klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v", task.Namespace, task.Name, node.Name, err) errorLock.Lock() fe.SetNodeError(node.Name, err) @@ -126,7 +126,7 @@ func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.Batc node := nodes[index] mapScores, orderScore, err := mapFn(task, node) if err != nil { - glog.Errorf("Error in Calculating Priority for the node:%v", err) + klog.Errorf("Error in Calculating Priority for the node:%v", err) return } @@ -147,13 +147,13 @@ func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.Batc workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), scoreNode) reduceScores, err := reduceFn(task, pluginNodeScoreMap) if err != nil { - glog.Errorf("Error in Calculating Priority for the node:%v", err) + klog.Errorf("Error in Calculating Priority for the node:%v", err) return nodeScores } batchNodeScore, err := batchFn(task, nodes) if err != nil { - glog.Errorf("Error in Calculating batch Priority for the node, err %v", err) + klog.Errorf("Error in Calculating batch Priority for the node, err %v", err) return nodeScores } diff --git a/pkg/scheduler/util/test_utils.go b/pkg/scheduler/util/test_utils.go index 9384c3f46a..6a14189d77 100644 --- a/pkg/scheduler/util/test_utils.go +++ b/pkg/scheduler/util/test_utils.go @@ -114,10 +114,17 @@ func (fb *FakeBinder) Bind(p *v1.Pod, hostname string) error { // FakeEvictor is used as fake evictor type FakeEvictor struct { sync.Mutex - Evicts []string + evicts []string Channel chan string } +// Evicts returns copy of evicted pods. +func (fe *FakeEvictor) Evicts() []string { + fe.Lock() + defer fe.Unlock() + return append([]string{}, fe.evicts...) +} + // Evict is used by fake evictor to evict pods func (fe *FakeEvictor) Evict(p *v1.Pod) error { fe.Lock() @@ -125,7 +132,7 @@ func (fe *FakeEvictor) Evict(p *v1.Pod) error { fmt.Println("PodName: ", p.Name) key := fmt.Sprintf("%v/%v", p.Namespace, p.Name) - fe.Evicts = append(fe.Evicts, key) + fe.evicts = append(fe.evicts, key) fe.Channel <- key diff --git a/pkg/scheduler/util_test.go b/pkg/scheduler/util_test.go index cd3829c9b4..a06e077b4b 100644 --- a/pkg/scheduler/util_test.go +++ b/pkg/scheduler/util_test.go @@ -26,7 +26,11 @@ import ( func TestLoadSchedulerConf(t *testing.T) { configuration := ` -actions: "allocate, backfill" +actions: "enqueue, allocate, backfill" +configurations: +- name: enqueue + arguments: + "overcommit-factor": 1.5 tiers: - plugins: - name: priority @@ -142,7 +146,16 @@ tiers: }, } - _, tiers, err := loadSchedulerConf(configuration) + expectedConfigurations := []conf.Configuration{ + { + Name: "enqueue", + Arguments: map[string]string{ + "overcommit-factor": "1.5", + }, + }, + } + + _, tiers, configurations, err := loadSchedulerConf(configuration) if err != nil { t.Errorf("Failed to load scheduler configuration: %v", err) } @@ -150,4 +163,8 @@ tiers: t.Errorf("Failed to set default settings for plugins, expected: %+v, got %+v", expectedTiers, tiers) } + if !reflect.DeepEqual(configurations, expectedConfigurations) { + t.Errorf("Wrong configuration, expected: %+v, got %+v", + expectedConfigurations, configurations) + } } diff --git a/test/e2e/admission.go b/test/e2e/admission.go index a6a631935c..02531eecb7 100644 --- a/test/e2e/admission.go +++ b/test/e2e/admission.go @@ -34,7 +34,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Default queue would be added", func() { jobName := "job-default-queue" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, err := createJobInner(context, &jobSpec{ @@ -60,7 +60,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Invalid CPU unit", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" @@ -109,7 +109,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Invalid memory unit", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" @@ -160,7 +160,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { It("Create default-scheduler pod", func() { podName := "pod-default-scheduler" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) pod := &corev1.Pod{ @@ -188,7 +188,7 @@ var _ = Describe("Job E2E Test: Test Admission service", func() { podName := "pod-volcano" pgName := "pending-pg" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) pg := &schedulingv1alpha2.PodGroup{ diff --git a/test/e2e/cli_util.go b/test/e2e/cli_util.go index cc69af6eb5..b4d8bd3b6e 100644 --- a/test/e2e/cli_util.go +++ b/test/e2e/cli_util.go @@ -72,7 +72,10 @@ func RunCliCommand(command []string) string { command = append(command, "--master", masterURL()) } command = append(command, "--kubeconfig", kubeconfigPath(homeDir())) - output, err := exec.Command(VolcanoCliBinary(), command...).Output() + vcctl := VolcanoCliBinary() + Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( + "vcctl binary: %s is required for E2E tests, please update VC_BIN environment", vcctl)) + output, err := exec.Command(vcctl, command...).Output() Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Command %s failed to execute: %s", strings.Join(command, ""), err)) return string(output) @@ -83,8 +86,10 @@ func RunCliCommandWithoutKubeConfig(command []string) string { if masterURL() != "" { command = append(command, "--master", masterURL()) } - - output, err := exec.Command(VolcanoCliBinary(), command...).Output() + vcctl := VolcanoCliBinary() + Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( + "vcctl binary: %s is required for E2E tests, please update VC_BIN environment", vcctl)) + output, err := exec.Command(vcctl, command...).Output() Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Command %s failed to execute: %s", strings.Join(command, ""), err)) return string(output) diff --git a/test/e2e/command.go b/test/e2e/command.go index aee135e681..8c5755f742 100644 --- a/test/e2e/command.go +++ b/test/e2e/command.go @@ -36,7 +36,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { var outBuffer bytes.Buffer jobName := "test-job" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -70,7 +70,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { jobName := "test-suspend-running-job" taskName := "long-live-task" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) job := createJob(context, &jobSpec{ @@ -113,7 +113,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { }) It("Suspend pending job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -157,7 +157,7 @@ var _ = Describe("Job E2E Test: Test Job Command", func() { jobName := "test-del-job" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) diff --git a/test/e2e/job_controlled_resource.go b/test/e2e/job_controlled_resource.go index 4f173719a3..9f1d094eae 100644 --- a/test/e2e/job_controlled_resource.go +++ b/test/e2e/job_controlled_resource.go @@ -26,54 +26,13 @@ import ( ) var _ = Describe("Job E2E Test: Test Job PVCs", func() { - It("Generate PVC name if not specified", func() { - jobName := "job-pvc-name-empty" - namespace := "test" - taskName := "task" - pvcName := "specifiedpvcname" - context := initTestContext() - defer cleanupTestContext(context) - - job := createJob(context, &jobSpec{ - namespace: namespace, - name: jobName, - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: oneCPU, - min: 1, - rep: 1, - name: taskName, - }, - }, - volumes: []v1alpha1.VolumeSpec{ - { - MountPath: "/mounttwo", - }, - }, - }) - - err := waitJobReady(context, job) - Expect(err).NotTo(HaveOccurred()) - - job, err = context.vcclient.BatchV1alpha1().Jobs(namespace).Get(jobName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Expect(len(job.Spec.Volumes)).To(Equal(1), - "Two volumes should be created") - for _, volume := range job.Spec.Volumes { - Expect(volume.VolumeClaimName).Should(Or(ContainSubstring(jobName), Equal(pvcName)), - "PVC name should be generated for manually specified.") - } - }) - It("use exisisting PVC in job", func() { jobName := "job-pvc-name-exist" namespace := "test" taskName := "pvctask" pvName := "job-pv-name" pvcName := "job-pvc-name-exist" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) var tt v12.HostPathType @@ -177,7 +136,7 @@ var _ = Describe("Job E2E Test: Test Job PVCs", func() { It("Generate PodGroup and valid minResource when creating job", func() { jobName := "job-name-podgroup" namespace := "test" - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) resource := v12.ResourceList{ diff --git a/test/e2e/job_error_handling.go b/test/e2e/job_error_handling.go index 7bc72c1837..7be56559f4 100644 --- a/test/e2e/job_error_handling.go +++ b/test/e2e/job_error_handling.go @@ -17,9 +17,10 @@ limitations under the License. package e2e import ( + "strconv" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "strconv" "k8s.io/api/core/v1" @@ -31,7 +32,7 @@ import ( var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -68,7 +69,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -105,7 +106,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: AbortJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -142,7 +143,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -186,7 +187,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -230,7 +231,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodEvicted; Action: AbortJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -274,7 +275,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: Any; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -318,7 +319,7 @@ var _ = Describe("Job Error Handling", func() { It("Job error handling: Restart job when job is unschedulable", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -377,7 +378,7 @@ var _ = Describe("Job Error Handling", func() { It("Job error handling: Abort job when job is unschedulable", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -432,7 +433,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: TaskCompleted; Action: CompletedJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -472,7 +473,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, error code: 3; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -510,7 +511,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event[]: PodEvicted, PodFailed; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -556,7 +557,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -592,7 +593,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodEvicted; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -636,7 +637,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: PodEvicted; Action: TerminateJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -679,7 +680,7 @@ var _ = Describe("Job Error Handling", func() { }) It("Task level LifecyclePolicy, Event: TaskCompleted; Action: CompletedJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -719,7 +720,7 @@ var _ = Describe("Job Error Handling", func() { It("job level LifecyclePolicy, Event: PodFailed; Action: AbortJob and Task level lifecyclePolicy, Event : PodFailed; Action: RestartJob", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -762,7 +763,9 @@ var _ = Describe("Job Error Handling", func() { It("Task Priority", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{ + priorityClasses: []string{masterPriority, workerPriority}, + }) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) diff --git a/test/e2e/job_lifecycle.go b/test/e2e/job_lifecycle.go index 17b750ae27..c01d1cef79 100644 --- a/test/e2e/job_lifecycle.go +++ b/test/e2e/job_lifecycle.go @@ -30,7 +30,7 @@ import ( var _ = Describe("Job Life Cycle", func() { It("Delete job that is pending state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -62,7 +62,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Running state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -93,7 +93,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Completed state", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -126,7 +126,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is Failed job", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -165,7 +165,7 @@ var _ = Describe("Job Life Cycle", func() { It("Delete job that is terminated job", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -204,7 +204,7 @@ var _ = Describe("Job Life Cycle", func() { It("Create and Delete job with CPU requirement", func() { By("init test context") - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) By("create job") @@ -243,7 +243,7 @@ var _ = Describe("Job Life Cycle", func() { }) It("Checking Event Generation for job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) job := createJob(context, &jobSpec{ @@ -271,7 +271,7 @@ var _ = Describe("Job Life Cycle", func() { }) It("Checking Unschedulable Event Generation for job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nodeName, rep := computeNode(context, oneCPU) diff --git a/test/e2e/job_plugins.go b/test/e2e/job_plugins.go index 24f688bae3..d8b4204f25 100644 --- a/test/e2e/job_plugins.go +++ b/test/e2e/job_plugins.go @@ -33,7 +33,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { namespace := "test" taskName := "task" foundVolume := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nodeName, rep := computeNode(context, oneCPU) @@ -105,7 +105,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { namespace := "test" taskName := "task" foundVolume := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, rep := computeNode(context, oneCPU) @@ -178,7 +178,7 @@ var _ = Describe("Job E2E Test: Test Job Plugins", func() { taskName := "task" foundVolume := false foundEnv := false - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) _, rep := computeNode(context, oneCPU) diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index 6e45c27b56..b1379afc80 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -36,7 +36,7 @@ import ( var _ = Describe("Job E2E Test", func() { It("Schedule Job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -57,7 +57,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule Multiple Jobs", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -91,7 +91,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Gang scheduling", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU)/2 + 1 @@ -128,7 +128,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Gang scheduling: Full Occupied", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rep := clusterSize(context, oneCPU) @@ -159,7 +159,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Preemption", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -191,7 +191,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Multiple Preemption", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -232,7 +232,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule BestEffort Job", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -262,7 +262,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Statement", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -299,7 +299,7 @@ var _ = Describe("Job E2E Test", func() { }) It("support binpack policy", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -382,7 +382,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule v1.Job type using Volcano scheduler", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" parallel := int32(2) @@ -418,7 +418,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Schedule v1.Job type using Volcano scheduler with error case", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) namespace := "test" parallel := int32(2) @@ -479,7 +479,7 @@ var _ = Describe("Job E2E Test", func() { }) It("Namespace Fair Share", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) const fairShareNamespace = "fairshare" @@ -585,7 +585,9 @@ var _ = Describe("Job E2E Test", func() { }) It("Queue Fair Share", func() { - context := initTestContext() + context := initTestContext(options{ + queues: []string{defaultQueue1, defaultQueue2}, + }) defer cleanupTestContext(context) slot := halfCPU diff --git a/test/e2e/mpi.go b/test/e2e/mpi.go index 4ba32c7112..1549210509 100644 --- a/test/e2e/mpi.go +++ b/test/e2e/mpi.go @@ -25,7 +25,7 @@ import ( var _ = Describe("MPI E2E Test", func() { It("will run and complete finally", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU diff --git a/test/e2e/pg_controller.go b/test/e2e/pg_controller.go index 8f3490230a..ac82cfd12b 100644 --- a/test/e2e/pg_controller.go +++ b/test/e2e/pg_controller.go @@ -30,7 +30,7 @@ var _ = Describe("PG E2E Test: Test PG controller", func() { podName := "pod-volcano" namespace := "test" label := map[string]string{"schedulerName": "volcano"} - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rc := &corev1.ReplicationController{ @@ -85,7 +85,7 @@ var _ = Describe("PG E2E Test: Test PG controller", func() { podName := "pod-default-scheduler" namespace := "test" label := map[string]string{"a": "b"} - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) rc := &corev1.ReplicationController{ diff --git a/test/e2e/predicates.go b/test/e2e/predicates.go index 1a106587d8..7d0e3d0b98 100644 --- a/test/e2e/predicates.go +++ b/test/e2e/predicates.go @@ -27,7 +27,7 @@ import ( var _ = Describe("Predicates E2E Test", func() { It("Hostport", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) nn := clusterNodeNumber(context) @@ -55,7 +55,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("NodeAffinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -104,7 +104,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Pod Affinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -153,7 +153,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Pod Anti-Affinity", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) slot := oneCPU @@ -203,7 +203,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Taints", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) taints := []v1.Taint{ @@ -242,7 +242,7 @@ var _ = Describe("Predicates E2E Test", func() { }) It("Taints and Tolerations", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) taints := []v1.Taint{ diff --git a/test/e2e/queue.go b/test/e2e/queue.go index cbdc46366a..dd7243b6e3 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -27,7 +27,9 @@ import ( var _ = Describe("Queue E2E Test", func() { It("Reclaim", func() { - context := initTestContext() + context := initTestContext(options{ + queues: []string{defaultQueue1, defaultQueue2}, + }) defer cleanupTestContext(context) slot := oneCPU diff --git a/test/e2e/tensorflow.go b/test/e2e/tensorflow.go index 50851dc90a..947e24c899 100644 --- a/test/e2e/tensorflow.go +++ b/test/e2e/tensorflow.go @@ -28,7 +28,7 @@ import ( var _ = Describe("TensorFlow E2E Test", func() { It("Will Start in pending state and goes through other phases to get complete phase", func() { - context := initTestContext() + context := initTestContext(options{}) defer cleanupTestContext(context) jobName := "tensorflow-dist-mnist" diff --git a/test/e2e/util.go b/test/e2e/util.go index f89d739346..ebf3be6693 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -97,9 +97,9 @@ func kubeconfigPath(home string) string { return filepath.Join(home, ".kube", "config") // default kubeconfig path is $HOME/.kube/config } -//VolcanoCliBinary function gets the volcano cli binary +// VolcanoCliBinary function gets the volcano cli binary func VolcanoCliBinary() string { - if bin := os.Getenv("VK_BIN"); bin != "" { + if bin := os.Getenv("VC_BIN"); bin != "" { return filepath.Join(bin, "vcctl") } return "" @@ -109,62 +109,49 @@ type context struct { kubeclient *kubernetes.Clientset vcclient *vcclient.Clientset - namespace string - queues []string + namespace string + queues []string + priorityClasses []string +} + +type options struct { + namespace string + queues []string + priorityClasses []string } -func initTestContext() *context { - cxt := &context{ - namespace: defaultNamespace, - queues: []string{defaultQueue1, defaultQueue2}, +func initTestContext(o options) *context { + if o.namespace == "" { + o.namespace = defaultNamespace + } + ctx := &context{ + namespace: o.namespace, + queues: o.queues, + priorityClasses: o.priorityClasses, } home := homeDir() Expect(home).NotTo(Equal("")) configPath := kubeconfigPath(home) Expect(configPath).NotTo(Equal("")) - vcctl := VolcanoCliBinary() - Expect(fileExist(vcctl)).To(BeTrue(), fmt.Sprintf( - "vcctl binary: %s is required for E2E tests, please update VK_BIN environment", vcctl)) + config, err := clientcmd.BuildConfigFromFlags(masterURL(), configPath) Expect(err).NotTo(HaveOccurred()) - cxt.vcclient = vcclient.NewForConfigOrDie(config) - cxt.kubeclient = kubernetes.NewForConfigOrDie(config) - - //Ensure at least one worker is ready - err = waitClusterReady(cxt) - Expect(err).NotTo(HaveOccurred(), - "k8s cluster is required to have one ready worker node at least.") + ctx.vcclient = vcclient.NewForConfigOrDie(config) + ctx.kubeclient = kubernetes.NewForConfigOrDie(config) - _, err = cxt.kubeclient.CoreV1().Namespaces().Create(&v1.Namespace{ + _, err = ctx.kubeclient.CoreV1().Namespaces().Create(&v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: cxt.namespace, + Name: ctx.namespace, }, }) Expect(err).NotTo(HaveOccurred()) - createQueues(cxt) + createQueues(ctx) + createPriorityClasses(ctx) - _, err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: masterPriority, - }, - Value: 100, - GlobalDefault: false, - }) - Expect(err).NotTo(HaveOccurred()) - - _, err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: workerPriority, - }, - Value: 1, - GlobalDefault: false, - }) - Expect(err).NotTo(HaveOccurred()) - - return cxt + return ctx } func namespaceNotExist(ctx *context) wait.ConditionFunc { @@ -181,20 +168,6 @@ func namespaceNotExistWithName(ctx *context, name string) wait.ConditionFunc { } } -func queueNotExist(ctx *context) wait.ConditionFunc { - return func() (bool, error) { - for _, q := range ctx.queues { - var err error - _, err = ctx.vcclient.SchedulingV1alpha2().Queues().Get(q, metav1.GetOptions{}) - if !(err != nil && errors.IsNotFound(err)) { - return false, err - } - } - - return true, nil - } -} - func fileExist(name string) bool { if _, err := os.Stat(name); err != nil { if os.IsNotExist(err) { @@ -204,40 +177,25 @@ func fileExist(name string) bool { return true } -func cleanupTestContext(cxt *context) { +func cleanupTestContext(ctx *context) { foreground := metav1.DeletePropagationForeground - - err := cxt.kubeclient.CoreV1().Namespaces().Delete(cxt.namespace, &metav1.DeleteOptions{ + err := ctx.kubeclient.CoreV1().Namespaces().Delete(ctx.namespace, &metav1.DeleteOptions{ PropagationPolicy: &foreground, }) Expect(err).NotTo(HaveOccurred()) - deleteQueues(cxt) - - err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(masterPriority, &metav1.DeleteOptions{ - PropagationPolicy: &foreground, - }) - Expect(err).NotTo(HaveOccurred()) + deleteQueues(ctx) - err = cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(workerPriority, &metav1.DeleteOptions{ - PropagationPolicy: &foreground, - }) - Expect(err).NotTo(HaveOccurred()) + deletePriorityClasses(ctx) // Wait for namespace deleted. - err = wait.Poll(100*time.Millisecond, twoMinute, namespaceNotExist(cxt)) - Expect(err).NotTo(HaveOccurred()) - - // Wait for queues deleted - err = wait.Poll(100*time.Millisecond, twoMinute, queueNotExist(cxt)) + err = wait.Poll(100*time.Millisecond, twoMinute, namespaceNotExist(ctx)) Expect(err).NotTo(HaveOccurred()) } func createQueues(cxt *context) { - var err error - for _, q := range cxt.queues { - _, err = cxt.vcclient.SchedulingV1alpha2().Queues().Create(&schedulingv1alpha2.Queue{ + _, err := cxt.vcclient.SchedulingV1alpha2().Queues().Create(&schedulingv1alpha2.Queue{ ObjectMeta: metav1.ObjectMeta{ Name: q, }, @@ -262,6 +220,26 @@ func deleteQueues(cxt *context) { } } +func createPriorityClasses(cxt *context) { + for _, pc := range cxt.priorityClasses { + _, err := cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Create(&schedv1.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: pc, + }, + Value: 100, + GlobalDefault: false, + }) + Expect(err).NotTo(HaveOccurred()) + } +} + +func deletePriorityClasses(cxt *context) { + for _, pc := range cxt.priorityClasses { + err := cxt.kubeclient.SchedulingV1beta1().PriorityClasses().Delete(pc, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } +} + type taskSpec struct { name string min, rep int32 diff --git a/test/e2e/vcctl.go b/test/e2e/vcctl.go index 0e83575d0e..fdfa5a1a63 100644 --- a/test/e2e/vcctl.go +++ b/test/e2e/vcctl.go @@ -106,7 +106,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("Command: vkctl job suspend -n {$JobName} --help", func() { + It("Command: vcctl job suspend -n {$JobName} --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` abort a job @@ -131,7 +131,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("vkctl job resume -n {$JobName} --help", func() { + It("vcctl job resume -n {$JobName} --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` resume a job @@ -156,7 +156,7 @@ Global Flags: Expect(exist).Should(Equal(true)) }) - It("vkctl job run --help", func() { + It("vcctl job run --help", func() { kubeConfig := os.Getenv("KUBECONFIG") var output = ` run job by parameters from the command line diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/github.com/golang/glog/LICENSE deleted file mode 100644 index 37ec93a14f..0000000000 --- a/vendor/github.com/golang/glog/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/golang/glog/README b/vendor/github.com/golang/glog/README deleted file mode 100644 index 387b4eb689..0000000000 --- a/vendor/github.com/golang/glog/README +++ /dev/null @@ -1,44 +0,0 @@ -glog -==== - -Leveled execution logs for Go. - -This is an efficient pure Go implementation of leveled logs in the -manner of the open source C++ package - https://github.com/google/glog - -By binding methods to booleans it is possible to use the log package -without paying the expense of evaluating the arguments to the log. -Through the -vmodule flag, the package also provides fine-grained -control over logging at the file level. - -The comment from glog.go introduces the ideas: - - Package glog implements logging analogous to the Google-internal - C++ INFO/ERROR/V setup. It provides functions Info, Warning, - Error, Fatal, plus formatting variants such as Infof. It - also provides V-style logging controlled by the -v and - -vmodule=file=2 flags. - - Basic examples: - - glog.Info("Prepare to repel boarders") - - glog.Fatalf("Initialization failed: %s", err) - - See the documentation for the V function for an explanation - of these examples: - - if glog.V(2) { - glog.Info("Starting transaction...") - } - - glog.V(2).Infoln("Processed", nItems, "elements") - - -The repository contains an open source version of the log package -used inside Google. The master copy of the source lives inside -Google, not here. The code in this repo is for export only and is not itself -under development. Feature requests will be ignored. - -Send bug reports to golang-nuts@googlegroups.com. diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/github.com/golang/glog/glog.go deleted file mode 100644 index 54bd7afdca..0000000000 --- a/vendor/github.com/golang/glog/glog.go +++ /dev/null @@ -1,1180 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. -// It provides functions Info, Warning, Error, Fatal, plus formatting variants such as -// Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. -// -// Basic examples: -// -// glog.Info("Prepare to repel boarders") -// -// glog.Fatalf("Initialization failed: %s", err) -// -// See the documentation for the V function for an explanation of these examples: -// -// if glog.V(2) { -// glog.Info("Starting transaction...") -// } -// -// glog.V(2).Infoln("Processed", nItems, "elements") -// -// Log output is buffered and written periodically using Flush. Programs -// should call Flush before exiting to guarantee all log output is written. -// -// By default, all log statements write to files in a temporary directory. -// This package provides several flags that modify this behavior. -// As a result, flag.Parse must be called before any logging is done. -// -// -logtostderr=false -// Logs are written to standard error instead of to files. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. -// -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". -// -package glog - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - stdLog "log" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// severity identifies the sort of log: info, warning etc. It also implements -// the flag.Value interface. The -stderrthreshold flag is of type severity and -// should be modified only through the flag.Value interface. The values match -// the corresponding constants in C++. -type severity int32 // sync/atomic int32 - -// These constants identify the log levels in order of increasing severity. -// A message written to a high-severity log file is also written to each -// lower-severity log file. -const ( - infoLog severity = iota - warningLog - errorLog - fatalLog - numSeverity = 4 -) - -const severityChar = "IWEF" - -var severityName = []string{ - infoLog: "INFO", - warningLog: "WARNING", - errorLog: "ERROR", - fatalLog: "FATAL", -} - -// get returns the value of the severity. -func (s *severity) get() severity { - return severity(atomic.LoadInt32((*int32)(s))) -} - -// set sets the value of the severity. -func (s *severity) set(val severity) { - atomic.StoreInt32((*int32)(s), int32(val)) -} - -// String is part of the flag.Value interface. -func (s *severity) String() string { - return strconv.FormatInt(int64(*s), 10) -} - -// Get is part of the flag.Value interface. -func (s *severity) Get() interface{} { - return *s -} - -// Set is part of the flag.Value interface. -func (s *severity) Set(value string) error { - var threshold severity - // Is it a known name? - if v, ok := severityByName(value); ok { - threshold = v - } else { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - threshold = severity(v) - } - logging.stderrThreshold.set(threshold) - return nil -} - -func severityByName(s string) (severity, bool) { - s = strings.ToUpper(s) - for i, name := range severityName { - if name == s { - return severity(i), true - } - } - return 0, false -} - -// OutputStats tracks the number of output lines and bytes written. -type OutputStats struct { - lines int64 - bytes int64 -} - -// Lines returns the number of lines written. -func (s *OutputStats) Lines() int64 { - return atomic.LoadInt64(&s.lines) -} - -// Bytes returns the number of bytes written. -func (s *OutputStats) Bytes() int64 { - return atomic.LoadInt64(&s.bytes) -} - -// Stats tracks the number of lines of output and number of bytes -// per severity level. Values must be read with atomic.LoadInt64. -var Stats struct { - Info, Warning, Error OutputStats -} - -var severityStats = [numSeverity]*OutputStats{ - infoLog: &Stats.Info, - warningLog: &Stats.Warning, - errorLog: &Stats.Error, -} - -// Level is exported because it appears in the arguments to V and is -// the type of the v flag, which can be set programmatically. -// It's a distinct type because we want to discriminate it from logType. -// Variables of type level are only changed under logging.mu. -// The -v flag is read only with atomic ops, so the state of the logging -// module is consistent. - -// Level is treated as a sync/atomic int32. - -// Level specifies a level of verbosity for V logs. *Level implements -// flag.Value; the -v flag is of type Level and should be modified -// only through the flag.Value interface. -type Level int32 - -// get returns the value of the Level. -func (l *Level) get() Level { - return Level(atomic.LoadInt32((*int32)(l))) -} - -// set sets the value of the Level. -func (l *Level) set(val Level) { - atomic.StoreInt32((*int32)(l), int32(val)) -} - -// String is part of the flag.Value interface. -func (l *Level) String() string { - return strconv.FormatInt(int64(*l), 10) -} - -// Get is part of the flag.Value interface. -func (l *Level) Get() interface{} { - return *l -} - -// Set is part of the flag.Value interface. -func (l *Level) Set(value string) error { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(Level(v), logging.vmodule.filter, false) - return nil -} - -// moduleSpec represents the setting of the -vmodule flag. -type moduleSpec struct { - filter []modulePat -} - -// modulePat contains a filter for the -vmodule flag. -// It holds a verbosity level and a file pattern to match. -type modulePat struct { - pattern string - literal bool // The pattern is a literal string - level Level -} - -// match reports whether the file matches the pattern. It uses a string -// comparison if the pattern contains no metacharacters. -func (m *modulePat) match(file string) bool { - if m.literal { - return file == m.pattern - } - match, _ := filepath.Match(m.pattern, file) - return match -} - -func (m *moduleSpec) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - var b bytes.Buffer - for i, f := range m.filter { - if i > 0 { - b.WriteRune(',') - } - fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) - } - return b.String() -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported. -func (m *moduleSpec) Get() interface{} { - return nil -} - -var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") - -// Syntax: -vmodule=recordio=2,file=1,gfs*=3 -func (m *moduleSpec) Set(value string) error { - var filter []modulePat - for _, pat := range strings.Split(value, ",") { - if len(pat) == 0 { - // Empty strings such as from a trailing comma can be ignored. - continue - } - patLev := strings.Split(pat, "=") - if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { - return errVmoduleSyntax - } - pattern := patLev[0] - v, err := strconv.Atoi(patLev[1]) - if err != nil { - return errors.New("syntax error: expect comma-separated list of filename=N") - } - if v < 0 { - return errors.New("negative value for vmodule level") - } - if v == 0 { - continue // Ignore. It's harmless but no point in paying the overhead. - } - // TODO: check syntax of filter? - filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(logging.verbosity, filter, true) - return nil -} - -// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters -// that require filepath.Match to be called to match the pattern. -func isLiteral(pattern string) bool { - return !strings.ContainsAny(pattern, `\*?[]`) -} - -// traceLocation represents the setting of the -log_backtrace_at flag. -type traceLocation struct { - file string - line int -} - -// isSet reports whether the trace location has been specified. -// logging.mu is held. -func (t *traceLocation) isSet() bool { - return t.line > 0 -} - -// match reports whether the specified file and line matches the trace location. -// The argument file name is the full path, not the basename specified in the flag. -// logging.mu is held. -func (t *traceLocation) match(file string, line int) bool { - if t.line != line { - return false - } - if i := strings.LastIndex(file, "/"); i >= 0 { - file = file[i+1:] - } - return t.file == file -} - -func (t *traceLocation) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - return fmt.Sprintf("%s:%d", t.file, t.line) -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported -func (t *traceLocation) Get() interface{} { - return nil -} - -var errTraceSyntax = errors.New("syntax error: expect file.go:234") - -// Syntax: -log_backtrace_at=gopherflakes.go:234 -// Note that unlike vmodule the file extension is included here. -func (t *traceLocation) Set(value string) error { - if value == "" { - // Unset. - t.line = 0 - t.file = "" - } - fields := strings.Split(value, ":") - if len(fields) != 2 { - return errTraceSyntax - } - file, line := fields[0], fields[1] - if !strings.Contains(file, ".") { - return errTraceSyntax - } - v, err := strconv.Atoi(line) - if err != nil { - return errTraceSyntax - } - if v <= 0 { - return errors.New("negative or zero value for level") - } - logging.mu.Lock() - defer logging.mu.Unlock() - t.line = v - t.file = file - return nil -} - -// flushSyncWriter is the interface satisfied by logging destinations. -type flushSyncWriter interface { - Flush() error - Sync() error - io.Writer -} - -func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - - // Default stderrThreshold is ERROR. - logging.stderrThreshold = errorLog - - logging.setVState(0, nil, false) - go logging.flushDaemon() -} - -// Flush flushes all pending log I/O. -func Flush() { - logging.lockAndFlushAll() -} - -// loggingT collects all the global state of the logging setup. -type loggingT struct { - // Boolean flags. Not handled atomically because the flag.Value interface - // does not let us avoid the =true, and that shorthand is necessary for - // compatibility. TODO: does this matter enough to fix? Seems unlikely. - toStderr bool // The -logtostderr flag. - alsoToStderr bool // The -alsologtostderr flag. - - // Level flag. Handled atomically. - stderrThreshold severity // The -stderrthreshold flag. - - // freeList is a list of byte buffers, maintained under freeListMu. - freeList *buffer - // freeListMu maintains the free list. It is separate from the main mutex - // so buffers can be grabbed and printed to without holding the main lock, - // for better parallelization. - freeListMu sync.Mutex - - // mu protects the remaining elements of this structure and is - // used to synchronize logging. - mu sync.Mutex - // file holds writer for each of the log types. - file [numSeverity]flushSyncWriter - // pcs is used in V to avoid an allocation when computing the caller's PC. - pcs [1]uintptr - // vmap is a cache of the V Level for each V() call site, identified by PC. - // It is wiped whenever the vmodule flag changes state. - vmap map[uintptr]Level - // filterLength stores the length of the vmodule filter chain. If greater - // than zero, it means vmodule is enabled. It may be read safely - // using sync.LoadInt32, but is only modified under mu. - filterLength int32 - // traceLocation is the state of the -log_backtrace_at flag. - traceLocation traceLocation - // These flags are modified only under lock, although verbosity may be fetched - // safely using atomic.LoadInt32. - vmodule moduleSpec // The state of the -vmodule flag. - verbosity Level // V logging level, the value of the -v flag/ -} - -// buffer holds a byte Buffer for reuse. The zero value is ready for use. -type buffer struct { - bytes.Buffer - tmp [64]byte // temporary byte array for creating headers. - next *buffer -} - -var logging loggingT - -// setVState sets a consistent state for V logging. -// l.mu is held. -func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { - // Turn verbosity off so V will not fire while we are in transition. - logging.verbosity.set(0) - // Ditto for filter length. - atomic.StoreInt32(&logging.filterLength, 0) - - // Set the new filters and wipe the pc->Level map if the filter has changed. - if setFilter { - logging.vmodule.filter = filter - logging.vmap = make(map[uintptr]Level) - } - - // Things are consistent now, so enable filtering and verbosity. - // They are enabled in order opposite to that in V. - atomic.StoreInt32(&logging.filterLength, int32(len(filter))) - logging.verbosity.set(verbosity) -} - -// getBuffer returns a new, ready-to-use buffer. -func (l *loggingT) getBuffer() *buffer { - l.freeListMu.Lock() - b := l.freeList - if b != nil { - l.freeList = b.next - } - l.freeListMu.Unlock() - if b == nil { - b = new(buffer) - } else { - b.next = nil - b.Reset() - } - return b -} - -// putBuffer returns a buffer to the free list. -func (l *loggingT) putBuffer(b *buffer) { - if b.Len() >= 256 { - // Let big buffers die a natural death. - return - } - l.freeListMu.Lock() - b.next = l.freeList - l.freeList = b - l.freeListMu.Unlock() -} - -var timeNow = time.Now // Stubbed out for testing. - -/* -header formats a log header as defined by the C++ implementation. -It returns a buffer containing the formatted header and the user's file and line number. -The depth specifies how many stack frames above lives the source line to be identified in the log message. - -Log lines have this form: - Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... -where the fields are defined as follows: - L A single character, representing the log level (eg 'I' for INFO) - mm The month (zero padded; ie May is '05') - dd The day (zero padded) - hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds - threadid The space-padded thread ID as returned by GetTID() - file The file name - line The line number - msg The user-supplied message -*/ -func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { - _, file, line, ok := runtime.Caller(3 + depth) - if !ok { - file = "???" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] - } - } - return l.formatHeader(s, file, line), file, line -} - -// formatHeader formats a log header using the provided file name and line number. -func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { - now := timeNow() - if line < 0 { - line = 0 // not a real line number, but acceptable to someDigits - } - if s > fatalLog { - s = infoLog // for safety. - } - buf := l.getBuffer() - - // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. - // It's worth about 3X. Fprintf is hard. - _, month, day := now.Date() - hour, minute, second := now.Clock() - // Lmmdd hh:mm:ss.uuuuuu threadid file:line] - buf.tmp[0] = severityChar[s] - buf.twoDigits(1, int(month)) - buf.twoDigits(3, day) - buf.tmp[5] = ' ' - buf.twoDigits(6, hour) - buf.tmp[8] = ':' - buf.twoDigits(9, minute) - buf.tmp[11] = ':' - buf.twoDigits(12, second) - buf.tmp[14] = '.' - buf.nDigits(6, 15, now.Nanosecond()/1000, '0') - buf.tmp[21] = ' ' - buf.nDigits(7, 22, pid, ' ') // TODO: should be TID - buf.tmp[29] = ' ' - buf.Write(buf.tmp[:30]) - buf.WriteString(file) - buf.tmp[0] = ':' - n := buf.someDigits(1, line) - buf.tmp[n+1] = ']' - buf.tmp[n+2] = ' ' - buf.Write(buf.tmp[:n+3]) - return buf -} - -// Some custom tiny helper functions to print the log header efficiently. - -const digits = "0123456789" - -// twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. -func (buf *buffer) twoDigits(i, d int) { - buf.tmp[i+1] = digits[d%10] - d /= 10 - buf.tmp[i] = digits[d%10] -} - -// nDigits formats an n-digit integer at buf.tmp[i], -// padding with pad on the left. -// It assumes d >= 0. -func (buf *buffer) nDigits(n, i, d int, pad byte) { - j := n - 1 - for ; j >= 0 && d > 0; j-- { - buf.tmp[i+j] = digits[d%10] - d /= 10 - } - for ; j >= 0; j-- { - buf.tmp[i+j] = pad - } -} - -// someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. -func (buf *buffer) someDigits(i, d int) int { - // Print into the top, then copy down. We know there's space for at least - // a 10-digit number. - j := len(buf.tmp) - for { - j-- - buf.tmp[j] = digits[d%10] - d /= 10 - if d == 0 { - break - } - } - return copy(buf.tmp[i:], buf.tmp[j:]) -} - -func (l *loggingT) println(s severity, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintln(buf, args...) - l.output(s, buf, file, line, false) -} - -func (l *loggingT) print(s severity, args ...interface{}) { - l.printDepth(s, 1, args...) -} - -func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { - buf, file, line := l.header(s, depth) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -func (l *loggingT) printf(s severity, format string, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintf(buf, format, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -// printWithFileLine behaves like print but uses the provided file and line number. If -// alsoLogToStderr is true, the log message always appears on standard error; it -// will also appear in the log file unless --logtostderr is set. -func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { - buf := l.formatHeader(s, file, line) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, alsoToStderr) -} - -// output writes the data to the log files and releases the buffer. -func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { - l.mu.Lock() - if l.traceLocation.isSet() { - if l.traceLocation.match(file, line) { - buf.Write(stacks(false)) - } - } - data := buf.Bytes() - if !flag.Parsed() { - os.Stderr.Write([]byte("ERROR: logging before flag.Parse: ")) - os.Stderr.Write(data) - } else if l.toStderr { - os.Stderr.Write(data) - } else { - if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { - os.Stderr.Write(data) - } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) - } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: - l.file[infoLog].Write(data) - } - } - if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. - } - l.putBuffer(buf) - l.mu.Unlock() - if stats := severityStats[s]; stats != nil { - atomic.AddInt64(&stats.lines, 1) - atomic.AddInt64(&stats.bytes, int64(len(data))) - } -} - -// timeoutFlush calls Flush and returns when it completes or after timeout -// elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds -// a lock. -func timeoutFlush(timeout time.Duration) { - done := make(chan bool, 1) - go func() { - Flush() // calls logging.lockAndFlushAll() - done <- true - }() - select { - case <-done: - case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) - } -} - -// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. -func stacks(all bool) []byte { - // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. - n := 10000 - if all { - n = 100000 - } - var trace []byte - for i := 0; i < 5; i++ { - trace = make([]byte, n) - nbytes := runtime.Stack(trace, all) - if nbytes < len(trace) { - return trace[:nbytes] - } - n *= 2 - } - return trace -} - -// logExitFunc provides a simple mechanism to override the default behavior -// of exiting on error. Used in testing and to guarantee we reach a required exit -// for fatal logs. Instead, exit could be a function rather than a method but that -// would make its use clumsier. -var logExitFunc func(error) - -// exit is called if there is trouble creating or writing log files. -// It flushes the logs and exits the program; there's no point in hanging around. -// l.mu is held. -func (l *loggingT) exit(err error) { - fmt.Fprintf(os.Stderr, "log: exiting because of error: %s\n", err) - // If logExitFunc is set, we do that instead of exiting. - if logExitFunc != nil { - logExitFunc(err) - return - } - l.flushAll() - os.Exit(2) -} - -// syncBuffer joins a bufio.Writer to its underlying file, providing access to the -// file's Sync method and providing a wrapper for the Write method that provides log -// file rotation. There are conflicting methods, so the file cannot be embedded. -// l.mu is held for all its methods. -type syncBuffer struct { - logger *loggingT - *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file -} - -func (sb *syncBuffer) Sync() error { - return sb.file.Sync() -} - -func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { - if err := sb.rotateFile(time.Now()); err != nil { - sb.logger.exit(err) - } - } - n, err = sb.Writer.Write(p) - sb.nbytes += uint64(n) - if err != nil { - sb.logger.exit(err) - } - return -} - -// rotateFile closes the syncBuffer's file and starts a new one. -func (sb *syncBuffer) rotateFile(now time.Time) error { - if sb.file != nil { - sb.Flush() - sb.file.Close() - } - var err error - sb.file, _, err = create(severityName[sb.sev], now) - sb.nbytes = 0 - if err != nil { - return err - } - - sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) - - // Write header. - var buf bytes.Buffer - fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) - fmt.Fprintf(&buf, "Running on machine: %s\n", host) - fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Fprintf(&buf, "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n") - n, err := sb.file.Write(buf.Bytes()) - sb.nbytes += uint64(n) - return err -} - -// bufferSize sizes the buffer associated with each log file. It's large -// so that log records can accumulate without the logging thread blocking -// on disk I/O. The flushDaemon will block instead. -const bufferSize = 256 * 1024 - -// createFiles creates all the log files for severity from sev down to infoLog. -// l.mu is held. -func (l *loggingT) createFiles(sev severity) error { - now := time.Now() - // Files are created in decreasing severity order, so as soon as we find one - // has already been created, we can stop. - for s := sev; s >= infoLog && l.file[s] == nil; s-- { - sb := &syncBuffer{ - logger: l, - sev: s, - } - if err := sb.rotateFile(now); err != nil { - return err - } - l.file[s] = sb - } - return nil -} - -const flushInterval = 30 * time.Second - -// flushDaemon periodically flushes the log file buffers. -func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { - l.lockAndFlushAll() - } -} - -// lockAndFlushAll is like flushAll but locks l.mu first. -func (l *loggingT) lockAndFlushAll() { - l.mu.Lock() - l.flushAll() - l.mu.Unlock() -} - -// flushAll flushes all the logs and attempts to "sync" their data to disk. -// l.mu is held. -func (l *loggingT) flushAll() { - // Flush from fatal down, in case there's trouble flushing. - for s := fatalLog; s >= infoLog; s-- { - file := l.file[s] - if file != nil { - file.Flush() // ignore error - file.Sync() // ignore error - } - } -} - -// CopyStandardLogTo arranges for messages written to the Go "log" package's -// default logs to also appear in the Google logs for the named and lower -// severities. Subsequent changes to the standard log's default output location -// or format may break this behavior. -// -// Valid names are "INFO", "WARNING", "ERROR", and "FATAL". If the name is not -// recognized, CopyStandardLogTo panics. -func CopyStandardLogTo(name string) { - sev, ok := severityByName(name) - if !ok { - panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name)) - } - // Set a log format that captures the user's file and line: - // d.go:23: message - stdLog.SetFlags(stdLog.Lshortfile) - stdLog.SetOutput(logBridge(sev)) -} - -// logBridge provides the Write method that enables CopyStandardLogTo to connect -// Go's standard logs to the logs provided by this package. -type logBridge severity - -// Write parses the standard logging line and passes its components to the -// logger for severity(lb). -func (lb logBridge) Write(b []byte) (n int, err error) { - var ( - file = "???" - line = 1 - text string - ) - // Split "d.go:23: message" into "d.go", "23", and "message". - if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 { - text = fmt.Sprintf("bad log format: %s", b) - } else { - file = string(parts[0]) - text = string(parts[2][1:]) // skip leading space - line, err = strconv.Atoi(string(parts[1])) - if err != nil { - text = fmt.Sprintf("bad line number: %s", b) - line = 1 - } - } - // printWithFileLine with alsoToStderr=true, so standard log messages - // always appear on standard error. - logging.printWithFileLine(severity(lb), file, line, true, text) - return len(b), nil -} - -// setV computes and remembers the V level for a given PC -// when vmodule is enabled. -// File pattern matching takes the basename of the file, stripped -// of its .go suffix, and uses filepath.Match, which is a little more -// general than the *? matching used in C++. -// l.mu is held. -func (l *loggingT) setV(pc uintptr) Level { - fn := runtime.FuncForPC(pc) - file, _ := fn.FileLine(pc) - // The file is something like /a/b/c/d.go. We want just the d. - if strings.HasSuffix(file, ".go") { - file = file[:len(file)-3] - } - if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - for _, filter := range l.vmodule.filter { - if filter.match(file) { - l.vmap[pc] = filter.level - return filter.level - } - } - l.vmap[pc] = 0 - return 0 -} - -// Verbose is a boolean type that implements Infof (like Printf) etc. -// See the documentation of V for more information. -type Verbose bool - -// V reports whether verbosity at the call site is at least the requested level. -// The returned value is a boolean of type Verbose, which implements Info, Infoln -// and Infof. These methods will write to the Info log if called. -// Thus, one may write either -// if glog.V(2) { glog.Info("log this") } -// or -// glog.V(2).Info("log this") -// The second form is shorter but the first is cheaper if logging is off because it does -// not evaluate its arguments. -// -// Whether an individual call to V generates a log record depends on the setting of -// the -v and --vmodule flags; both are off by default. If the level in the call to -// V is at least the value of -v, or of -vmodule for the source file containing the -// call, the V call will log. -func V(level Level) Verbose { - // This function tries hard to be cheap unless there's work to do. - // The fast path is two atomic loads and compares. - - // Here is a cheap but safe test to see if V logging is enabled globally. - if logging.verbosity.get() >= level { - return Verbose(true) - } - - // It's off globally but it vmodule may still be set. - // Here is another cheap but safe test to see if vmodule is enabled. - if atomic.LoadInt32(&logging.filterLength) > 0 { - // Now we need a proper lock to use the logging structure. The pcs field - // is shared so we must lock before accessing it. This is fairly expensive, - // but if V logging is enabled we're slow anyway. - logging.mu.Lock() - defer logging.mu.Unlock() - if runtime.Callers(2, logging.pcs[:]) == 0 { - return Verbose(false) - } - v, ok := logging.vmap[logging.pcs[0]] - if !ok { - v = logging.setV(logging.pcs[0]) - } - return Verbose(v >= level) - } - return Verbose(false) -} - -// Info is equivalent to the global Info function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Info(args ...interface{}) { - if v { - logging.print(infoLog, args...) - } -} - -// Infoln is equivalent to the global Infoln function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infoln(args ...interface{}) { - if v { - logging.println(infoLog, args...) - } -} - -// Infof is equivalent to the global Infof function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infof(format string, args ...interface{}) { - if v { - logging.printf(infoLog, format, args...) - } -} - -// Info logs to the INFO log. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Info(args ...interface{}) { - logging.print(infoLog, args...) -} - -// InfoDepth acts as Info but uses depth to determine which call frame to log. -// InfoDepth(0, "msg") is the same as Info("msg"). -func InfoDepth(depth int, args ...interface{}) { - logging.printDepth(infoLog, depth, args...) -} - -// Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Infoln(args ...interface{}) { - logging.println(infoLog, args...) -} - -// Infof logs to the INFO log. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Infof(format string, args ...interface{}) { - logging.printf(infoLog, format, args...) -} - -// Warning logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Warning(args ...interface{}) { - logging.print(warningLog, args...) -} - -// WarningDepth acts as Warning but uses depth to determine which call frame to log. -// WarningDepth(0, "msg") is the same as Warning("msg"). -func WarningDepth(depth int, args ...interface{}) { - logging.printDepth(warningLog, depth, args...) -} - -// Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Warningln(args ...interface{}) { - logging.println(warningLog, args...) -} - -// Warningf logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Warningf(format string, args ...interface{}) { - logging.printf(warningLog, format, args...) -} - -// Error logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Error(args ...interface{}) { - logging.print(errorLog, args...) -} - -// ErrorDepth acts as Error but uses depth to determine which call frame to log. -// ErrorDepth(0, "msg") is the same as Error("msg"). -func ErrorDepth(depth int, args ...interface{}) { - logging.printDepth(errorLog, depth, args...) -} - -// Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Errorln(args ...interface{}) { - logging.println(errorLog, args...) -} - -// Errorf logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Errorf(format string, args ...interface{}) { - logging.printf(errorLog, format, args...) -} - -// Fatal logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Fatal(args ...interface{}) { - logging.print(fatalLog, args...) -} - -// FatalDepth acts as Fatal but uses depth to determine which call frame to log. -// FatalDepth(0, "msg") is the same as Fatal("msg"). -func FatalDepth(depth int, args ...interface{}) { - logging.printDepth(fatalLog, depth, args...) -} - -// Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Fatalln(args ...interface{}) { - logging.println(fatalLog, args...) -} - -// Fatalf logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Fatalf(format string, args ...interface{}) { - logging.printf(fatalLog, format, args...) -} - -// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks. -// It allows Exit and relatives to use the Fatal logs. -var fatalNoStacks uint32 - -// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Exit(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.print(fatalLog, args...) -} - -// ExitDepth acts as Exit but uses depth to determine which call frame to log. -// ExitDepth(0, "msg") is the same as Exit("msg"). -func ExitDepth(depth int, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printDepth(fatalLog, depth, args...) -} - -// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -func Exitln(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.println(fatalLog, args...) -} - -// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Exitf(format string, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printf(fatalLog, format, args...) -} diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/github.com/golang/glog/glog_file.go deleted file mode 100644 index 65075d2811..0000000000 --- a/vendor/github.com/golang/glog/glog_file.go +++ /dev/null @@ -1,124 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// File I/O for logs. - -package glog - -import ( - "errors" - "flag" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "sync" - "time" -) - -// MaxSize is the maximum size of a log file in bytes. -var MaxSize uint64 = 1024 * 1024 * 1800 - -// logDirs lists the candidate directories for new log files. -var logDirs []string - -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - -func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) - } - logDirs = append(logDirs, os.TempDir()) -} - -var ( - pid = os.Getpid() - program = filepath.Base(os.Args[0]) - host = "unknownhost" - userName = "unknownuser" -) - -func init() { - h, err := os.Hostname() - if err == nil { - host = shortHostname(h) - } - - current, err := user.Current() - if err == nil { - userName = current.Username - } - - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) -} - -// shortHostname returns its argument, truncating at the first period. -// For instance, given "www.google.com" it returns "www". -func shortHostname(hostname string) string { - if i := strings.Index(hostname, "."); i >= 0 { - return hostname[:i] - } - return hostname -} - -// logName returns a new log file name containing tag, with start time t, and -// the name for the symlink for tag. -func logName(tag string, t time.Time) (name, link string) { - name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d", - program, - host, - userName, - tag, - t.Year(), - t.Month(), - t.Day(), - t.Hour(), - t.Minute(), - t.Second(), - pid) - return name, program + "." + tag -} - -var onceLogDirs sync.Once - -// create creates a new log file and returns the file and its filename, which -// contains tag ("INFO", "FATAL", etc.) and t. If the file is created -// successfully, create also attempts to update the symlink for that tag, ignoring -// errors. -func create(tag string, t time.Time) (f *os.File, filename string, err error) { - onceLogDirs.Do(createLogDirs) - if len(logDirs) == 0 { - return nil, "", errors.New("log: no log dirs") - } - name, link := logName(tag, t) - var lastErr error - for _, dir := range logDirs { - fname := filepath.Join(dir, name) - f, err := os.Create(fname) - if err == nil { - symlink := filepath.Join(dir, link) - os.Remove(symlink) // ignore err - os.Symlink(name, symlink) // ignore err - return f, fname, nil - } - lastErr = err - } - return nil, "", fmt.Errorf("log: cannot create log: %v", lastErr) -}