From c25876f841cbb89e1b467f89007ed8d12031256d Mon Sep 17 00:00:00 2001 From: tizhou86 Date: Mon, 8 Jun 2020 19:49:40 +0800 Subject: [PATCH 1/5] Add gpu sharing in predicate logic. Signed-off-by: tizhou86 --- pkg/scheduler/api/device_info.go | 63 ++ pkg/scheduler/api/node_info.go | 89 +++ .../plugins/predicates/predicates.go | 38 +- vendor/modules.txt | 714 ------------------ 4 files changed, 189 insertions(+), 715 deletions(-) create mode 100644 pkg/scheduler/api/device_info.go diff --git a/pkg/scheduler/api/device_info.go b/pkg/scheduler/api/device_info.go new file mode 100644 index 0000000000..1c15a56390 --- /dev/null +++ b/pkg/scheduler/api/device_info.go @@ -0,0 +1,63 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strconv" + + v1 "k8s.io/api/core/v1" +) + +type DeviceInfo struct { + Id int + PodMap map[string]*v1.Pod + GPUTotalMemory uint +} + +func (di *DeviceInfo) GetPods() []*v1.Pod { + pods := []*v1.Pod{} + for _, pod := range di.PodMap { + pods = append(pods, pod) + } + return pods +} + +func NewDeviceInfo(id int, mem uint) *DeviceInfo { + return &DeviceInfo{ + Id: id, + GPUTotalMemory: mem, + PodMap: map[string]*v1.Pod{}, + } +} + +func (di *DeviceInfo) GetUsedGPUMemory() uint { + res := uint(0) + for _, pod := range di.PodMap { + if pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + continue + } else { + if len(pod.ObjectMeta.Annotations) > 0 { + mem, found := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] + if found { + m, _ := strconv.Atoi(mem) + res += uint(m) + } + } + } + } + return res +} diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 94bc7d5d6a..7b035022d7 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -18,6 +18,7 @@ package api import ( "fmt" + "strconv" v1 "k8s.io/api/core/v1" "k8s.io/klog" @@ -46,6 +47,10 @@ type NodeInfo struct { Tasks map[TaskID]*TaskInfo + Devices map[int]*DeviceInfo + GPUTotalCore int + GPUTotalMemory int + // Used to store custom information Others map[string]interface{} } @@ -78,6 +83,10 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { Capability: EmptyResource(), Tasks: make(map[TaskID]*TaskInfo), + + Devices: make(map[int]*DeviceInfo), + GPUTotalCore: 0, + GPUTotalMemory: 0, } } else { ni = &NodeInfo{ @@ -93,9 +102,14 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { Capability: NewResource(node.Status.Capacity), Tasks: make(map[TaskID]*TaskInfo), + + Devices: make(map[int]*DeviceInfo), + GPUTotalCore: 0, + GPUTotalMemory: 0, } } + ni.SetNodeGPUInfo(node) ni.setNodeState(node) return ni @@ -154,6 +168,24 @@ func (ni *NodeInfo) setNodeState(node *v1.Node) { } } +func (ni *NodeInfo) SetNodeGPUInfo(node *v1.Node) { + + core, ok := node.Status.Capacity["volcano.sh/node-gpu-core"] + if ok { + ni.GPUTotalCore = int(core.Value()) + } + + mem, ok := node.Status.Capacity["volcano.sh/node-gpu-memory"] + if ok { + ni.GPUTotalMemory = int(mem.Value()) + } + + for i := 0; i < int(core.Value()); i++ { + ni.Devices[i] = NewDeviceInfo(i, uint(int(mem.Value())/int(core.Value()))) + } + +} + // SetNode sets kubernetes node object to nodeInfo object func (ni *NodeInfo) SetNode(node *v1.Node) { ni.setNodeState(node) @@ -314,3 +346,60 @@ func (ni *NodeInfo) Pods() (pods []*v1.Pod) { return } + +func (ni *NodeInfo) CheckPredicatePodOnGPUNode(pod *v1.Pod) bool { + res := false + memReq := uint(0) + + remainMems := ni.GetDevicesRemainGPUMemory() + if len(pod.ObjectMeta.Annotations) > 0 { + mem, found := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] + if found { + m, _ := strconv.Atoi(mem) + memReq = uint(m) + } + } + + if len(remainMems) > 0 { + for devID := 0; devID < len(ni.Devices); devID++ { + availableGPU, ok := remainMems[devID] + if ok { + if availableGPU >= memReq { + res = true + break + } + } + } + } + + return res + +} + +func (ni *NodeInfo) GetDevicesRemainGPUMemory() map[int]uint { + devicesAllGPUMemory := ni.GetDevicesAllGPUMemory() + devicesUsedGPUMemory := ni.GetDevicesUsedGPUMemory() + res := map[int]uint{} + for id, allMemory := range devicesAllGPUMemory { + if usedMemory, found := devicesUsedGPUMemory[id]; found { + res[id] = allMemory - usedMemory + } + } + return res +} + +func (ni *NodeInfo) GetDevicesUsedGPUMemory() map[int]uint { + res := map[int]uint{} + for _, device := range ni.Devices { + res[device.Id] = device.GetUsedGPUMemory() + } + return res +} + +func (ni *NodeInfo) GetDevicesAllGPUMemory() map[int]uint { + res := map[int]uint{} + for _, device := range ni.Devices { + res[device.Id] = device.GPUTotalMemory + } + return res +} diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index 9572c343d7..79a030687f 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" @@ -44,6 +45,8 @@ const ( DiskPressurePredicate = "predicate.DiskPressureEnable" // PIDPressurePredicate is the key for enabling PID Pressure Predicate in YAML PIDPressurePredicate = "predicate.PIDPressureEnable" + // GPUSharingPredicate is the key for enabling GPU Sharing Predicate in YAML + GPUSharingPredicate = "predicate.GPUSharingEnable" ) type predicatesPlugin struct { @@ -64,12 +67,13 @@ type predicateEnable struct { memoryPressureEnable bool diskPressureEnable bool pidPressureEnable bool + gpuSharingEnable bool } func enablePredicate(args framework.Arguments) predicateEnable { /* - User Should give predicatesEnable in this format(predicate.MemoryPressureEnable, predicate.DiskPressureEnable, predicate.PIDPressureEnable. + User Should give predicatesEnable in this format(predicate.MemoryPressureEnable, predicate.DiskPressureEnable, predicate.PIDPressureEnable, predicate.GPUSharingEnable. Currently supported only for MemoryPressure, DiskPressure, PIDPressure predicate checks. actions: "reclaim, allocate, backfill, preempt" @@ -85,6 +89,7 @@ func enablePredicate(args framework.Arguments) predicateEnable { predicate.MemoryPressureEnable: true predicate.DiskPressureEnable: true predicate.PIDPressureEnable: true + predicate.GPUSharingEnable: true - name: proportion - name: nodeorder */ @@ -93,6 +98,7 @@ func enablePredicate(args framework.Arguments) predicateEnable { memoryPressureEnable: false, diskPressureEnable: false, pidPressureEnable: false, + gpuSharingEnable: false, } // Checks whether predicate.MemoryPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. @@ -104,6 +110,9 @@ func enablePredicate(args framework.Arguments) predicateEnable { // Checks whether predicate.PIDPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. args.GetBool(&predicate.pidPressureEnable, PIDPressurePredicate) + // Checks whether predicate.GPUSharingEnable is provided or not, if given, modifies the value in predicateEnable struct. + args.GetBool(&predicate.gpuSharingEnable, GPUSharingPredicate) + return predicate } @@ -202,13 +211,40 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { if !status.IsSuccess() { return fmt.Errorf("plugin %s pre-predicates failed %s", interpodaffinity.Name, status.Message()) } + status = podAffinityFilter.Filter(context.TODO(), state, task.Pod, nodeInfo) if !status.IsSuccess() { return fmt.Errorf("plugin %s predicates failed %s", interpodaffinity.Name, status.Message()) } + // if predicate.gpuSharingEnable + if true { + // CheckGPUSharingPredicate + fit, err := CheckNodeGPUSharingPredicate(task.Pod, node) + if err != nil { + return err + } + + klog.V(4).Infof("CheckNodeGPUSharingPredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", + task.Namespace, task.Name, node.Name, fit, err) + } + return nil }) } func (pp *predicatesPlugin) OnSessionClose(ssn *framework.Session) {} + +// CheckNodeGPUSharingPredicate checks if a gpu sharing pod can be scheduled on a node. +func CheckNodeGPUSharingPredicate(pod *v1.Pod, nodeInfo *api.NodeInfo) (bool, error) { + _, ok := nodeInfo.Node.Status.Capacity["volcano.sh/node-gpu-core"] + if !ok { + return false, fmt.Errorf("node is not gpu sharing") + } else { + isEnoughGPUMemoryOnNode := nodeInfo.CheckPredicatePodOnGPUNode(pod) + if !isEnoughGPUMemoryOnNode { + return false, fmt.Errorf("no enough gpu memory on single device") + } + } + return true, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index dee69dc0f8..e69de29bb2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,714 +0,0 @@ -# cloud.google.com/go v0.38.0 -cloud.google.com/go/compute/metadata -# github.com/beorn7/perks v1.0.0 -github.com/beorn7/perks/quantile -# github.com/blang/semver v3.5.0+incompatible -github.com/blang/semver -# github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e -github.com/coreos/go-systemd/journal -# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea -github.com/coreos/pkg/capnslog -# github.com/davecgh/go-spew v1.1.1 -github.com/davecgh/go-spew/spew -# github.com/docker/distribution v2.7.1+incompatible -github.com/docker/distribution/digestset -github.com/docker/distribution/reference -# github.com/emicklei/go-restful v2.9.5+incompatible -github.com/emicklei/go-restful -github.com/emicklei/go-restful/log -# github.com/evanphx/json-patch v4.2.0+incompatible -github.com/evanphx/json-patch -# github.com/gogo/protobuf v1.3.1 -github.com/gogo/protobuf/gogoproto -github.com/gogo/protobuf/proto -github.com/gogo/protobuf/protoc-gen-gogo/descriptor -github.com/gogo/protobuf/sortkeys -# github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 -github.com/golang/groupcache/lru -# github.com/golang/protobuf v1.3.2 -github.com/golang/protobuf/proto -github.com/golang/protobuf/ptypes -github.com/golang/protobuf/ptypes/any -github.com/golang/protobuf/ptypes/duration -github.com/golang/protobuf/ptypes/timestamp -# github.com/google/go-cmp v0.3.0 -github.com/google/go-cmp/cmp -github.com/google/go-cmp/cmp/internal/diff -github.com/google/go-cmp/cmp/internal/flags -github.com/google/go-cmp/cmp/internal/function -github.com/google/go-cmp/cmp/internal/value -# github.com/google/gofuzz v1.1.0 -github.com/google/gofuzz -# github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 -## explicit -github.com/google/shlex -# github.com/google/uuid v1.1.1 -github.com/google/uuid -# github.com/googleapis/gnostic v0.1.0 -github.com/googleapis/gnostic/OpenAPIv2 -github.com/googleapis/gnostic/compiler -github.com/googleapis/gnostic/extensions -# github.com/hashicorp/errwrap v1.0.0 -github.com/hashicorp/errwrap -# github.com/hashicorp/go-multierror v1.0.0 -## explicit -github.com/hashicorp/go-multierror -# github.com/hashicorp/golang-lru v0.5.1 -github.com/hashicorp/golang-lru -github.com/hashicorp/golang-lru/simplelru -# github.com/hpcloud/tail v1.0.0 -github.com/hpcloud/tail -github.com/hpcloud/tail/ratelimiter -github.com/hpcloud/tail/util -github.com/hpcloud/tail/watch -github.com/hpcloud/tail/winfile -# github.com/imdario/mergo v0.3.5 -github.com/imdario/mergo -# github.com/inconshreveable/mousetrap v1.0.0 -github.com/inconshreveable/mousetrap -# github.com/json-iterator/go v1.1.8 -github.com/json-iterator/go -# github.com/matttproud/golang_protobuf_extensions v1.0.1 -github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd -github.com/modern-go/concurrent -# github.com/modern-go/reflect2 v1.0.1 -github.com/modern-go/reflect2 -# github.com/onsi/ginkgo v1.11.0 -## explicit -github.com/onsi/ginkgo -github.com/onsi/ginkgo/config -github.com/onsi/ginkgo/internal/codelocation -github.com/onsi/ginkgo/internal/containernode -github.com/onsi/ginkgo/internal/failer -github.com/onsi/ginkgo/internal/leafnodes -github.com/onsi/ginkgo/internal/remote -github.com/onsi/ginkgo/internal/spec -github.com/onsi/ginkgo/internal/spec_iterator -github.com/onsi/ginkgo/internal/specrunner -github.com/onsi/ginkgo/internal/suite -github.com/onsi/ginkgo/internal/testingtproxy -github.com/onsi/ginkgo/internal/writer -github.com/onsi/ginkgo/reporters -github.com/onsi/ginkgo/reporters/stenographer -github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable -github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty -github.com/onsi/ginkgo/types -# github.com/onsi/gomega v1.7.0 -## explicit -github.com/onsi/gomega -github.com/onsi/gomega/format -github.com/onsi/gomega/internal/assertion -github.com/onsi/gomega/internal/asyncassertion -github.com/onsi/gomega/internal/oraclematcher -github.com/onsi/gomega/internal/testingtsupport -github.com/onsi/gomega/matchers -github.com/onsi/gomega/matchers/support/goraph/bipartitegraph -github.com/onsi/gomega/matchers/support/goraph/edge -github.com/onsi/gomega/matchers/support/goraph/node -github.com/onsi/gomega/matchers/support/goraph/util -github.com/onsi/gomega/types -# github.com/opencontainers/go-digest v1.0.0-rc1 -github.com/opencontainers/go-digest -# github.com/prometheus/client_golang v1.0.0 -## explicit -github.com/prometheus/client_golang/prometheus -github.com/prometheus/client_golang/prometheus/internal -github.com/prometheus/client_golang/prometheus/promauto -github.com/prometheus/client_golang/prometheus/promhttp -# github.com/prometheus/client_model v0.2.0 -github.com/prometheus/client_model/go -# github.com/prometheus/common v0.4.1 -github.com/prometheus/common/expfmt -github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg -github.com/prometheus/common/model -# github.com/prometheus/procfs v0.0.2 -github.com/prometheus/procfs -github.com/prometheus/procfs/internal/fs -# github.com/spf13/cobra v0.0.5 -## explicit -github.com/spf13/cobra -# github.com/spf13/pflag v1.0.5 -## explicit -github.com/spf13/pflag -# go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 -go.etcd.io/etcd/auth/authpb -go.etcd.io/etcd/clientv3 -go.etcd.io/etcd/clientv3/balancer -go.etcd.io/etcd/clientv3/balancer/connectivity -go.etcd.io/etcd/clientv3/balancer/picker -go.etcd.io/etcd/clientv3/balancer/resolver/endpoint -go.etcd.io/etcd/clientv3/credentials -go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes -go.etcd.io/etcd/etcdserver/etcdserverpb -go.etcd.io/etcd/mvcc/mvccpb -go.etcd.io/etcd/pkg/logutil -go.etcd.io/etcd/pkg/systemd -go.etcd.io/etcd/pkg/types -go.etcd.io/etcd/raft -go.etcd.io/etcd/raft/confchange -go.etcd.io/etcd/raft/quorum -go.etcd.io/etcd/raft/raftpb -go.etcd.io/etcd/raft/tracker -# go.uber.org/atomic v1.3.2 -go.uber.org/atomic -# go.uber.org/multierr v1.1.0 -go.uber.org/multierr -# go.uber.org/zap v1.10.0 -go.uber.org/zap -go.uber.org/zap/buffer -go.uber.org/zap/internal/bufferpool -go.uber.org/zap/internal/color -go.uber.org/zap/internal/exit -go.uber.org/zap/zapcore -# golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 -## explicit -golang.org/x/crypto/blowfish -golang.org/x/crypto/chacha20 -golang.org/x/crypto/curve25519 -golang.org/x/crypto/ed25519 -golang.org/x/crypto/ed25519/internal/edwards25519 -golang.org/x/crypto/internal/subtle -golang.org/x/crypto/poly1305 -golang.org/x/crypto/ssh -golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -golang.org/x/crypto/ssh/terminal -# golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 -golang.org/x/net/context -golang.org/x/net/context/ctxhttp -golang.org/x/net/html -golang.org/x/net/html/atom -golang.org/x/net/html/charset -golang.org/x/net/http/httpguts -golang.org/x/net/http2 -golang.org/x/net/http2/hpack -golang.org/x/net/idna -golang.org/x/net/internal/timeseries -golang.org/x/net/trace -# golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 -golang.org/x/oauth2 -golang.org/x/oauth2/google -golang.org/x/oauth2/internal -golang.org/x/oauth2/jws -golang.org/x/oauth2/jwt -# golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 -golang.org/x/sys/cpu -golang.org/x/sys/unix -golang.org/x/sys/windows -# golang.org/x/text v0.3.2 -golang.org/x/text/encoding -golang.org/x/text/encoding/charmap -golang.org/x/text/encoding/htmlindex -golang.org/x/text/encoding/internal -golang.org/x/text/encoding/internal/identifier -golang.org/x/text/encoding/japanese -golang.org/x/text/encoding/korean -golang.org/x/text/encoding/simplifiedchinese -golang.org/x/text/encoding/traditionalchinese -golang.org/x/text/encoding/unicode -golang.org/x/text/internal/language -golang.org/x/text/internal/language/compact -golang.org/x/text/internal/tag -golang.org/x/text/internal/utf8internal -golang.org/x/text/language -golang.org/x/text/runes -golang.org/x/text/secure/bidirule -golang.org/x/text/transform -golang.org/x/text/unicode/bidi -golang.org/x/text/unicode/norm -# golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 -## explicit -golang.org/x/time/rate -# google.golang.org/appengine v1.5.0 -google.golang.org/appengine -google.golang.org/appengine/internal -google.golang.org/appengine/internal/app_identity -google.golang.org/appengine/internal/base -google.golang.org/appengine/internal/datastore -google.golang.org/appengine/internal/log -google.golang.org/appengine/internal/modules -google.golang.org/appengine/internal/remote_api -google.golang.org/appengine/internal/urlfetch -google.golang.org/appengine/urlfetch -# google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 -google.golang.org/genproto/googleapis/rpc/status -# google.golang.org/grpc v1.26.0 -google.golang.org/grpc -google.golang.org/grpc/attributes -google.golang.org/grpc/backoff -google.golang.org/grpc/balancer -google.golang.org/grpc/balancer/base -google.golang.org/grpc/balancer/roundrobin -google.golang.org/grpc/binarylog/grpc_binarylog_v1 -google.golang.org/grpc/codes -google.golang.org/grpc/connectivity -google.golang.org/grpc/credentials -google.golang.org/grpc/credentials/internal -google.golang.org/grpc/encoding -google.golang.org/grpc/encoding/proto -google.golang.org/grpc/grpclog -google.golang.org/grpc/internal -google.golang.org/grpc/internal/backoff -google.golang.org/grpc/internal/balancerload -google.golang.org/grpc/internal/binarylog -google.golang.org/grpc/internal/buffer -google.golang.org/grpc/internal/channelz -google.golang.org/grpc/internal/envconfig -google.golang.org/grpc/internal/grpcrand -google.golang.org/grpc/internal/grpcsync -google.golang.org/grpc/internal/resolver/dns -google.golang.org/grpc/internal/resolver/passthrough -google.golang.org/grpc/internal/syscall -google.golang.org/grpc/internal/transport -google.golang.org/grpc/keepalive -google.golang.org/grpc/metadata -google.golang.org/grpc/naming -google.golang.org/grpc/peer -google.golang.org/grpc/resolver -google.golang.org/grpc/resolver/dns -google.golang.org/grpc/resolver/passthrough -google.golang.org/grpc/serviceconfig -google.golang.org/grpc/stats -google.golang.org/grpc/status -google.golang.org/grpc/tap -# gopkg.in/fsnotify.v1 v1.4.7 -gopkg.in/fsnotify.v1 -# gopkg.in/inf.v0 v0.9.1 -gopkg.in/inf.v0 -# gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 -gopkg.in/tomb.v1 -# gopkg.in/yaml.v2 v2.2.8 -## explicit -gopkg.in/yaml.v2 -# k8s.io/api v0.18.3 => k8s.io/api v0.18.3 -## explicit -k8s.io/api/admission/v1beta1 -k8s.io/api/admissionregistration/v1 -k8s.io/api/admissionregistration/v1beta1 -k8s.io/api/apps/v1 -k8s.io/api/apps/v1beta1 -k8s.io/api/apps/v1beta2 -k8s.io/api/auditregistration/v1alpha1 -k8s.io/api/authentication/v1 -k8s.io/api/authentication/v1beta1 -k8s.io/api/authorization/v1 -k8s.io/api/authorization/v1beta1 -k8s.io/api/autoscaling/v1 -k8s.io/api/autoscaling/v2beta1 -k8s.io/api/autoscaling/v2beta2 -k8s.io/api/batch/v1 -k8s.io/api/batch/v1beta1 -k8s.io/api/batch/v2alpha1 -k8s.io/api/certificates/v1beta1 -k8s.io/api/coordination/v1 -k8s.io/api/coordination/v1beta1 -k8s.io/api/core/v1 -k8s.io/api/discovery/v1alpha1 -k8s.io/api/discovery/v1beta1 -k8s.io/api/events/v1beta1 -k8s.io/api/extensions/v1beta1 -k8s.io/api/flowcontrol/v1alpha1 -k8s.io/api/networking/v1 -k8s.io/api/networking/v1beta1 -k8s.io/api/node/v1alpha1 -k8s.io/api/node/v1beta1 -k8s.io/api/policy/v1beta1 -k8s.io/api/rbac/v1 -k8s.io/api/rbac/v1alpha1 -k8s.io/api/rbac/v1beta1 -k8s.io/api/scheduling/v1 -k8s.io/api/scheduling/v1alpha1 -k8s.io/api/scheduling/v1beta1 -k8s.io/api/settings/v1alpha1 -k8s.io/api/storage/v1 -k8s.io/api/storage/v1alpha1 -k8s.io/api/storage/v1beta1 -# k8s.io/apimachinery v0.18.3 => k8s.io/apimachinery v0.18.3 -## explicit -k8s.io/apimachinery/pkg/api/equality -k8s.io/apimachinery/pkg/api/errors -k8s.io/apimachinery/pkg/api/meta -k8s.io/apimachinery/pkg/api/resource -k8s.io/apimachinery/pkg/api/validation -k8s.io/apimachinery/pkg/api/validation/path -k8s.io/apimachinery/pkg/apis/meta/internalversion -k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme -k8s.io/apimachinery/pkg/apis/meta/v1 -k8s.io/apimachinery/pkg/apis/meta/v1/unstructured -k8s.io/apimachinery/pkg/apis/meta/v1/validation -k8s.io/apimachinery/pkg/apis/meta/v1beta1 -k8s.io/apimachinery/pkg/conversion -k8s.io/apimachinery/pkg/conversion/queryparams -k8s.io/apimachinery/pkg/fields -k8s.io/apimachinery/pkg/labels -k8s.io/apimachinery/pkg/runtime -k8s.io/apimachinery/pkg/runtime/schema -k8s.io/apimachinery/pkg/runtime/serializer -k8s.io/apimachinery/pkg/runtime/serializer/json -k8s.io/apimachinery/pkg/runtime/serializer/protobuf -k8s.io/apimachinery/pkg/runtime/serializer/recognizer -k8s.io/apimachinery/pkg/runtime/serializer/streaming -k8s.io/apimachinery/pkg/runtime/serializer/versioning -k8s.io/apimachinery/pkg/selection -k8s.io/apimachinery/pkg/types -k8s.io/apimachinery/pkg/util/cache -k8s.io/apimachinery/pkg/util/clock -k8s.io/apimachinery/pkg/util/diff -k8s.io/apimachinery/pkg/util/errors -k8s.io/apimachinery/pkg/util/framer -k8s.io/apimachinery/pkg/util/intstr -k8s.io/apimachinery/pkg/util/json -k8s.io/apimachinery/pkg/util/mergepatch -k8s.io/apimachinery/pkg/util/naming -k8s.io/apimachinery/pkg/util/net -k8s.io/apimachinery/pkg/util/runtime -k8s.io/apimachinery/pkg/util/sets -k8s.io/apimachinery/pkg/util/strategicpatch -k8s.io/apimachinery/pkg/util/uuid -k8s.io/apimachinery/pkg/util/validation -k8s.io/apimachinery/pkg/util/validation/field -k8s.io/apimachinery/pkg/util/wait -k8s.io/apimachinery/pkg/util/yaml -k8s.io/apimachinery/pkg/version -k8s.io/apimachinery/pkg/watch -k8s.io/apimachinery/third_party/forked/golang/json -k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.18.3 => k8s.io/apiserver v0.18.3 -## explicit -k8s.io/apiserver/pkg/apis/audit -k8s.io/apiserver/pkg/authentication/user -k8s.io/apiserver/pkg/endpoints/metrics -k8s.io/apiserver/pkg/endpoints/request -k8s.io/apiserver/pkg/features -k8s.io/apiserver/pkg/server/healthz -k8s.io/apiserver/pkg/server/httplog -k8s.io/apiserver/pkg/server/mux -k8s.io/apiserver/pkg/storage -k8s.io/apiserver/pkg/storage/etcd3 -k8s.io/apiserver/pkg/storage/etcd3/metrics -k8s.io/apiserver/pkg/storage/value -k8s.io/apiserver/pkg/util/feature -# k8s.io/client-go v0.18.3 => k8s.io/client-go v0.18.3 -## explicit -k8s.io/client-go/discovery -k8s.io/client-go/discovery/fake -k8s.io/client-go/informers -k8s.io/client-go/informers/admissionregistration -k8s.io/client-go/informers/admissionregistration/v1 -k8s.io/client-go/informers/admissionregistration/v1beta1 -k8s.io/client-go/informers/apps -k8s.io/client-go/informers/apps/v1 -k8s.io/client-go/informers/apps/v1beta1 -k8s.io/client-go/informers/apps/v1beta2 -k8s.io/client-go/informers/auditregistration -k8s.io/client-go/informers/auditregistration/v1alpha1 -k8s.io/client-go/informers/autoscaling -k8s.io/client-go/informers/autoscaling/v1 -k8s.io/client-go/informers/autoscaling/v2beta1 -k8s.io/client-go/informers/autoscaling/v2beta2 -k8s.io/client-go/informers/batch -k8s.io/client-go/informers/batch/v1 -k8s.io/client-go/informers/batch/v1beta1 -k8s.io/client-go/informers/batch/v2alpha1 -k8s.io/client-go/informers/certificates -k8s.io/client-go/informers/certificates/v1beta1 -k8s.io/client-go/informers/coordination -k8s.io/client-go/informers/coordination/v1 -k8s.io/client-go/informers/coordination/v1beta1 -k8s.io/client-go/informers/core -k8s.io/client-go/informers/core/v1 -k8s.io/client-go/informers/discovery -k8s.io/client-go/informers/discovery/v1alpha1 -k8s.io/client-go/informers/discovery/v1beta1 -k8s.io/client-go/informers/events -k8s.io/client-go/informers/events/v1beta1 -k8s.io/client-go/informers/extensions -k8s.io/client-go/informers/extensions/v1beta1 -k8s.io/client-go/informers/flowcontrol -k8s.io/client-go/informers/flowcontrol/v1alpha1 -k8s.io/client-go/informers/internalinterfaces -k8s.io/client-go/informers/networking -k8s.io/client-go/informers/networking/v1 -k8s.io/client-go/informers/networking/v1beta1 -k8s.io/client-go/informers/node -k8s.io/client-go/informers/node/v1alpha1 -k8s.io/client-go/informers/node/v1beta1 -k8s.io/client-go/informers/policy -k8s.io/client-go/informers/policy/v1beta1 -k8s.io/client-go/informers/rbac -k8s.io/client-go/informers/rbac/v1 -k8s.io/client-go/informers/rbac/v1alpha1 -k8s.io/client-go/informers/rbac/v1beta1 -k8s.io/client-go/informers/scheduling -k8s.io/client-go/informers/scheduling/v1 -k8s.io/client-go/informers/scheduling/v1alpha1 -k8s.io/client-go/informers/scheduling/v1beta1 -k8s.io/client-go/informers/settings -k8s.io/client-go/informers/settings/v1alpha1 -k8s.io/client-go/informers/storage -k8s.io/client-go/informers/storage/v1 -k8s.io/client-go/informers/storage/v1alpha1 -k8s.io/client-go/informers/storage/v1beta1 -k8s.io/client-go/kubernetes -k8s.io/client-go/kubernetes/fake -k8s.io/client-go/kubernetes/scheme -k8s.io/client-go/kubernetes/typed/admissionregistration/v1 -k8s.io/client-go/kubernetes/typed/admissionregistration/v1/fake -k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 -k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/fake -k8s.io/client-go/kubernetes/typed/apps/v1 -k8s.io/client-go/kubernetes/typed/apps/v1/fake -k8s.io/client-go/kubernetes/typed/apps/v1beta1 -k8s.io/client-go/kubernetes/typed/apps/v1beta1/fake -k8s.io/client-go/kubernetes/typed/apps/v1beta2 -k8s.io/client-go/kubernetes/typed/apps/v1beta2/fake -k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 -k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/authentication/v1 -k8s.io/client-go/kubernetes/typed/authentication/v1/fake -k8s.io/client-go/kubernetes/typed/authentication/v1beta1 -k8s.io/client-go/kubernetes/typed/authentication/v1beta1/fake -k8s.io/client-go/kubernetes/typed/authorization/v1 -k8s.io/client-go/kubernetes/typed/authorization/v1/fake -k8s.io/client-go/kubernetes/typed/authorization/v1beta1 -k8s.io/client-go/kubernetes/typed/authorization/v1beta1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v1 -k8s.io/client-go/kubernetes/typed/autoscaling/v1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/fake -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 -k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/fake -k8s.io/client-go/kubernetes/typed/batch/v1 -k8s.io/client-go/kubernetes/typed/batch/v1/fake -k8s.io/client-go/kubernetes/typed/batch/v1beta1 -k8s.io/client-go/kubernetes/typed/batch/v1beta1/fake -k8s.io/client-go/kubernetes/typed/batch/v2alpha1 -k8s.io/client-go/kubernetes/typed/batch/v2alpha1/fake -k8s.io/client-go/kubernetes/typed/certificates/v1beta1 -k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake -k8s.io/client-go/kubernetes/typed/coordination/v1 -k8s.io/client-go/kubernetes/typed/coordination/v1/fake -k8s.io/client-go/kubernetes/typed/coordination/v1beta1 -k8s.io/client-go/kubernetes/typed/coordination/v1beta1/fake -k8s.io/client-go/kubernetes/typed/core/v1 -k8s.io/client-go/kubernetes/typed/core/v1/fake -k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 -k8s.io/client-go/kubernetes/typed/discovery/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/discovery/v1beta1 -k8s.io/client-go/kubernetes/typed/discovery/v1beta1/fake -k8s.io/client-go/kubernetes/typed/events/v1beta1 -k8s.io/client-go/kubernetes/typed/events/v1beta1/fake -k8s.io/client-go/kubernetes/typed/extensions/v1beta1 -k8s.io/client-go/kubernetes/typed/extensions/v1beta1/fake -k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1 -k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/networking/v1 -k8s.io/client-go/kubernetes/typed/networking/v1/fake -k8s.io/client-go/kubernetes/typed/networking/v1beta1 -k8s.io/client-go/kubernetes/typed/networking/v1beta1/fake -k8s.io/client-go/kubernetes/typed/node/v1alpha1 -k8s.io/client-go/kubernetes/typed/node/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/node/v1beta1 -k8s.io/client-go/kubernetes/typed/node/v1beta1/fake -k8s.io/client-go/kubernetes/typed/policy/v1beta1 -k8s.io/client-go/kubernetes/typed/policy/v1beta1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1 -k8s.io/client-go/kubernetes/typed/rbac/v1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 -k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/rbac/v1beta1 -k8s.io/client-go/kubernetes/typed/rbac/v1beta1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1 -k8s.io/client-go/kubernetes/typed/scheduling/v1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 -k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 -k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/fake -k8s.io/client-go/kubernetes/typed/settings/v1alpha1 -k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/storage/v1 -k8s.io/client-go/kubernetes/typed/storage/v1/fake -k8s.io/client-go/kubernetes/typed/storage/v1alpha1 -k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake -k8s.io/client-go/kubernetes/typed/storage/v1beta1 -k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake -k8s.io/client-go/listers/admissionregistration/v1 -k8s.io/client-go/listers/admissionregistration/v1beta1 -k8s.io/client-go/listers/apps/v1 -k8s.io/client-go/listers/apps/v1beta1 -k8s.io/client-go/listers/apps/v1beta2 -k8s.io/client-go/listers/auditregistration/v1alpha1 -k8s.io/client-go/listers/autoscaling/v1 -k8s.io/client-go/listers/autoscaling/v2beta1 -k8s.io/client-go/listers/autoscaling/v2beta2 -k8s.io/client-go/listers/batch/v1 -k8s.io/client-go/listers/batch/v1beta1 -k8s.io/client-go/listers/batch/v2alpha1 -k8s.io/client-go/listers/certificates/v1beta1 -k8s.io/client-go/listers/coordination/v1 -k8s.io/client-go/listers/coordination/v1beta1 -k8s.io/client-go/listers/core/v1 -k8s.io/client-go/listers/discovery/v1alpha1 -k8s.io/client-go/listers/discovery/v1beta1 -k8s.io/client-go/listers/events/v1beta1 -k8s.io/client-go/listers/extensions/v1beta1 -k8s.io/client-go/listers/flowcontrol/v1alpha1 -k8s.io/client-go/listers/networking/v1 -k8s.io/client-go/listers/networking/v1beta1 -k8s.io/client-go/listers/node/v1alpha1 -k8s.io/client-go/listers/node/v1beta1 -k8s.io/client-go/listers/policy/v1beta1 -k8s.io/client-go/listers/rbac/v1 -k8s.io/client-go/listers/rbac/v1alpha1 -k8s.io/client-go/listers/rbac/v1beta1 -k8s.io/client-go/listers/scheduling/v1 -k8s.io/client-go/listers/scheduling/v1alpha1 -k8s.io/client-go/listers/scheduling/v1beta1 -k8s.io/client-go/listers/settings/v1alpha1 -k8s.io/client-go/listers/storage/v1 -k8s.io/client-go/listers/storage/v1alpha1 -k8s.io/client-go/listers/storage/v1beta1 -k8s.io/client-go/pkg/apis/clientauthentication -k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 -k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 -k8s.io/client-go/pkg/version -k8s.io/client-go/plugin/pkg/client/auth/exec -k8s.io/client-go/plugin/pkg/client/auth/gcp -k8s.io/client-go/rest -k8s.io/client-go/rest/watch -k8s.io/client-go/testing -k8s.io/client-go/third_party/forked/golang/template -k8s.io/client-go/tools/auth -k8s.io/client-go/tools/cache -k8s.io/client-go/tools/clientcmd -k8s.io/client-go/tools/clientcmd/api -k8s.io/client-go/tools/clientcmd/api/latest -k8s.io/client-go/tools/clientcmd/api/v1 -k8s.io/client-go/tools/leaderelection -k8s.io/client-go/tools/leaderelection/resourcelock -k8s.io/client-go/tools/metrics -k8s.io/client-go/tools/pager -k8s.io/client-go/tools/record -k8s.io/client-go/tools/record/util -k8s.io/client-go/tools/reference -k8s.io/client-go/transport -k8s.io/client-go/util/cert -k8s.io/client-go/util/connrotation -k8s.io/client-go/util/flowcontrol -k8s.io/client-go/util/homedir -k8s.io/client-go/util/jsonpath -k8s.io/client-go/util/keyutil -k8s.io/client-go/util/workqueue -# k8s.io/cloud-provider v0.18.3 => k8s.io/cloud-provider v0.18.3 -k8s.io/cloud-provider -k8s.io/cloud-provider/volume -# k8s.io/component-base v0.18.3 => k8s.io/component-base v0.18.3 -## explicit -k8s.io/component-base/cli/flag -k8s.io/component-base/config -k8s.io/component-base/featuregate -k8s.io/component-base/metrics -k8s.io/component-base/metrics/legacyregistry -k8s.io/component-base/version -# k8s.io/csi-translation-lib v0.18.3 => k8s.io/csi-translation-lib v0.18.3 -k8s.io/csi-translation-lib -k8s.io/csi-translation-lib/plugins -# k8s.io/klog v1.0.0 => k8s.io/klog v1.0.0 -## explicit -k8s.io/klog -# k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 -k8s.io/kube-openapi/pkg/util/proto -# k8s.io/kube-scheduler v0.0.0 => k8s.io/kube-scheduler v0.18.3 -## explicit -k8s.io/kube-scheduler/extender/v1 -# k8s.io/kubernetes v1.18.3 -## explicit -k8s.io/kubernetes/pkg/api/legacyscheme -k8s.io/kubernetes/pkg/api/service -k8s.io/kubernetes/pkg/api/v1/pod -k8s.io/kubernetes/pkg/apis/apps -k8s.io/kubernetes/pkg/apis/autoscaling -k8s.io/kubernetes/pkg/apis/core -k8s.io/kubernetes/pkg/apis/core/helper -k8s.io/kubernetes/pkg/apis/core/pods -k8s.io/kubernetes/pkg/apis/core/v1 -k8s.io/kubernetes/pkg/apis/core/v1/helper -k8s.io/kubernetes/pkg/apis/core/validation -k8s.io/kubernetes/pkg/apis/scheduling -k8s.io/kubernetes/pkg/capabilities -k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util -k8s.io/kubernetes/pkg/controller/volume/scheduling -k8s.io/kubernetes/pkg/controller/volume/scheduling/metrics -k8s.io/kubernetes/pkg/features -k8s.io/kubernetes/pkg/fieldpath -k8s.io/kubernetes/pkg/kubelet/types -k8s.io/kubernetes/pkg/master/ports -k8s.io/kubernetes/pkg/scheduler/apis/config -k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper -k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity -k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity -k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports -k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources -k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable -k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration -k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1 -k8s.io/kubernetes/pkg/scheduler/listers -k8s.io/kubernetes/pkg/scheduler/metrics -k8s.io/kubernetes/pkg/scheduler/nodeinfo -k8s.io/kubernetes/pkg/scheduler/util -k8s.io/kubernetes/pkg/security/apparmor -k8s.io/kubernetes/pkg/util/parsers -k8s.io/kubernetes/pkg/util/resizefs -k8s.io/kubernetes/pkg/volume -k8s.io/kubernetes/pkg/volume/util -k8s.io/kubernetes/pkg/volume/util/fs -k8s.io/kubernetes/pkg/volume/util/fsquota -k8s.io/kubernetes/pkg/volume/util/fsquota/common -k8s.io/kubernetes/pkg/volume/util/hostutil -k8s.io/kubernetes/pkg/volume/util/recyclerclient -k8s.io/kubernetes/pkg/volume/util/subpath -k8s.io/kubernetes/pkg/volume/util/types -k8s.io/kubernetes/pkg/volume/util/volumepathhandler -# k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 -k8s.io/utils/buffer -k8s.io/utils/exec -k8s.io/utils/integer -k8s.io/utils/io -k8s.io/utils/keymutex -k8s.io/utils/mount -k8s.io/utils/net -k8s.io/utils/nsenter -k8s.io/utils/path -k8s.io/utils/pointer -k8s.io/utils/strings -k8s.io/utils/trace -# sigs.k8s.io/structured-merge-diff/v3 v3.0.0 -sigs.k8s.io/structured-merge-diff/v3/value -# sigs.k8s.io/yaml v1.2.0 -## explicit -sigs.k8s.io/yaml -# k8s.io/api => k8s.io/api v0.18.3 -# k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.3 -# k8s.io/apimachinery => k8s.io/apimachinery v0.18.3 -# k8s.io/apiserver => k8s.io/apiserver v0.18.3 -# k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.3 -# k8s.io/client-go => k8s.io/client-go v0.18.3 -# k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.3 -# k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.3 -# k8s.io/code-generator => k8s.io/code-generator v0.18.3 -# k8s.io/component-base => k8s.io/component-base v0.18.3 -# k8s.io/cri-api => k8s.io/cri-api v0.18.3 -# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.3 -# k8s.io/klog => k8s.io/klog v1.0.0 -# k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.3 -# k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.3 -# k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.3 -# k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.3 -# k8s.io/kubectl => k8s.io/kubectl v0.18.3 -# k8s.io/kubelet => k8s.io/kubelet v0.18.3 -# k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.3 -# k8s.io/metrics => k8s.io/metrics v0.18.3 -# k8s.io/node-api => k8s.io/node-api v0.18.3 -# k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.3 -# k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.18.3 -# k8s.io/sample-controller => k8s.io/sample-controller v0.18.3 From c2ca8dd44d6f39e6e6cad171dbe15ec6e055885d Mon Sep 17 00:00:00 2001 From: tizhou86 Date: Thu, 11 Jun 2020 20:56:07 +0800 Subject: [PATCH 2/5] Add the allocate logic for gpu sharing. --- pkg/scheduler/api/node_info.go | 37 ++++++++++++ pkg/scheduler/api/pod_info.go | 13 +++++ .../plugins/predicates/predicates.go | 57 ++++++++++++++++--- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 7b035022d7..a489553daa 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -403,3 +403,40 @@ func (ni *NodeInfo) GetDevicesAllGPUMemory() map[int]uint { } return res } + +func (ni *NodeInfo) GetDeviceCoreId(pod *v1.Pod) (int, bool) { + + resId := -1 + resOk := false + cMem := uint(0) + rMems := ni.GetDevicesRemainGPUMemory() + + gpuReq := uint(0) + + if len(pod.ObjectMeta.Annotations) > 0 { + req, ok := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] + if ok { + s, _ := strconv.Atoi(req) + gpuReq = uint(s) + } + } + + if gpuReq > uint(0) { + if len(rMems) > 0 { + for i := 0; i < len(ni.Devices); i++ { + rMem, ok := rMems[i] + if ok { + if rMem >= gpuReq { + if resId == -1 || cMem > rMem { + resId = i + cMem = rMem + } + resOk = true + } + } + } + } + } + + return resId, resOk +} diff --git a/pkg/scheduler/api/pod_info.go b/pkg/scheduler/api/pod_info.go index 56a94034bb..0a0e68ad41 100644 --- a/pkg/scheduler/api/pod_info.go +++ b/pkg/scheduler/api/pod_info.go @@ -17,6 +17,8 @@ limitations under the License. package api import ( + "fmt" + v1 "k8s.io/api/core/v1" ) @@ -71,3 +73,14 @@ func GetPodResourceWithoutInitContainers(pod *v1.Pod) *Resource { return result } + +func UpdateGPUPod(oldPod *v1.Pod, coreId int, memoryPerCore int) (newPod *v1.Pod) { + newPod = oldPod.DeepCopy() + if len(newPod.ObjectMeta.Annotations) == 0 { + newPod.ObjectMeta.Annotations = map[string]string{} + } + + newPod.ObjectMeta.Annotations["volcano.sh/gpu-core-id"] = fmt.Sprintf("%d", coreId) + + return newPod +} diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index 79a030687f..d0988eddab 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -20,6 +20,8 @@ import ( "context" "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog" @@ -121,6 +123,11 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { pods, _ := pl.List(labels.NewSelector()) nodeMap, nodeSlice := util.GenerateNodeMapAndSlice(ssn.Nodes) + predicate := enablePredicate(pp.pluginArguments) + + //TODO: (tizhou86) zhonghu will pass the client set into this function + clientSet := NewClientSetOrClientSetInstance() + // Register event handlers to update task info in PodLister & nodeMap ssn.AddEventHandler(&framework.EventHandler{ AllocateFunc: func(event *framework.Event) { @@ -128,23 +135,57 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] - if !found { - klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + nodeInfo, foundInfo := ssn.Nodes[nodeName] + + if predicate.gpuSharingEnable { + if !foundInfo { + klog.Warningf("predicates with gpu sharing, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + } else { + coreId, found := nodeInfo.GetDeviceCoreId(pod) + if found { + var err error + pod, err = clientSet.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) + newPod := api.UpdateGPUPod(pod, coreId, nodeInfo.GPUTotalMemory/nodeInfo.GPUTotalCore) + _, err = clientSet.CoreV1().Pods(newPod.Namespace).Update(newPod) + } else { + klog.Errorf("The node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace) + } + + dev, found := nodeInfo.Devices[coreId] + if !found { + + } else { + dev.PodMap[newPod.UID] = newPod + node.AddPod(pod) + } + + klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + } } else { - node.AddPod(pod) - klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + if !found { + klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + } else { + node.AddPod(pod) + klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + } } + }, DeallocateFunc: func(event *framework.Event) { pod := pl.UpdateTask(event.Task, "") nodeName := event.Task.NodeName node, found := nodeMap[nodeName] - if !found { - klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + + if predicate.gpuSharingEnable { + //TODO: (tizhou86) add deallocate logic } else { - node.RemovePod(pod) - klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + if !found { + klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + } else { + node.RemovePod(pod) + klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + } } }, }) From 38cec5c2825ad88e2db5da57c01343718aabb464 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Tue, 23 Jun 2020 15:29:18 +0800 Subject: [PATCH 3/5] Continue working on gpu sharing --- .../chart/volcano/templates/scheduler.yaml | 2 +- installer/volcano-development.yaml | 2 +- pkg/scheduler/api/device_info.go | 67 +++--- pkg/scheduler/api/node_info.go | 194 +++++++----------- pkg/scheduler/api/pod_info.go | 43 +++- pkg/scheduler/api/well_known_labels.go | 30 +++ pkg/scheduler/framework/session.go | 5 + pkg/scheduler/plugins/predicates/gpu.go | 56 +++++ .../plugins/predicates/predicates.go | 116 +++++------ 9 files changed, 290 insertions(+), 225 deletions(-) create mode 100644 pkg/scheduler/api/well_known_labels.go create mode 100644 pkg/scheduler/plugins/predicates/gpu.go diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml index 26252969a8..b810442d9d 100644 --- a/installer/helm/chart/volcano/templates/scheduler.yaml +++ b/installer/helm/chart/volcano/templates/scheduler.yaml @@ -31,7 +31,7 @@ rules: verbs: ["create", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["pods", "pods/status"] - verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"] + verbs: ["create", "get", "list", "watch", "update", "patch", "bind", "updateStatus", "delete"] - apiGroups: [""] resources: ["pods/binding"] verbs: ["create"] diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index a030715211..83b07d095a 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -50,7 +50,7 @@ rules: verbs: ["create", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["pods", "pods/status"] - verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"] + verbs: ["create", "get", "list", "watch", "update", "patch", "bind", "updateStatus", "delete"] - apiGroups: [""] resources: ["pods/binding"] verbs: ["create"] diff --git a/pkg/scheduler/api/device_info.go b/pkg/scheduler/api/device_info.go index 1c15a56390..11edae292a 100644 --- a/pkg/scheduler/api/device_info.go +++ b/pkg/scheduler/api/device_info.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2020 The Volcano Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,47 +17,56 @@ limitations under the License. package api import ( - "strconv" - v1 "k8s.io/api/core/v1" ) -type DeviceInfo struct { - Id int - PodMap map[string]*v1.Pod - GPUTotalMemory uint +// GPUDevice include gpu id, memory and the pods that are sharing it. +type GPUDevice struct { + // GPU ID + ID int + // The pods that are sharing this GPU + PodMap map[string]*v1.Pod + // memory per card + Memory uint } -func (di *DeviceInfo) GetPods() []*v1.Pod { - pods := []*v1.Pod{} - for _, pod := range di.PodMap { - pods = append(pods, pod) +// NewGPUDevice creates a device +func NewGPUDevice(id int, mem uint) *GPUDevice { + return &GPUDevice{ + ID: id, + Memory: mem, + PodMap: map[string]*v1.Pod{}, } - return pods } -func NewDeviceInfo(id int, mem uint) *DeviceInfo { - return &DeviceInfo{ - Id: id, - GPUTotalMemory: mem, - PodMap: map[string]*v1.Pod{}, - } -} - -func (di *DeviceInfo) GetUsedGPUMemory() uint { +// getUsedGPUMemory calculates the used memory of the device. +func (g *GPUDevice) getUsedGPUMemory() uint { res := uint(0) - for _, pod := range di.PodMap { + for _, pod := range g.PodMap { if pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { continue } else { - if len(pod.ObjectMeta.Annotations) > 0 { - mem, found := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] - if found { - m, _ := strconv.Atoi(mem) - res += uint(m) - } - } + gpuRequest := GetGPUResourceOfPod(pod) + res += gpuRequest } } return res } + +// GetGPUResourceOfPod returns the GPU resource required by the pod. +func GetGPUResourceOfPod(pod *v1.Pod) uint { + var mem uint + for _, container := range pod.Spec.Containers { + mem += getGPUResourceOfContainer(&container) + } + return mem +} + +// getGPUResourceOfPod returns the GPU resource required by the container. +func getGPUResourceOfContainer(container *v1.Container) uint { + var mem uint + if val, ok := container.Resources.Limits[VolcanoGPUResource]; ok { + mem = uint(val.Value()) + } + return mem +} diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index a489553daa..7737a66ae0 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -18,7 +18,6 @@ package api import ( "fmt" - "strconv" v1 "k8s.io/api/core/v1" "k8s.io/klog" @@ -47,12 +46,9 @@ type NodeInfo struct { Tasks map[TaskID]*TaskInfo - Devices map[int]*DeviceInfo - GPUTotalCore int - GPUTotalMemory int - // Used to store custom information - Others map[string]interface{} + Others map[string]interface{} + GPUDevices map[int]*GPUDevice } // FutureIdle returns resources that will be idle in the future: @@ -70,49 +66,31 @@ type NodeState struct { // NewNodeInfo is used to create new nodeInfo object func NewNodeInfo(node *v1.Node) *NodeInfo { - var ni *NodeInfo - - if node == nil { - ni = &NodeInfo{ - Releasing: EmptyResource(), - Pipelined: EmptyResource(), - Idle: EmptyResource(), - Used: EmptyResource(), - - Allocatable: EmptyResource(), - Capability: EmptyResource(), - - Tasks: make(map[TaskID]*TaskInfo), + nodeinfo := &NodeInfo{ + Releasing: EmptyResource(), + Pipelined: EmptyResource(), + Idle: EmptyResource(), + Used: EmptyResource(), - Devices: make(map[int]*DeviceInfo), - GPUTotalCore: 0, - GPUTotalMemory: 0, - } - } else { - ni = &NodeInfo{ - Name: node.Name, - Node: node, - - Releasing: EmptyResource(), - Pipelined: EmptyResource(), - Idle: NewResource(node.Status.Allocatable), - Used: EmptyResource(), - - Allocatable: NewResource(node.Status.Allocatable), - Capability: NewResource(node.Status.Capacity), + Allocatable: EmptyResource(), + Capability: EmptyResource(), - Tasks: make(map[TaskID]*TaskInfo), + Tasks: make(map[TaskID]*TaskInfo), - Devices: make(map[int]*DeviceInfo), - GPUTotalCore: 0, - GPUTotalMemory: 0, - } + GPUDevices: make(map[int]*GPUDevice), } - ni.SetNodeGPUInfo(node) - ni.setNodeState(node) + if node != nil { + nodeinfo.Name = node.Name + nodeinfo.Node = node + nodeinfo.Idle = NewResource(node.Status.Allocatable) + nodeinfo.Allocatable = NewResource(node.Status.Allocatable) + nodeinfo.Capability = NewResource(node.Status.Capacity) + } + nodeinfo.setNodeGPUInfo(node) + nodeinfo.setNodeState(node) - return ni + return nodeinfo } // Clone used to clone nodeInfo Object @@ -122,7 +100,6 @@ func (ni *NodeInfo) Clone() *NodeInfo { for _, p := range ni.Tasks { res.AddTask(p) } - res.Others = ni.Others return res } @@ -168,27 +145,35 @@ func (ni *NodeInfo) setNodeState(node *v1.Node) { } } -func (ni *NodeInfo) SetNodeGPUInfo(node *v1.Node) { - - core, ok := node.Status.Capacity["volcano.sh/node-gpu-core"] - if ok { - ni.GPUTotalCore = int(core.Value()) +func (ni *NodeInfo) setNodeGPUInfo(node *v1.Node) { + if node == nil { + return } - - mem, ok := node.Status.Capacity["volcano.sh/node-gpu-memory"] - if ok { - ni.GPUTotalMemory = int(mem.Value()) + memory, ok := node.Status.Capacity[VolcanoGPUResource] + if !ok { + return } + totalMemory := memory.Value() - for i := 0; i < int(core.Value()); i++ { - ni.Devices[i] = NewDeviceInfo(i, uint(int(mem.Value())/int(core.Value()))) + res, ok := node.Status.Capacity[VolcanoGPUNumber] + if !ok { + return + } + gpuNumber := res.Value() + if gpuNumber == 0 { + klog.Warningf("invalid %s=%s", VolcanoGPUNumber, res) } + memoryPerCard := uint(totalMemory / gpuNumber) + for i := 0; i < int(gpuNumber); i++ { + ni.GPUDevices[i] = NewGPUDevice(i, memoryPerCard) + } } // SetNode sets kubernetes node object to nodeInfo object func (ni *NodeInfo) SetNode(node *v1.Node) { ni.setNodeState(node) + ni.setNodeGPUInfo(node) if !ni.Ready() { klog.Warningf("Failed to set node info, phase: %s, reason: %s", @@ -212,11 +197,13 @@ func (ni *NodeInfo) SetNode(node *v1.Node) { ni.Idle.Sub(ti.Resreq) ni.Releasing.Add(ti.Resreq) ni.Used.Add(ti.Resreq) + ni.AddGPUResource(ti.Pod) case Pipelined: ni.Pipelined.Add(ti.Resreq) default: ni.Idle.Sub(ti.Resreq) ni.Used.Add(ti.Resreq) + ni.AddGPUResource(ti.Pod) } } } @@ -257,6 +244,7 @@ func (ni *NodeInfo) AddTask(task *TaskInfo) error { } ni.Releasing.Add(ti.Resreq) ni.Used.Add(ti.Resreq) + ni.AddGPUResource(ti.Pod) case Pipelined: ni.Pipelined.Add(ti.Resreq) default: @@ -264,6 +252,7 @@ func (ni *NodeInfo) AddTask(task *TaskInfo) error { return err } ni.Used.Add(ti.Resreq) + ni.AddGPUResource(ti.Pod) } } @@ -293,11 +282,13 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error { ni.Releasing.Sub(task.Resreq) ni.Idle.Add(task.Resreq) ni.Used.Sub(task.Resreq) + ni.AddGPUResource(ti.Pod) case Pipelined: ni.Pipelined.Sub(task.Resreq) default: ni.Idle.Add(task.Resreq) ni.Used.Sub(task.Resreq) + ni.SubGPUResource(ti.Pod) } } @@ -347,38 +338,10 @@ func (ni *NodeInfo) Pods() (pods []*v1.Pod) { return } -func (ni *NodeInfo) CheckPredicatePodOnGPUNode(pod *v1.Pod) bool { - res := false - memReq := uint(0) - - remainMems := ni.GetDevicesRemainGPUMemory() - if len(pod.ObjectMeta.Annotations) > 0 { - mem, found := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] - if found { - m, _ := strconv.Atoi(mem) - memReq = uint(m) - } - } - - if len(remainMems) > 0 { - for devID := 0; devID < len(ni.Devices); devID++ { - availableGPU, ok := remainMems[devID] - if ok { - if availableGPU >= memReq { - res = true - break - } - } - } - } - - return res - -} - -func (ni *NodeInfo) GetDevicesRemainGPUMemory() map[int]uint { - devicesAllGPUMemory := ni.GetDevicesAllGPUMemory() - devicesUsedGPUMemory := ni.GetDevicesUsedGPUMemory() +// GetDevicesIdleGPUMemory returns all the idle GPU memory by gpu card. +func (ni *NodeInfo) GetDevicesIdleGPUMemory() map[int]uint { + devicesAllGPUMemory := ni.getDevicesAllGPUMemory() + devicesUsedGPUMemory := ni.getDevicesUsedGPUMemory() res := map[int]uint{} for id, allMemory := range devicesAllGPUMemory { if usedMemory, found := devicesUsedGPUMemory[id]; found { @@ -388,55 +351,40 @@ func (ni *NodeInfo) GetDevicesRemainGPUMemory() map[int]uint { return res } -func (ni *NodeInfo) GetDevicesUsedGPUMemory() map[int]uint { +func (ni *NodeInfo) getDevicesUsedGPUMemory() map[int]uint { res := map[int]uint{} - for _, device := range ni.Devices { - res[device.Id] = device.GetUsedGPUMemory() + for _, device := range ni.GPUDevices { + res[device.ID] = device.getUsedGPUMemory() } return res } -func (ni *NodeInfo) GetDevicesAllGPUMemory() map[int]uint { +func (ni *NodeInfo) getDevicesAllGPUMemory() map[int]uint { res := map[int]uint{} - for _, device := range ni.Devices { - res[device.Id] = device.GPUTotalMemory + for _, device := range ni.GPUDevices { + res[device.ID] = device.Memory } return res } -func (ni *NodeInfo) GetDeviceCoreId(pod *v1.Pod) (int, bool) { - - resId := -1 - resOk := false - cMem := uint(0) - rMems := ni.GetDevicesRemainGPUMemory() - - gpuReq := uint(0) - - if len(pod.ObjectMeta.Annotations) > 0 { - req, ok := pod.ObjectMeta.Annotations["volcano.sh/pod-gpu-memory"] - if ok { - s, _ := strconv.Atoi(req) - gpuReq = uint(s) +// AddGPUResource adds the pod to GPU pool if it is assigned +func (ni *NodeInfo) AddGPUResource(pod *v1.Pod) { + gpuRes := GetGPUResourceOfPod(pod) + if gpuRes > 0 { + id := GetGPUIndex(pod) + if dev := ni.GPUDevices[id]; dev != nil { + dev.PodMap[string(pod.UID)] = pod } } +} - if gpuReq > uint(0) { - if len(rMems) > 0 { - for i := 0; i < len(ni.Devices); i++ { - rMem, ok := rMems[i] - if ok { - if rMem >= gpuReq { - if resId == -1 || cMem > rMem { - resId = i - cMem = rMem - } - resOk = true - } - } - } +// SubGPUResource frees the gpu hold by the pod +func (ni *NodeInfo) SubGPUResource(pod *v1.Pod) { + gpuRes := GetGPUResourceOfPod(pod) + if gpuRes > 0 { + id := GetGPUIndex(pod) + if dev := ni.GPUDevices[id]; dev != nil { + delete(dev.PodMap, string(pod.UID)) } } - - return resId, resOk } diff --git a/pkg/scheduler/api/pod_info.go b/pkg/scheduler/api/pod_info.go index 0a0e68ad41..56405ecc77 100644 --- a/pkg/scheduler/api/pod_info.go +++ b/pkg/scheduler/api/pod_info.go @@ -18,8 +18,12 @@ package api import ( "fmt" + "strconv" + "strings" + "time" v1 "k8s.io/api/core/v1" + "k8s.io/klog" ) // Refer k8s.io/kubernetes/pkg/scheduler/algorithm/predicates/predicates.go#GetResourceRequest. @@ -74,13 +78,40 @@ func GetPodResourceWithoutInitContainers(pod *v1.Pod) *Resource { return result } -func UpdateGPUPod(oldPod *v1.Pod, coreId int, memoryPerCore int) (newPod *v1.Pod) { - newPod = oldPod.DeepCopy() - if len(newPod.ObjectMeta.Annotations) == 0 { - newPod.ObjectMeta.Annotations = map[string]string{} +// GetGPUIndex returns the ID of the GPU +func GetGPUIndex(pod *v1.Pod) int { + if len(pod.Annotations) > 0 { + value, found := pod.Annotations[GPUIndex] + if found { + id, err := strconv.Atoi(value) + if err != nil { + klog.Error("invalid %s=%s", GPUIndex, value) + return -1 + } + return id + } } - newPod.ObjectMeta.Annotations["volcano.sh/gpu-core-id"] = fmt.Sprintf("%d", coreId) + return -1 +} + +func escapeJSONPointer(p string) string { + // Escaping reference name using https://tools.ietf.org/html/rfc6901 + p = strings.Replace(p, "~", "~0", -1) + p = strings.Replace(p, "/", "~1", -1) + return p +} + +// AddGPUIndexPatch returns the patch adding GPU index +func AddGPUIndexPatch(id int) string { + return fmt.Sprintf(`[{"op": "add", "path": "/metadata/annotations/%s", "value":"%d"},`+ + `{"op": "add", "path": "/metadata/annotations/%s", "value": "%d"}]`, + escapeJSONPointer(PredicateTime), time.Now().UnixNano(), + escapeJSONPointer(GPUIndex), id) +} - return newPod +// RemoveGPUIndexPatch returns the patch removing GPU index +func RemoveGPUIndexPatch() string { + return fmt.Sprintf(`[{"op": "remove", "path": "/metadata/annotations/%s"},`+ + `{"op": "remove", "path": "/metadata/annotations/%s"]`, escapeJSONPointer(PredicateTime), escapeJSONPointer(GPUIndex)) } diff --git a/pkg/scheduler/api/well_known_labels.go b/pkg/scheduler/api/well_known_labels.go new file mode 100644 index 0000000000..c412d24529 --- /dev/null +++ b/pkg/scheduler/api/well_known_labels.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +const ( + + // VolcanoGPUResource extended gpu resource + VolcanoGPUResource = "volcano.sh/gpu-memory" + // VolcanoGPUNumber virtual GPU card number + VolcanoGPUNumber = "volcano.sh/gpu-number" + + // PredicateTime is the key of predicate time + PredicateTime = "volcano.sh/predicate-time" + // GPUIndex is the key of gpu index + GPUIndex = "volcano.sh/gpu-index" +) diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index b67f9b1f01..ac0e5c977f 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -401,6 +401,11 @@ func (ssn *Session) AddEventHandler(eh *EventHandler) { ssn.eventHandlers = append(ssn.eventHandlers, eh) } +// KubeClient returns the kubernetes client +func (ssn Session) KubeClient() kubernetes.Interface { + return ssn.kubeClient +} + //String return nodes and jobs information in the session func (ssn Session) String() string { msg := fmt.Sprintf("Session %v: \n", ssn.UID) diff --git a/pkg/scheduler/plugins/predicates/gpu.go b/pkg/scheduler/plugins/predicates/gpu.go new file mode 100644 index 0000000000..cd28ac0db4 --- /dev/null +++ b/pkg/scheduler/plugins/predicates/gpu.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicates + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + + "volcano.sh/volcano/pkg/scheduler/api" +) + +// checkNodeGPUSharingPredicate checks if a gpu sharing pod can be scheduled on a node. +func checkNodeGPUSharingPredicate(pod *v1.Pod, nodeInfo *api.NodeInfo) (bool, error) { + // no gpu sharing request + if api.GetGPUResourceOfPod(pod) <= 0 { + return true, nil + } + + id := predicateGPU(pod, nodeInfo) + if id < 0 { + return false, fmt.Errorf("no enough gpu memory on single device of node %s", nodeInfo.Name) + } + return true, nil +} + +// predicateGPU returns the available GPU ID +func predicateGPU(pod *v1.Pod, node *api.NodeInfo) int { + gpuRequest := api.GetGPUResourceOfPod(pod) + allocatableGPUs := node.GetDevicesIdleGPUMemory() + + for devID := 0; devID < len(allocatableGPUs); devID++ { + availableGPU, ok := allocatableGPUs[devID] + if ok { + if availableGPU >= gpuRequest { + return devID + } + } + } + + return -1 +} diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index d0988eddab..ac8c524432 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -21,9 +21,8 @@ import ( "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" @@ -100,7 +99,7 @@ func enablePredicate(args framework.Arguments) predicateEnable { memoryPressureEnable: false, diskPressureEnable: false, pidPressureEnable: false, - gpuSharingEnable: false, + gpuSharingEnable: true, // enable for debug } // Checks whether predicate.MemoryPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. @@ -125,9 +124,7 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { predicate := enablePredicate(pp.pluginArguments) - //TODO: (tizhou86) zhonghu will pass the client set into this function - clientSet := NewClientSetOrClientSetInstance() - + kubeClient := ssn.KubeClient() // Register event handlers to update task info in PodLister & nodeMap ssn.AddEventHandler(&framework.EventHandler{ AllocateFunc: func(event *framework.Event) { @@ -135,58 +132,62 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { nodeName := event.Task.NodeName node, found := nodeMap[nodeName] - nodeInfo, foundInfo := ssn.Nodes[nodeName] - - if predicate.gpuSharingEnable { - if !foundInfo { - klog.Warningf("predicates with gpu sharing, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) - } else { - coreId, found := nodeInfo.GetDeviceCoreId(pod) - if found { - var err error - pod, err = clientSet.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) - newPod := api.UpdateGPUPod(pod, coreId, nodeInfo.GPUTotalMemory/nodeInfo.GPUTotalCore) - _, err = clientSet.CoreV1().Pods(newPod.Namespace).Update(newPod) - } else { - klog.Errorf("The node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace) - } - - dev, found := nodeInfo.Devices[coreId] - if !found { - - } else { - dev.PodMap[newPod.UID] = newPod - node.AddPod(pod) - } - - klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + if !found { + klog.Errorf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + return + } + + if predicate.gpuSharingEnable && api.GetGPUResourceOfPod(pod) > 0 { + nodeInfo, _ := ssn.Nodes[nodeName] + id := predicateGPU(pod, nodeInfo) + if id < 0 { + klog.Errorf("The node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace) + return } - } else { - if !found { - klog.Warningf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) - } else { - node.AddPod(pod) - klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) + patch := api.AddGPUIndexPatch(id) + pod, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err) + return } + dev, _ := nodeInfo.GPUDevices[id] + dev.PodMap[string(pod.UID)] = pod + klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) } + node.AddPod(pod) + klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName) }, DeallocateFunc: func(event *framework.Event) { pod := pl.UpdateTask(event.Task, "") - nodeName := event.Task.NodeName node, found := nodeMap[nodeName] + if !found { + klog.Errorf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) + return + } + + if predicate.gpuSharingEnable && api.GetGPUResourceOfPod(pod) > 0 { + // deallocate pod gpu id + id := api.GetGPUIndex(pod) + patch := api.RemoveGPUIndexPatch() + _, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err) + return + } - if predicate.gpuSharingEnable { - //TODO: (tizhou86) add deallocate logic - } else { - if !found { - klog.Warningf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName) - } else { - node.RemovePod(pod) - klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + nodeInfo, _ := ssn.Nodes[nodeName] + if dev, ok := nodeInfo.GPUDevices[id]; ok { + delete(dev.PodMap, string(pod.UID)) } + + klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) } + + node.RemovePod(pod) + klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName) + }, }) @@ -258,16 +259,15 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { return fmt.Errorf("plugin %s predicates failed %s", interpodaffinity.Name, status.Message()) } - // if predicate.gpuSharingEnable - if true { + if predicate.gpuSharingEnable { // CheckGPUSharingPredicate - fit, err := CheckNodeGPUSharingPredicate(task.Pod, node) + fit, err := checkNodeGPUSharingPredicate(task.Pod, node) if err != nil { return err } - klog.V(4).Infof("CheckNodeGPUSharingPredicate predicates Task <%s/%s> on Node <%s>: fit %t, err %v", - task.Namespace, task.Name, node.Name, fit, err) + klog.V(4).Infof("checkNodeGPUSharingPredicate predicates Task <%s/%s> on Node <%s>: fit %v", + task.Namespace, task.Name, node.Name, fit) } return nil @@ -275,17 +275,3 @@ func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) { } func (pp *predicatesPlugin) OnSessionClose(ssn *framework.Session) {} - -// CheckNodeGPUSharingPredicate checks if a gpu sharing pod can be scheduled on a node. -func CheckNodeGPUSharingPredicate(pod *v1.Pod, nodeInfo *api.NodeInfo) (bool, error) { - _, ok := nodeInfo.Node.Status.Capacity["volcano.sh/node-gpu-core"] - if !ok { - return false, fmt.Errorf("node is not gpu sharing") - } else { - isEnoughGPUMemoryOnNode := nodeInfo.CheckPredicatePodOnGPUNode(pod) - if !isEnoughGPUMemoryOnNode { - return false, fmt.Errorf("no enough gpu memory on single device") - } - } - return true, nil -} From 2328308f2c78f9dacebb1640d68222fb32c17293 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Wed, 1 Jul 2020 17:55:26 +0800 Subject: [PATCH 4/5] go mod --- vendor/modules.txt | 714 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 714 insertions(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index e69de29bb2..dee69dc0f8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -0,0 +1,714 @@ +# cloud.google.com/go v0.38.0 +cloud.google.com/go/compute/metadata +# github.com/beorn7/perks v1.0.0 +github.com/beorn7/perks/quantile +# github.com/blang/semver v3.5.0+incompatible +github.com/blang/semver +# github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e +github.com/coreos/go-systemd/journal +# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea +github.com/coreos/pkg/capnslog +# github.com/davecgh/go-spew v1.1.1 +github.com/davecgh/go-spew/spew +# github.com/docker/distribution v2.7.1+incompatible +github.com/docker/distribution/digestset +github.com/docker/distribution/reference +# github.com/emicklei/go-restful v2.9.5+incompatible +github.com/emicklei/go-restful +github.com/emicklei/go-restful/log +# github.com/evanphx/json-patch v4.2.0+incompatible +github.com/evanphx/json-patch +# github.com/gogo/protobuf v1.3.1 +github.com/gogo/protobuf/gogoproto +github.com/gogo/protobuf/proto +github.com/gogo/protobuf/protoc-gen-gogo/descriptor +github.com/gogo/protobuf/sortkeys +# github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 +github.com/golang/groupcache/lru +# github.com/golang/protobuf v1.3.2 +github.com/golang/protobuf/proto +github.com/golang/protobuf/ptypes +github.com/golang/protobuf/ptypes/any +github.com/golang/protobuf/ptypes/duration +github.com/golang/protobuf/ptypes/timestamp +# github.com/google/go-cmp v0.3.0 +github.com/google/go-cmp/cmp +github.com/google/go-cmp/cmp/internal/diff +github.com/google/go-cmp/cmp/internal/flags +github.com/google/go-cmp/cmp/internal/function +github.com/google/go-cmp/cmp/internal/value +# github.com/google/gofuzz v1.1.0 +github.com/google/gofuzz +# github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +## explicit +github.com/google/shlex +# github.com/google/uuid v1.1.1 +github.com/google/uuid +# github.com/googleapis/gnostic v0.1.0 +github.com/googleapis/gnostic/OpenAPIv2 +github.com/googleapis/gnostic/compiler +github.com/googleapis/gnostic/extensions +# github.com/hashicorp/errwrap v1.0.0 +github.com/hashicorp/errwrap +# github.com/hashicorp/go-multierror v1.0.0 +## explicit +github.com/hashicorp/go-multierror +# github.com/hashicorp/golang-lru v0.5.1 +github.com/hashicorp/golang-lru +github.com/hashicorp/golang-lru/simplelru +# github.com/hpcloud/tail v1.0.0 +github.com/hpcloud/tail +github.com/hpcloud/tail/ratelimiter +github.com/hpcloud/tail/util +github.com/hpcloud/tail/watch +github.com/hpcloud/tail/winfile +# github.com/imdario/mergo v0.3.5 +github.com/imdario/mergo +# github.com/inconshreveable/mousetrap v1.0.0 +github.com/inconshreveable/mousetrap +# github.com/json-iterator/go v1.1.8 +github.com/json-iterator/go +# github.com/matttproud/golang_protobuf_extensions v1.0.1 +github.com/matttproud/golang_protobuf_extensions/pbutil +# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd +github.com/modern-go/concurrent +# github.com/modern-go/reflect2 v1.0.1 +github.com/modern-go/reflect2 +# github.com/onsi/ginkgo v1.11.0 +## explicit +github.com/onsi/ginkgo +github.com/onsi/ginkgo/config +github.com/onsi/ginkgo/internal/codelocation +github.com/onsi/ginkgo/internal/containernode +github.com/onsi/ginkgo/internal/failer +github.com/onsi/ginkgo/internal/leafnodes +github.com/onsi/ginkgo/internal/remote +github.com/onsi/ginkgo/internal/spec +github.com/onsi/ginkgo/internal/spec_iterator +github.com/onsi/ginkgo/internal/specrunner +github.com/onsi/ginkgo/internal/suite +github.com/onsi/ginkgo/internal/testingtproxy +github.com/onsi/ginkgo/internal/writer +github.com/onsi/ginkgo/reporters +github.com/onsi/ginkgo/reporters/stenographer +github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable +github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty +github.com/onsi/ginkgo/types +# github.com/onsi/gomega v1.7.0 +## explicit +github.com/onsi/gomega +github.com/onsi/gomega/format +github.com/onsi/gomega/internal/assertion +github.com/onsi/gomega/internal/asyncassertion +github.com/onsi/gomega/internal/oraclematcher +github.com/onsi/gomega/internal/testingtsupport +github.com/onsi/gomega/matchers +github.com/onsi/gomega/matchers/support/goraph/bipartitegraph +github.com/onsi/gomega/matchers/support/goraph/edge +github.com/onsi/gomega/matchers/support/goraph/node +github.com/onsi/gomega/matchers/support/goraph/util +github.com/onsi/gomega/types +# github.com/opencontainers/go-digest v1.0.0-rc1 +github.com/opencontainers/go-digest +# github.com/prometheus/client_golang v1.0.0 +## explicit +github.com/prometheus/client_golang/prometheus +github.com/prometheus/client_golang/prometheus/internal +github.com/prometheus/client_golang/prometheus/promauto +github.com/prometheus/client_golang/prometheus/promhttp +# github.com/prometheus/client_model v0.2.0 +github.com/prometheus/client_model/go +# github.com/prometheus/common v0.4.1 +github.com/prometheus/common/expfmt +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +github.com/prometheus/common/model +# github.com/prometheus/procfs v0.0.2 +github.com/prometheus/procfs +github.com/prometheus/procfs/internal/fs +# github.com/spf13/cobra v0.0.5 +## explicit +github.com/spf13/cobra +# github.com/spf13/pflag v1.0.5 +## explicit +github.com/spf13/pflag +# go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 +go.etcd.io/etcd/auth/authpb +go.etcd.io/etcd/clientv3 +go.etcd.io/etcd/clientv3/balancer +go.etcd.io/etcd/clientv3/balancer/connectivity +go.etcd.io/etcd/clientv3/balancer/picker +go.etcd.io/etcd/clientv3/balancer/resolver/endpoint +go.etcd.io/etcd/clientv3/credentials +go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes +go.etcd.io/etcd/etcdserver/etcdserverpb +go.etcd.io/etcd/mvcc/mvccpb +go.etcd.io/etcd/pkg/logutil +go.etcd.io/etcd/pkg/systemd +go.etcd.io/etcd/pkg/types +go.etcd.io/etcd/raft +go.etcd.io/etcd/raft/confchange +go.etcd.io/etcd/raft/quorum +go.etcd.io/etcd/raft/raftpb +go.etcd.io/etcd/raft/tracker +# go.uber.org/atomic v1.3.2 +go.uber.org/atomic +# go.uber.org/multierr v1.1.0 +go.uber.org/multierr +# go.uber.org/zap v1.10.0 +go.uber.org/zap +go.uber.org/zap/buffer +go.uber.org/zap/internal/bufferpool +go.uber.org/zap/internal/color +go.uber.org/zap/internal/exit +go.uber.org/zap/zapcore +# golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 +## explicit +golang.org/x/crypto/blowfish +golang.org/x/crypto/chacha20 +golang.org/x/crypto/curve25519 +golang.org/x/crypto/ed25519 +golang.org/x/crypto/ed25519/internal/edwards25519 +golang.org/x/crypto/internal/subtle +golang.org/x/crypto/poly1305 +golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/internal/bcrypt_pbkdf +golang.org/x/crypto/ssh/terminal +# golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 +golang.org/x/net/context +golang.org/x/net/context/ctxhttp +golang.org/x/net/html +golang.org/x/net/html/atom +golang.org/x/net/html/charset +golang.org/x/net/http/httpguts +golang.org/x/net/http2 +golang.org/x/net/http2/hpack +golang.org/x/net/idna +golang.org/x/net/internal/timeseries +golang.org/x/net/trace +# golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 +golang.org/x/oauth2 +golang.org/x/oauth2/google +golang.org/x/oauth2/internal +golang.org/x/oauth2/jws +golang.org/x/oauth2/jwt +# golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 +golang.org/x/sys/cpu +golang.org/x/sys/unix +golang.org/x/sys/windows +# golang.org/x/text v0.3.2 +golang.org/x/text/encoding +golang.org/x/text/encoding/charmap +golang.org/x/text/encoding/htmlindex +golang.org/x/text/encoding/internal +golang.org/x/text/encoding/internal/identifier +golang.org/x/text/encoding/japanese +golang.org/x/text/encoding/korean +golang.org/x/text/encoding/simplifiedchinese +golang.org/x/text/encoding/traditionalchinese +golang.org/x/text/encoding/unicode +golang.org/x/text/internal/language +golang.org/x/text/internal/language/compact +golang.org/x/text/internal/tag +golang.org/x/text/internal/utf8internal +golang.org/x/text/language +golang.org/x/text/runes +golang.org/x/text/secure/bidirule +golang.org/x/text/transform +golang.org/x/text/unicode/bidi +golang.org/x/text/unicode/norm +# golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 +## explicit +golang.org/x/time/rate +# google.golang.org/appengine v1.5.0 +google.golang.org/appengine +google.golang.org/appengine/internal +google.golang.org/appengine/internal/app_identity +google.golang.org/appengine/internal/base +google.golang.org/appengine/internal/datastore +google.golang.org/appengine/internal/log +google.golang.org/appengine/internal/modules +google.golang.org/appengine/internal/remote_api +google.golang.org/appengine/internal/urlfetch +google.golang.org/appengine/urlfetch +# google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 +google.golang.org/genproto/googleapis/rpc/status +# google.golang.org/grpc v1.26.0 +google.golang.org/grpc +google.golang.org/grpc/attributes +google.golang.org/grpc/backoff +google.golang.org/grpc/balancer +google.golang.org/grpc/balancer/base +google.golang.org/grpc/balancer/roundrobin +google.golang.org/grpc/binarylog/grpc_binarylog_v1 +google.golang.org/grpc/codes +google.golang.org/grpc/connectivity +google.golang.org/grpc/credentials +google.golang.org/grpc/credentials/internal +google.golang.org/grpc/encoding +google.golang.org/grpc/encoding/proto +google.golang.org/grpc/grpclog +google.golang.org/grpc/internal +google.golang.org/grpc/internal/backoff +google.golang.org/grpc/internal/balancerload +google.golang.org/grpc/internal/binarylog +google.golang.org/grpc/internal/buffer +google.golang.org/grpc/internal/channelz +google.golang.org/grpc/internal/envconfig +google.golang.org/grpc/internal/grpcrand +google.golang.org/grpc/internal/grpcsync +google.golang.org/grpc/internal/resolver/dns +google.golang.org/grpc/internal/resolver/passthrough +google.golang.org/grpc/internal/syscall +google.golang.org/grpc/internal/transport +google.golang.org/grpc/keepalive +google.golang.org/grpc/metadata +google.golang.org/grpc/naming +google.golang.org/grpc/peer +google.golang.org/grpc/resolver +google.golang.org/grpc/resolver/dns +google.golang.org/grpc/resolver/passthrough +google.golang.org/grpc/serviceconfig +google.golang.org/grpc/stats +google.golang.org/grpc/status +google.golang.org/grpc/tap +# gopkg.in/fsnotify.v1 v1.4.7 +gopkg.in/fsnotify.v1 +# gopkg.in/inf.v0 v0.9.1 +gopkg.in/inf.v0 +# gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 +gopkg.in/tomb.v1 +# gopkg.in/yaml.v2 v2.2.8 +## explicit +gopkg.in/yaml.v2 +# k8s.io/api v0.18.3 => k8s.io/api v0.18.3 +## explicit +k8s.io/api/admission/v1beta1 +k8s.io/api/admissionregistration/v1 +k8s.io/api/admissionregistration/v1beta1 +k8s.io/api/apps/v1 +k8s.io/api/apps/v1beta1 +k8s.io/api/apps/v1beta2 +k8s.io/api/auditregistration/v1alpha1 +k8s.io/api/authentication/v1 +k8s.io/api/authentication/v1beta1 +k8s.io/api/authorization/v1 +k8s.io/api/authorization/v1beta1 +k8s.io/api/autoscaling/v1 +k8s.io/api/autoscaling/v2beta1 +k8s.io/api/autoscaling/v2beta2 +k8s.io/api/batch/v1 +k8s.io/api/batch/v1beta1 +k8s.io/api/batch/v2alpha1 +k8s.io/api/certificates/v1beta1 +k8s.io/api/coordination/v1 +k8s.io/api/coordination/v1beta1 +k8s.io/api/core/v1 +k8s.io/api/discovery/v1alpha1 +k8s.io/api/discovery/v1beta1 +k8s.io/api/events/v1beta1 +k8s.io/api/extensions/v1beta1 +k8s.io/api/flowcontrol/v1alpha1 +k8s.io/api/networking/v1 +k8s.io/api/networking/v1beta1 +k8s.io/api/node/v1alpha1 +k8s.io/api/node/v1beta1 +k8s.io/api/policy/v1beta1 +k8s.io/api/rbac/v1 +k8s.io/api/rbac/v1alpha1 +k8s.io/api/rbac/v1beta1 +k8s.io/api/scheduling/v1 +k8s.io/api/scheduling/v1alpha1 +k8s.io/api/scheduling/v1beta1 +k8s.io/api/settings/v1alpha1 +k8s.io/api/storage/v1 +k8s.io/api/storage/v1alpha1 +k8s.io/api/storage/v1beta1 +# k8s.io/apimachinery v0.18.3 => k8s.io/apimachinery v0.18.3 +## explicit +k8s.io/apimachinery/pkg/api/equality +k8s.io/apimachinery/pkg/api/errors +k8s.io/apimachinery/pkg/api/meta +k8s.io/apimachinery/pkg/api/resource +k8s.io/apimachinery/pkg/api/validation +k8s.io/apimachinery/pkg/api/validation/path +k8s.io/apimachinery/pkg/apis/meta/internalversion +k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme +k8s.io/apimachinery/pkg/apis/meta/v1 +k8s.io/apimachinery/pkg/apis/meta/v1/unstructured +k8s.io/apimachinery/pkg/apis/meta/v1/validation +k8s.io/apimachinery/pkg/apis/meta/v1beta1 +k8s.io/apimachinery/pkg/conversion +k8s.io/apimachinery/pkg/conversion/queryparams +k8s.io/apimachinery/pkg/fields +k8s.io/apimachinery/pkg/labels +k8s.io/apimachinery/pkg/runtime +k8s.io/apimachinery/pkg/runtime/schema +k8s.io/apimachinery/pkg/runtime/serializer +k8s.io/apimachinery/pkg/runtime/serializer/json +k8s.io/apimachinery/pkg/runtime/serializer/protobuf +k8s.io/apimachinery/pkg/runtime/serializer/recognizer +k8s.io/apimachinery/pkg/runtime/serializer/streaming +k8s.io/apimachinery/pkg/runtime/serializer/versioning +k8s.io/apimachinery/pkg/selection +k8s.io/apimachinery/pkg/types +k8s.io/apimachinery/pkg/util/cache +k8s.io/apimachinery/pkg/util/clock +k8s.io/apimachinery/pkg/util/diff +k8s.io/apimachinery/pkg/util/errors +k8s.io/apimachinery/pkg/util/framer +k8s.io/apimachinery/pkg/util/intstr +k8s.io/apimachinery/pkg/util/json +k8s.io/apimachinery/pkg/util/mergepatch +k8s.io/apimachinery/pkg/util/naming +k8s.io/apimachinery/pkg/util/net +k8s.io/apimachinery/pkg/util/runtime +k8s.io/apimachinery/pkg/util/sets +k8s.io/apimachinery/pkg/util/strategicpatch +k8s.io/apimachinery/pkg/util/uuid +k8s.io/apimachinery/pkg/util/validation +k8s.io/apimachinery/pkg/util/validation/field +k8s.io/apimachinery/pkg/util/wait +k8s.io/apimachinery/pkg/util/yaml +k8s.io/apimachinery/pkg/version +k8s.io/apimachinery/pkg/watch +k8s.io/apimachinery/third_party/forked/golang/json +k8s.io/apimachinery/third_party/forked/golang/reflect +# k8s.io/apiserver v0.18.3 => k8s.io/apiserver v0.18.3 +## explicit +k8s.io/apiserver/pkg/apis/audit +k8s.io/apiserver/pkg/authentication/user +k8s.io/apiserver/pkg/endpoints/metrics +k8s.io/apiserver/pkg/endpoints/request +k8s.io/apiserver/pkg/features +k8s.io/apiserver/pkg/server/healthz +k8s.io/apiserver/pkg/server/httplog +k8s.io/apiserver/pkg/server/mux +k8s.io/apiserver/pkg/storage +k8s.io/apiserver/pkg/storage/etcd3 +k8s.io/apiserver/pkg/storage/etcd3/metrics +k8s.io/apiserver/pkg/storage/value +k8s.io/apiserver/pkg/util/feature +# k8s.io/client-go v0.18.3 => k8s.io/client-go v0.18.3 +## explicit +k8s.io/client-go/discovery +k8s.io/client-go/discovery/fake +k8s.io/client-go/informers +k8s.io/client-go/informers/admissionregistration +k8s.io/client-go/informers/admissionregistration/v1 +k8s.io/client-go/informers/admissionregistration/v1beta1 +k8s.io/client-go/informers/apps +k8s.io/client-go/informers/apps/v1 +k8s.io/client-go/informers/apps/v1beta1 +k8s.io/client-go/informers/apps/v1beta2 +k8s.io/client-go/informers/auditregistration +k8s.io/client-go/informers/auditregistration/v1alpha1 +k8s.io/client-go/informers/autoscaling +k8s.io/client-go/informers/autoscaling/v1 +k8s.io/client-go/informers/autoscaling/v2beta1 +k8s.io/client-go/informers/autoscaling/v2beta2 +k8s.io/client-go/informers/batch +k8s.io/client-go/informers/batch/v1 +k8s.io/client-go/informers/batch/v1beta1 +k8s.io/client-go/informers/batch/v2alpha1 +k8s.io/client-go/informers/certificates +k8s.io/client-go/informers/certificates/v1beta1 +k8s.io/client-go/informers/coordination +k8s.io/client-go/informers/coordination/v1 +k8s.io/client-go/informers/coordination/v1beta1 +k8s.io/client-go/informers/core +k8s.io/client-go/informers/core/v1 +k8s.io/client-go/informers/discovery +k8s.io/client-go/informers/discovery/v1alpha1 +k8s.io/client-go/informers/discovery/v1beta1 +k8s.io/client-go/informers/events +k8s.io/client-go/informers/events/v1beta1 +k8s.io/client-go/informers/extensions +k8s.io/client-go/informers/extensions/v1beta1 +k8s.io/client-go/informers/flowcontrol +k8s.io/client-go/informers/flowcontrol/v1alpha1 +k8s.io/client-go/informers/internalinterfaces +k8s.io/client-go/informers/networking +k8s.io/client-go/informers/networking/v1 +k8s.io/client-go/informers/networking/v1beta1 +k8s.io/client-go/informers/node +k8s.io/client-go/informers/node/v1alpha1 +k8s.io/client-go/informers/node/v1beta1 +k8s.io/client-go/informers/policy +k8s.io/client-go/informers/policy/v1beta1 +k8s.io/client-go/informers/rbac +k8s.io/client-go/informers/rbac/v1 +k8s.io/client-go/informers/rbac/v1alpha1 +k8s.io/client-go/informers/rbac/v1beta1 +k8s.io/client-go/informers/scheduling +k8s.io/client-go/informers/scheduling/v1 +k8s.io/client-go/informers/scheduling/v1alpha1 +k8s.io/client-go/informers/scheduling/v1beta1 +k8s.io/client-go/informers/settings +k8s.io/client-go/informers/settings/v1alpha1 +k8s.io/client-go/informers/storage +k8s.io/client-go/informers/storage/v1 +k8s.io/client-go/informers/storage/v1alpha1 +k8s.io/client-go/informers/storage/v1beta1 +k8s.io/client-go/kubernetes +k8s.io/client-go/kubernetes/fake +k8s.io/client-go/kubernetes/scheme +k8s.io/client-go/kubernetes/typed/admissionregistration/v1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1/fake +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1 +k8s.io/client-go/kubernetes/typed/apps/v1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta1 +k8s.io/client-go/kubernetes/typed/apps/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta2 +k8s.io/client-go/kubernetes/typed/apps/v1beta2/fake +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1 +k8s.io/client-go/kubernetes/typed/authentication/v1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1beta1 +k8s.io/client-go/kubernetes/typed/authentication/v1beta1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1 +k8s.io/client-go/kubernetes/typed/authorization/v1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1beta1 +k8s.io/client-go/kubernetes/typed/authorization/v1beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v1 +k8s.io/client-go/kubernetes/typed/autoscaling/v1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/fake +k8s.io/client-go/kubernetes/typed/batch/v1 +k8s.io/client-go/kubernetes/typed/batch/v1/fake +k8s.io/client-go/kubernetes/typed/batch/v1beta1 +k8s.io/client-go/kubernetes/typed/batch/v1beta1/fake +k8s.io/client-go/kubernetes/typed/batch/v2alpha1 +k8s.io/client-go/kubernetes/typed/batch/v2alpha1/fake +k8s.io/client-go/kubernetes/typed/certificates/v1beta1 +k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1 +k8s.io/client-go/kubernetes/typed/coordination/v1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1beta1 +k8s.io/client-go/kubernetes/typed/coordination/v1beta1/fake +k8s.io/client-go/kubernetes/typed/core/v1 +k8s.io/client-go/kubernetes/typed/core/v1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1beta1 +k8s.io/client-go/kubernetes/typed/discovery/v1beta1/fake +k8s.io/client-go/kubernetes/typed/events/v1beta1 +k8s.io/client-go/kubernetes/typed/events/v1beta1/fake +k8s.io/client-go/kubernetes/typed/extensions/v1beta1 +k8s.io/client-go/kubernetes/typed/extensions/v1beta1/fake +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1 +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/networking/v1 +k8s.io/client-go/kubernetes/typed/networking/v1/fake +k8s.io/client-go/kubernetes/typed/networking/v1beta1 +k8s.io/client-go/kubernetes/typed/networking/v1beta1/fake +k8s.io/client-go/kubernetes/typed/node/v1alpha1 +k8s.io/client-go/kubernetes/typed/node/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/node/v1beta1 +k8s.io/client-go/kubernetes/typed/node/v1beta1/fake +k8s.io/client-go/kubernetes/typed/policy/v1beta1 +k8s.io/client-go/kubernetes/typed/policy/v1beta1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1 +k8s.io/client-go/kubernetes/typed/rbac/v1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1beta1 +k8s.io/client-go/kubernetes/typed/rbac/v1beta1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1 +k8s.io/client-go/kubernetes/typed/scheduling/v1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/fake +k8s.io/client-go/kubernetes/typed/settings/v1alpha1 +k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1 +k8s.io/client-go/kubernetes/typed/storage/v1/fake +k8s.io/client-go/kubernetes/typed/storage/v1alpha1 +k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1beta1 +k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake +k8s.io/client-go/listers/admissionregistration/v1 +k8s.io/client-go/listers/admissionregistration/v1beta1 +k8s.io/client-go/listers/apps/v1 +k8s.io/client-go/listers/apps/v1beta1 +k8s.io/client-go/listers/apps/v1beta2 +k8s.io/client-go/listers/auditregistration/v1alpha1 +k8s.io/client-go/listers/autoscaling/v1 +k8s.io/client-go/listers/autoscaling/v2beta1 +k8s.io/client-go/listers/autoscaling/v2beta2 +k8s.io/client-go/listers/batch/v1 +k8s.io/client-go/listers/batch/v1beta1 +k8s.io/client-go/listers/batch/v2alpha1 +k8s.io/client-go/listers/certificates/v1beta1 +k8s.io/client-go/listers/coordination/v1 +k8s.io/client-go/listers/coordination/v1beta1 +k8s.io/client-go/listers/core/v1 +k8s.io/client-go/listers/discovery/v1alpha1 +k8s.io/client-go/listers/discovery/v1beta1 +k8s.io/client-go/listers/events/v1beta1 +k8s.io/client-go/listers/extensions/v1beta1 +k8s.io/client-go/listers/flowcontrol/v1alpha1 +k8s.io/client-go/listers/networking/v1 +k8s.io/client-go/listers/networking/v1beta1 +k8s.io/client-go/listers/node/v1alpha1 +k8s.io/client-go/listers/node/v1beta1 +k8s.io/client-go/listers/policy/v1beta1 +k8s.io/client-go/listers/rbac/v1 +k8s.io/client-go/listers/rbac/v1alpha1 +k8s.io/client-go/listers/rbac/v1beta1 +k8s.io/client-go/listers/scheduling/v1 +k8s.io/client-go/listers/scheduling/v1alpha1 +k8s.io/client-go/listers/scheduling/v1beta1 +k8s.io/client-go/listers/settings/v1alpha1 +k8s.io/client-go/listers/storage/v1 +k8s.io/client-go/listers/storage/v1alpha1 +k8s.io/client-go/listers/storage/v1beta1 +k8s.io/client-go/pkg/apis/clientauthentication +k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 +k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 +k8s.io/client-go/pkg/version +k8s.io/client-go/plugin/pkg/client/auth/exec +k8s.io/client-go/plugin/pkg/client/auth/gcp +k8s.io/client-go/rest +k8s.io/client-go/rest/watch +k8s.io/client-go/testing +k8s.io/client-go/third_party/forked/golang/template +k8s.io/client-go/tools/auth +k8s.io/client-go/tools/cache +k8s.io/client-go/tools/clientcmd +k8s.io/client-go/tools/clientcmd/api +k8s.io/client-go/tools/clientcmd/api/latest +k8s.io/client-go/tools/clientcmd/api/v1 +k8s.io/client-go/tools/leaderelection +k8s.io/client-go/tools/leaderelection/resourcelock +k8s.io/client-go/tools/metrics +k8s.io/client-go/tools/pager +k8s.io/client-go/tools/record +k8s.io/client-go/tools/record/util +k8s.io/client-go/tools/reference +k8s.io/client-go/transport +k8s.io/client-go/util/cert +k8s.io/client-go/util/connrotation +k8s.io/client-go/util/flowcontrol +k8s.io/client-go/util/homedir +k8s.io/client-go/util/jsonpath +k8s.io/client-go/util/keyutil +k8s.io/client-go/util/workqueue +# k8s.io/cloud-provider v0.18.3 => k8s.io/cloud-provider v0.18.3 +k8s.io/cloud-provider +k8s.io/cloud-provider/volume +# k8s.io/component-base v0.18.3 => k8s.io/component-base v0.18.3 +## explicit +k8s.io/component-base/cli/flag +k8s.io/component-base/config +k8s.io/component-base/featuregate +k8s.io/component-base/metrics +k8s.io/component-base/metrics/legacyregistry +k8s.io/component-base/version +# k8s.io/csi-translation-lib v0.18.3 => k8s.io/csi-translation-lib v0.18.3 +k8s.io/csi-translation-lib +k8s.io/csi-translation-lib/plugins +# k8s.io/klog v1.0.0 => k8s.io/klog v1.0.0 +## explicit +k8s.io/klog +# k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 +k8s.io/kube-openapi/pkg/util/proto +# k8s.io/kube-scheduler v0.0.0 => k8s.io/kube-scheduler v0.18.3 +## explicit +k8s.io/kube-scheduler/extender/v1 +# k8s.io/kubernetes v1.18.3 +## explicit +k8s.io/kubernetes/pkg/api/legacyscheme +k8s.io/kubernetes/pkg/api/service +k8s.io/kubernetes/pkg/api/v1/pod +k8s.io/kubernetes/pkg/apis/apps +k8s.io/kubernetes/pkg/apis/autoscaling +k8s.io/kubernetes/pkg/apis/core +k8s.io/kubernetes/pkg/apis/core/helper +k8s.io/kubernetes/pkg/apis/core/pods +k8s.io/kubernetes/pkg/apis/core/v1 +k8s.io/kubernetes/pkg/apis/core/v1/helper +k8s.io/kubernetes/pkg/apis/core/validation +k8s.io/kubernetes/pkg/apis/scheduling +k8s.io/kubernetes/pkg/capabilities +k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util +k8s.io/kubernetes/pkg/controller/volume/scheduling +k8s.io/kubernetes/pkg/controller/volume/scheduling/metrics +k8s.io/kubernetes/pkg/features +k8s.io/kubernetes/pkg/fieldpath +k8s.io/kubernetes/pkg/kubelet/types +k8s.io/kubernetes/pkg/master/ports +k8s.io/kubernetes/pkg/scheduler/apis/config +k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper +k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity +k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity +k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports +k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources +k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable +k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration +k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1 +k8s.io/kubernetes/pkg/scheduler/listers +k8s.io/kubernetes/pkg/scheduler/metrics +k8s.io/kubernetes/pkg/scheduler/nodeinfo +k8s.io/kubernetes/pkg/scheduler/util +k8s.io/kubernetes/pkg/security/apparmor +k8s.io/kubernetes/pkg/util/parsers +k8s.io/kubernetes/pkg/util/resizefs +k8s.io/kubernetes/pkg/volume +k8s.io/kubernetes/pkg/volume/util +k8s.io/kubernetes/pkg/volume/util/fs +k8s.io/kubernetes/pkg/volume/util/fsquota +k8s.io/kubernetes/pkg/volume/util/fsquota/common +k8s.io/kubernetes/pkg/volume/util/hostutil +k8s.io/kubernetes/pkg/volume/util/recyclerclient +k8s.io/kubernetes/pkg/volume/util/subpath +k8s.io/kubernetes/pkg/volume/util/types +k8s.io/kubernetes/pkg/volume/util/volumepathhandler +# k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 +k8s.io/utils/buffer +k8s.io/utils/exec +k8s.io/utils/integer +k8s.io/utils/io +k8s.io/utils/keymutex +k8s.io/utils/mount +k8s.io/utils/net +k8s.io/utils/nsenter +k8s.io/utils/path +k8s.io/utils/pointer +k8s.io/utils/strings +k8s.io/utils/trace +# sigs.k8s.io/structured-merge-diff/v3 v3.0.0 +sigs.k8s.io/structured-merge-diff/v3/value +# sigs.k8s.io/yaml v1.2.0 +## explicit +sigs.k8s.io/yaml +# k8s.io/api => k8s.io/api v0.18.3 +# k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.3 +# k8s.io/apimachinery => k8s.io/apimachinery v0.18.3 +# k8s.io/apiserver => k8s.io/apiserver v0.18.3 +# k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.3 +# k8s.io/client-go => k8s.io/client-go v0.18.3 +# k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.3 +# k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.3 +# k8s.io/code-generator => k8s.io/code-generator v0.18.3 +# k8s.io/component-base => k8s.io/component-base v0.18.3 +# k8s.io/cri-api => k8s.io/cri-api v0.18.3 +# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.3 +# k8s.io/klog => k8s.io/klog v1.0.0 +# k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.3 +# k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.3 +# k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.3 +# k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.3 +# k8s.io/kubectl => k8s.io/kubectl v0.18.3 +# k8s.io/kubelet => k8s.io/kubelet v0.18.3 +# k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.3 +# k8s.io/metrics => k8s.io/metrics v0.18.3 +# k8s.io/node-api => k8s.io/node-api v0.18.3 +# k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.3 +# k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.18.3 +# k8s.io/sample-controller => k8s.io/sample-controller v0.18.3 From c8052c4b63f6006298b7fdf69bec6043dc9e1478 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Wed, 1 Jul 2020 19:34:25 +0800 Subject: [PATCH 5/5] Fix build --- pkg/scheduler/api/node_info.go | 3 +- pkg/scheduler/api/node_info_test.go | 3 ++ pkg/scheduler/api/pod_info.go | 2 +- .../plugins/predicates/predicates.go | 32 +++---------------- 4 files changed, 10 insertions(+), 30 deletions(-) diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go index 7737a66ae0..86aaca910f 100644 --- a/pkg/scheduler/api/node_info.go +++ b/pkg/scheduler/api/node_info.go @@ -161,7 +161,8 @@ func (ni *NodeInfo) setNodeGPUInfo(node *v1.Node) { } gpuNumber := res.Value() if gpuNumber == 0 { - klog.Warningf("invalid %s=%s", VolcanoGPUNumber, res) + klog.Warningf("invalid %s=%s", VolcanoGPUNumber, res.String()) + return } memoryPerCard := uint(totalMemory / gpuNumber) diff --git a/pkg/scheduler/api/node_info_test.go b/pkg/scheduler/api/node_info_test.go index 4b3d5eaf12..590fc8b1e9 100644 --- a/pkg/scheduler/api/node_info_test.go +++ b/pkg/scheduler/api/node_info_test.go @@ -62,6 +62,7 @@ func TestNodeInfo_AddPod(t *testing.T) { "c1/p1": NewTaskInfo(case01Pod1), "c1/p2": NewTaskInfo(case01Pod2), }, + GPUDevices: make(map[int]*GPUDevice), }, }, { @@ -79,6 +80,7 @@ func TestNodeInfo_AddPod(t *testing.T) { Capability: buildResource("2000m", "1G"), State: NodeState{Phase: Ready}, Tasks: map[TaskID]*TaskInfo{}, + GPUDevices: make(map[int]*GPUDevice), }, expectedFailure: true, }, @@ -138,6 +140,7 @@ func TestNodeInfo_RemovePod(t *testing.T) { "c1/p1": NewTaskInfo(case01Pod1), "c1/p3": NewTaskInfo(case01Pod3), }, + GPUDevices: make(map[int]*GPUDevice), }, }, } diff --git a/pkg/scheduler/api/pod_info.go b/pkg/scheduler/api/pod_info.go index 56405ecc77..f6bf36d647 100644 --- a/pkg/scheduler/api/pod_info.go +++ b/pkg/scheduler/api/pod_info.go @@ -85,7 +85,7 @@ func GetGPUIndex(pod *v1.Pod) int { if found { id, err := strconv.Atoi(value) if err != nil { - klog.Error("invalid %s=%s", GPUIndex, value) + klog.Errorf("invalid %s=%s", GPUIndex, value) return -1 } return id diff --git a/pkg/scheduler/plugins/predicates/predicates.go b/pkg/scheduler/plugins/predicates/predicates.go index ac8c524432..7ceed61118 100644 --- a/pkg/scheduler/plugins/predicates/predicates.go +++ b/pkg/scheduler/plugins/predicates/predicates.go @@ -40,12 +40,6 @@ const ( // PluginName indicates name of volcano scheduler plugin. PluginName = "predicates" - // MemoryPressurePredicate is the key for enabling Memory Pressure Predicate in YAML - MemoryPressurePredicate = "predicate.MemoryPressureEnable" - // DiskPressurePredicate is the key for enabling Disk Pressure Predicate in YAML - DiskPressurePredicate = "predicate.DiskPressureEnable" - // PIDPressurePredicate is the key for enabling PID Pressure Predicate in YAML - PIDPressurePredicate = "predicate.PIDPressureEnable" // GPUSharingPredicate is the key for enabling GPU Sharing Predicate in YAML GPUSharingPredicate = "predicate.GPUSharingEnable" ) @@ -65,17 +59,14 @@ func (pp *predicatesPlugin) Name() string { } type predicateEnable struct { - memoryPressureEnable bool - diskPressureEnable bool - pidPressureEnable bool - gpuSharingEnable bool + gpuSharingEnable bool } func enablePredicate(args framework.Arguments) predicateEnable { /* - User Should give predicatesEnable in this format(predicate.MemoryPressureEnable, predicate.DiskPressureEnable, predicate.PIDPressureEnable, predicate.GPUSharingEnable. - Currently supported only for MemoryPressure, DiskPressure, PIDPressure predicate checks. + User Should give predicatesEnable in this format(predicate.GPUSharingEnable). + Currently supported only GPUSharing predicate checks. actions: "reclaim, allocate, backfill, preempt" tiers: @@ -87,30 +78,15 @@ func enablePredicate(args framework.Arguments) predicateEnable { - name: drf - name: predicates arguments: - predicate.MemoryPressureEnable: true - predicate.DiskPressureEnable: true - predicate.PIDPressureEnable: true predicate.GPUSharingEnable: true - name: proportion - name: nodeorder */ predicate := predicateEnable{ - memoryPressureEnable: false, - diskPressureEnable: false, - pidPressureEnable: false, - gpuSharingEnable: true, // enable for debug + gpuSharingEnable: false, } - // Checks whether predicate.MemoryPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. - args.GetBool(&predicate.memoryPressureEnable, MemoryPressurePredicate) - - // Checks whether predicate.DiskPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. - args.GetBool(&predicate.diskPressureEnable, DiskPressurePredicate) - - // Checks whether predicate.PIDPressureEnable is provided or not, if given, modifies the value in predicateEnable struct. - args.GetBool(&predicate.pidPressureEnable, PIDPressurePredicate) - // Checks whether predicate.GPUSharingEnable is provided or not, if given, modifies the value in predicateEnable struct. args.GetBool(&predicate.gpuSharingEnable, GPUSharingPredicate)