Multiple RP with scheduler backporting from 1.18.5

api/api-rules/violation_exceptions.list

@@ -343,6 +343,9 @@ API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,EnvFrom
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,VolumeMounts
 API rule violation: list_type_missing,k8s.io/api/settings/v1alpha1,PodPresetSpec,Volumes
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeDriver,TopologyKeys
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeList,Items
+API rule violation: list_type_missing,k8s.io/api/storage/v1,CSINodeSpec,Drivers
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,AllowedTopologies
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClass,MountOptions
 API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClassList,Items
@@ -481,9 +484,17 @@ API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1al
 API rule violation: list_type_missing,k8s.io/kube-controller-manager/config/v1alpha1,NodeLifecycleControllerConfiguration,TenantPartitionKubeConfigs
 API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyConfiguration,NodePortAddresses
 API rule violation: list_type_missing,k8s.io/kube-proxy/config/v1alpha1,KubeProxyIPVSConfiguration,ExcludeCIDRs
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,KubeSchedulerConfiguration,PluginConfig
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,PluginSet,Disabled
-API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1alpha1,PluginSet,Enabled
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Extender,ManagedResources
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,CAData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,CertData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ExtenderTLSConfig,KeyData
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,LabelsPresence,Labels
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Extenders
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Predicates
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,Policy,Priorities
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Resources
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,RequestedToCapacityRatioArguments,Shape
+API rule violation: list_type_missing,k8s.io/kube-scheduler/config/v1,ServiceAffinity,Labels
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ClusterDNS
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,EnforceNodeAllocatable
 API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,TLSCipherSuites
@@ -681,6 +692,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,V
 API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,FlexVolumePluginDir
 API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,PersistentVolumeRecyclerConfiguration
 API rule violation: names_match,k8s.io/kube-proxy/config/v1alpha1,KubeProxyConfiguration,IPTables
+API rule violation: names_match,k8s.io/kube-scheduler/config/v1,Extender,EnableHTTPS
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,IPTablesDropBit
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,IPTablesMasqueradeBit
 API rule violation: names_match,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ResolverConfig

build/kazel_generated.bzl

@@ -97,7 +97,9 @@ tags_values_pkgs = {"openapi-gen": {
  "staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1",
  "staging/src/k8s.io/kube-controller-manager/config/v1alpha1",
  "staging/src/k8s.io/kube-proxy/config/v1alpha1",
+ "staging/src/k8s.io/kube-scheduler/config/v1",
  "staging/src/k8s.io/kube-scheduler/config/v1alpha1",
+ "staging/src/k8s.io/kube-scheduler/config/v1alpha2",
  "staging/src/k8s.io/kubelet/config/v1beta1",
  "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1",
  "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2",
@@ -181,7 +183,9 @@ tags_pkgs_values = {"openapi-gen": {
  "staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1": ["true"],
  "staging/src/k8s.io/kube-controller-manager/config/v1alpha1": ["true"],
  "staging/src/k8s.io/kube-proxy/config/v1alpha1": ["true"],
+ "staging/src/k8s.io/kube-scheduler/config/v1": ["true"],
  "staging/src/k8s.io/kube-scheduler/config/v1alpha1": ["true"],
+ "staging/src/k8s.io/kube-scheduler/config/v1alpha2": ["true"],
  "staging/src/k8s.io/kubelet/config/v1beta1": ["true"],
  "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta1": ["true"],
  "staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1beta2": ["true"],

cmd/cloud-controller-manager/app/core.go

@@ -36,12 +36,17 @@ import (
 
 func startCloudNodeController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
  // Start the CloudNodeController
- nodeController := cloudcontrollers.NewCloudNodeController(
+ nodeController, err := cloudcontrollers.NewCloudNodeController(
  ctx.SharedInformers.Core().V1().Nodes(),
  // cloud node controller uses existing cluster role from node-controller
  ctx.ClientBuilder.ClientOrDie("node-controller"),
  cloud,
- ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration)
+ ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration,
+ )
+ if err != nil {
+ klog.Warningf("failed to start cloud node controller: %s", err)
+ return nil, false, nil
+ }
 
  go nodeController.Run(stopCh)
 

cmd/controller-manager/app/serve.go

@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// File modified by cherrypick from kubernetes on 05/06/2021
 package app
 
 import (
- "github.com/prometheus/client_golang/prometheus"
  "net/http"
  goruntime "runtime"
 
+ "github.com/prometheus/client_golang/prometheus"
+
  genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
  apirequest "k8s.io/apiserver/pkg/endpoints/request"
  apiserver "k8s.io/apiserver/pkg/server"
@@ -46,6 +48,7 @@ func BuildHandlerChain(apiHandler http.Handler, authorizationInfo *apiserver.Aut
  handler = genericapifilters.WithAuthentication(handler, authenticationInfo.Authenticator, failedHandler, nil)
  }
  handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver)
+ handler = genericapifilters.WithCacheControl(handler)
  handler = genericfilters.WithPanicRecovery(handler)
 
  return handler

cmd/kube-controller-manager/app/BUILD

@@ -147,6 +147,7 @@ go_library(
  "//staging/src/k8s.io/cloud-provider:go_default_library",
  "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
  "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
+ "//staging/src/k8s.io/csi-translation-lib:go_default_library",
  "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library",
  "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
  "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library",

cmd/kube-controller-manager/app/core.go

@@ -23,6 +23,7 @@ package app
 
 import (
  "fmt"
+ csitrans "k8s.io/csi-translation-lib"
  "net"
  "net/http"
  "strings"
@@ -270,7 +271,8 @@ func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, err
  ctx.InformerFactory.Core().V1().PersistentVolumes(),
  ctx.InformerFactory.Storage().V1().StorageClasses(),
  ctx.Cloud,
- ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration))
+ ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
+ csitrans.New())
 
  if expandControllerErr != nil {
  return nil, true, fmt.Errorf("failed to start volume expand controller : %v", expandControllerErr)

cmd/kube-controller-manager/app/policy.go

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 The Kubernetes Authors.
+Copyright 2020 Authors of Arktos - file modified.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,14 +22,16 @@ limitations under the License.
 package app
 
 import (
+ "net/http"
+
+ "k8s.io/klog"
+
  "k8s.io/apimachinery/pkg/runtime/schema"
+ utilfeature "k8s.io/apiserver/pkg/util/feature"
  "k8s.io/client-go/dynamic"
  "k8s.io/client-go/scale"
  "k8s.io/kubernetes/pkg/controller/disruption"
-
- "net/http"
-
- "k8s.io/klog"
+ kubefeatures "k8s.io/kubernetes/pkg/features"
 )
 
 func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error) {
@@ -42,6 +45,10 @@ func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error
  resource, group+"/"+version)
  return nil, false, nil
  }
+ if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodDisruptionBudget) {
+ klog.Infof("Refusing to start disruption because the PodDisruptionBudget feature is disabled")
+ return nil, false, nil
+ }
 
  client := ctx.ClientBuilder.ClientOrDie("disruption-controller")
  config := ctx.ClientBuilder.ConfigOrDie("disruption-controller")

cmd/kube-scheduler/BUILD

@@ -5,7 +5,7 @@ load(
  "go_binary",
  "go_library",
 )
-load("//pkg/version:def.bzl", "version_x_defs")
+load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
 
 go_binary(
  name = "kube-scheduler",
@@ -20,10 +20,9 @@ go_library(
  importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
  deps = [
  "//cmd/kube-scheduler/app:go_default_library",
- "//pkg/util/prometheusclientgo:go_default_library",
- "//pkg/version/prometheus:go_default_library",
  "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
  "//staging/src/k8s.io/component-base/logs:go_default_library",
+ "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library",
  "//vendor/github.com/spf13/pflag:go_default_library",
  ],
 )

cmd/kube-scheduler/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- sig-scheduling-maintainers
+reviewers:
+- sig-scheduling
+labels:
+- sig/scheduling

cmd/kube-scheduler/app/BUILD

@@ -14,17 +14,16 @@ go_library(
  "//cmd/kube-scheduler/app/options:go_default_library",
  "//pkg/api/legacyscheme:go_default_library",
  "//pkg/scheduler:go_default_library",
- "//pkg/scheduler/algorithmprovider:go_default_library",
  "//pkg/scheduler/apis/config:go_default_library",
  "//pkg/scheduler/framework/v1alpha1:go_default_library",
  "//pkg/scheduler/metrics:go_default_library",
+ "//pkg/scheduler/profile:go_default_library",
  "//pkg/util/configz:go_default_library",
  "//pkg/util/flag:go_default_library",
- "//pkg/version:go_default_library",
  "//pkg/version/verflag:go_default_library",
- "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+ "//staging/src/k8s.io/api/core/v1:go_default_library",
+ "//staging/src/k8s.io/api/events/v1beta1:go_default_library",
  "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
- "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
  "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
  "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
  "//staging/src/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
@@ -34,12 +33,16 @@ go_library(
  "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library",
  "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library",
  "//staging/src/k8s.io/apiserver/pkg/util/term:go_default_library",
- "//staging/src/k8s.io/client-go/datapartition:go_default_library",
+ "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
  "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+ "//staging/src/k8s.io/client-go/tools/events:go_default_library",
  "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
+ "//staging/src/k8s.io/client-go/tools/record:go_default_library",
  "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
  "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
- "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+ "//staging/src/k8s.io/component-base/logs:go_default_library",
+ "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
+ "//staging/src/k8s.io/component-base/version:go_default_library",
  "//vendor/github.com/spf13/cobra:go_default_library",
  "//vendor/k8s.io/klog:go_default_library",
  ],

cmd/kube-scheduler/app/config/BUILD

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
  name = "go_default_library",
@@ -12,7 +12,9 @@ go_library(
  "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
  "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
  "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+ "//staging/src/k8s.io/client-go/kubernetes/typed/events/v1beta1:go_default_library",
  "//staging/src/k8s.io/client-go/rest:go_default_library",
+ "//staging/src/k8s.io/client-go/tools/events:go_default_library",
  "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
  "//staging/src/k8s.io/client-go/tools/record:go_default_library",
  ],
@@ -31,3 +33,13 @@ filegroup(
  tags = ["automanaged"],
  visibility = ["//visibility:public"],
 )
+
+go_test(
+ name = "go_default_test",
+ srcs = ["config_test.go"],
+ embed = [":go_default_library"],
+ deps = [
+ "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+ "//vendor/github.com/google/go-cmp/cmp:go_default_library",
+ ],
+)

cmd/kube-scheduler/app/config/config.go

@@ -15,6 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// File modified by backporting scheduler 1.18.5 from kubernetes on 05/04/2021
 package config
 
 import (
@@ -23,15 +24,17 @@ import (
  coreinformers "k8s.io/client-go/informers/core/v1"
  clientset "k8s.io/client-go/kubernetes"
  v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+ "k8s.io/client-go/kubernetes/typed/events/v1beta1"
  restclient "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/events"
  "k8s.io/client-go/tools/leaderelection"
  "k8s.io/client-go/tools/record"
  kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 )
 
 // Config has all the context to run a Scheduler
 type Config struct {
- // config is the scheduler server's configuration object.
+ // ComponentConfig is the scheduler server's configuration object.
  ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration
 
  // LoopbackClientConfig is a config for a privileged loopback connection
@@ -43,16 +46,16 @@ type Config struct {
  Authorization apiserver.AuthorizationInfo
  SecureServing *apiserver.SecureServingInfo
 
- // explictly define node informer from the resource provider client
- ResourceProviderClient clientset.Interface
- ResourceInformer coreinformers.NodeInformer
-
  Client clientset.Interface
  InformerFactory informers.SharedInformerFactory
  PodInformer coreinformers.PodInformer
- EventClient v1core.EventsGetter
- Recorder record.EventRecorder
- Broadcaster record.EventBroadcaster
+
+ // TODO: Remove the following after fully migrating to the new events api.
+ CoreEventClient v1core.EventsGetter
+ CoreBroadcaster record.EventBroadcaster
+
+ EventClient v1beta1.EventsGetter
+ Broadcaster events.EventBroadcaster
 
  // LeaderElection is optional.
  LeaderElection *leaderelection.LeaderElectionConfig

cmd/kube-scheduler/app/config/config_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Copyright 2020 Authors of Arktos - file modified.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// File modified by backporting scheduler 1.18.5 from kubernetes on 05/04/2021
+package config
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ apiserver "k8s.io/apiserver/pkg/server"
+)
+
+func TestConfigComplete(t *testing.T) {
+ scenarios := []struct {
+ name string
+ want *Config
+ config *Config
+ }{
+ {
+ name: "SetInsecureServingName",
+ want: &Config{
+ InsecureServing: &apiserver.DeprecatedInsecureServingInfo{
+ Name: "healthz",
+ },
+ },
+ config: &Config{
+ InsecureServing: &apiserver.DeprecatedInsecureServingInfo{},
+ },
+ },
+ {
+ name: "SetMetricsInsecureServingName",
+ want: &Config{
+ InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{
+ Name: "metrics",
+ },
+ },
+ config: &Config{
+ InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{},
+ },
+ },
+ }
+
+ for _, scenario := range scenarios {
+ t.Run(scenario.name, func(t *testing.T) {
+ cc := scenario.config.Complete()
+
+ returnValue := cc.completedConfig.Config
+
+ if diff := cmp.Diff(scenario.want, returnValue); diff != "" {
+ t.Errorf("Complete(): Unexpected return value (-want, +got): %s", diff)
+ }
+ })
+ }
+}