diff --git a/WORKSPACE b/WORKSPACE index 5853ef5da78..74cb167c27d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -537,7 +537,7 @@ maven_jar( # end Pulsar Client # Kubernetes java client -kubernetes_client_version = "1.0.0-beta1" +kubernetes_client_version = "3.0.0" squareup_okhttp_version = "2.7.5" maven_jar( diff --git a/deploy/kubernetes/general/apiserver.yaml b/deploy/kubernetes/general/apiserver.yaml index b11f7acf63d..470dff330d2 100644 --- a/deploy/kubernetes/general/apiserver.yaml +++ b/deploy/kubernetes/general/apiserver.yaml @@ -45,7 +45,7 @@ subjects: --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: heron-apiserver diff --git a/deploy/kubernetes/general/bookkeeper.statefulset.yaml b/deploy/kubernetes/general/bookkeeper.statefulset.yaml index 89aaf2d9130..e5c9fe2c19a 100644 --- a/deploy/kubernetes/general/bookkeeper.statefulset.yaml +++ b/deploy/kubernetes/general/bookkeeper.statefulset.yaml @@ -36,7 +36,7 @@ data: BK_useHostNameAsBookieID: "true" --- -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: bookie @@ -44,6 +44,10 @@ metadata: app: bookkeeper component: bookie spec: + selector: + matchLabels: + app: bookkeeper + component: bookie serviceName: "bookkeeper" replicas: 3 template: @@ -145,11 +149,15 @@ spec: ## Auto-Recovery makes sure to restore the replication factor when any bookie ## crashes and it's not recovering on its own. ## -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookie-autorecovery spec: + selector: + matchLabels: + app: bookkeeper + component: bookkeeper-replication replicas: 2 template: metadata: diff --git a/deploy/kubernetes/general/bookkeeper.statefulset_empty.yaml b/deploy/kubernetes/general/bookkeeper.statefulset_empty.yaml index 219e3c276a3..90a158f71ae 100644 --- a/deploy/kubernetes/general/bookkeeper.statefulset_empty.yaml +++ b/deploy/kubernetes/general/bookkeeper.statefulset_empty.yaml @@ -36,7 +36,7 @@ data: BK_useHostNameAsBookieID: "true" --- -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: bookie @@ -44,6 +44,10 @@ metadata: app: bookkeeper component: bookie spec: + selector: + matchLabels: + app: bookkeeper + component: bookie serviceName: "bookkeeper" replicas: 3 template: @@ -129,11 +133,15 @@ spec: ## Auto-Recovery makes sure to restore the replication factor when any bookie ## crashes and it's not recovering on its own. ## -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookie-autorecovery spec: + selector: + matchLabels: + app: bookkeeper + component: bookkeeper-replication replicas: 2 template: metadata: diff --git a/deploy/kubernetes/general/bookkeeper.yaml b/deploy/kubernetes/general/bookkeeper.yaml index 4fd985bc20f..2ba48912276 100644 --- a/deploy/kubernetes/general/bookkeeper.yaml +++ b/deploy/kubernetes/general/bookkeeper.yaml @@ -38,7 +38,7 @@ data: ## cannot be moved across different nodes. ## For this reason, we run BK as a daemon set, one for each node in the ## cluster, unless restricted by label selectors -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: bookie @@ -46,6 +46,11 @@ metadata: app: bookkeeper component: bookie spec: + selector: + matchLabels: + app: bookkeeper + component: bookie + cluster: bookkeeper template: metadata: labels: @@ -130,11 +135,15 @@ spec: ## Auto-Recovery makes sure to restore the replication factor when any bookie ## crashes and it's not recovering on its own. ## -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: bookie-autorecovery spec: + selector: + matchLabels: + app: bookkeeper + component: bookkeeper-replication replicas: 2 template: metadata: diff --git a/deploy/kubernetes/general/tools.yaml b/deploy/kubernetes/general/tools.yaml index d4ab5ce52ba..096841b1a81 100644 --- a/deploy/kubernetes/general/tools.yaml +++ b/deploy/kubernetes/general/tools.yaml @@ -18,12 +18,15 @@ ## ## Deployment Pod for tracker and ui ## -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: heron-tracker namespace: default spec: + selector: + matchLabels: + app: heron-tracker template: metadata: labels: @@ -57,6 +60,13 @@ spec: --name=localzk --hostport=zookeeper:2181 --rootpath="/heron" + resources: + requests: + cpu: "100m" + memory: "200M" + limits: + cpu: "200m" + memory: "300M" - name: heron-ui image: heron/heron:latest ports: @@ -68,6 +78,13 @@ spec: heron-ui --port=8889 --base_url=/api/v1/namespaces/default/services/heron-ui:8889/proxy + resources: + requests: + cpu: "100m" + memory: "200M" + limits: + cpu: "200m" + memory: "300M" --- ## diff --git a/deploy/kubernetes/general/zookeeper.yaml b/deploy/kubernetes/general/zookeeper.yaml index c644621ace9..835bee57a31 100644 --- a/deploy/kubernetes/general/zookeeper.yaml +++ b/deploy/kubernetes/general/zookeeper.yaml @@ -33,7 +33,7 @@ spec: --- ## Define a StatefulSet for ZK servers -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: zk @@ -43,6 +43,10 @@ metadata: spec: serviceName: zookeeper replicas: 1 + selector: + matchLabels: + app: heron + component: zookeeper template: metadata: labels: @@ -66,7 +70,7 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: zookeeper - image: heron/heron:0.16.2 + image: heron/heron:latest command: ["sh", "-c"] args: - > diff --git a/deploy/kubernetes/gke/gcs-apiserver.yaml b/deploy/kubernetes/gke/gcs-apiserver.yaml index ee0b3808333..d268a0f7f3a 100644 --- a/deploy/kubernetes/gke/gcs-apiserver.yaml +++ b/deploy/kubernetes/gke/gcs-apiserver.yaml @@ -45,7 +45,7 @@ subjects: --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: heron-apiserver diff --git a/deploy/kubernetes/helm/templates/bookie.yaml b/deploy/kubernetes/helm/templates/bookie.yaml index 7a76c56ee0f..ed01baf5e72 100644 --- a/deploy/kubernetes/helm/templates/bookie.yaml +++ b/deploy/kubernetes/helm/templates/bookie.yaml @@ -57,10 +57,10 @@ data: ## For this reason, we run BK as a daemon set, one for each node in the ## cluster, unless restricted by label selectors {{- if or (eq .Values.platform "gke") (eq .Values.platform "minikube") }} -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet {{- else }} -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet {{- end }} @@ -70,6 +70,11 @@ metadata: app: {{ .Release.Name }}-bookkeeper component: {{ .Release.Name }}-bookie spec: + selector: + matchLabels: + app: {{ .Release.Name }}-bookkeeper + component: {{ .Release.Name }}-bookie + cluster: {{ .Release.Name }}-bookkeeper {{- if or (eq .Values.platform "gke") (eq .Values.platform "minikube") }} serviceName: {{ .Release.Name }}-bookie replicas: {{ $bookieReplicas }} diff --git a/deploy/kubernetes/helm/templates/tools.yaml b/deploy/kubernetes/helm/templates/tools.yaml index 72f92dd7d40..779f0623930 100644 --- a/deploy/kubernetes/helm/templates/tools.yaml +++ b/deploy/kubernetes/helm/templates/tools.yaml @@ -28,7 +28,7 @@ data: HERON_APISERVER_MEM_MAX: {{ $apiServerMemory | quote }} --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Release.Name }}-tools @@ -38,6 +38,10 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: + selector: + matchLabels: + app: {{ .Release.Name }}-tools + release: {{ .Release.Name }} template: metadata: labels: diff --git a/deploy/kubernetes/minikube/apiserver.yaml b/deploy/kubernetes/minikube/apiserver.yaml index 2a89c1a6623..3c4a9b29950 100644 --- a/deploy/kubernetes/minikube/apiserver.yaml +++ b/deploy/kubernetes/minikube/apiserver.yaml @@ -46,7 +46,7 @@ subjects: --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: heron-apiserver diff --git a/deploy/kubernetes/minikube/bookkeeper.yaml b/deploy/kubernetes/minikube/bookkeeper.yaml index d5ec6687e7e..f5778c3c303 100644 --- a/deploy/kubernetes/minikube/bookkeeper.yaml +++ b/deploy/kubernetes/minikube/bookkeeper.yaml @@ -36,7 +36,7 @@ data: ## cannot be moved across different nodes. ## For this reason, we run BK as a daemon set, one for each node in the ## cluster, unless restricted by label selectors -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: bookie @@ -44,6 +44,11 @@ metadata: app: bookkeeper component: bookie spec: + selector: + matchLabels: + app: bookkeeper + component: bookie + cluster: bookkeeper template: metadata: labels: diff --git a/deploy/kubernetes/minikube/tools.yaml b/deploy/kubernetes/minikube/tools.yaml index 535d9b6112d..75643d8e76f 100644 --- a/deploy/kubernetes/minikube/tools.yaml +++ b/deploy/kubernetes/minikube/tools.yaml @@ -18,12 +18,15 @@ ## ## Deployment Pod for tracker and ui ## -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: name: heron-tracker namespace: default spec: + selector: + matchLabels: + app: heron-tracker template: metadata: labels: @@ -43,6 +46,13 @@ spec: --name=kubernetes --hostport=zookeeper:2181 --rootpath="/heron" + resources: + requests: + cpu: "100m" + memory: "200M" + limits: + cpu: "200m" + memory: "300M" - name: heron-ui image: heron/heron:latest ports: @@ -54,6 +64,13 @@ spec: heron-ui --port=8889 --base_url=/api/v1/namespaces/default/services/heron-ui:8889/proxy + resources: + requests: + cpu: "100m" + memory: "200M" + limits: + cpu: "200m" + memory: "300M" --- ## diff --git a/deploy/kubernetes/minikube/zookeeper.yaml b/deploy/kubernetes/minikube/zookeeper.yaml index f4e5f047ee2..779db9d0fd1 100644 --- a/deploy/kubernetes/minikube/zookeeper.yaml +++ b/deploy/kubernetes/minikube/zookeeper.yaml @@ -20,7 +20,7 @@ ## ## Define a StatefulSet for ZK servers -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: zk @@ -30,6 +30,10 @@ metadata: spec: serviceName: zookeeper replicas: 1 + selector: + matchLabels: + app: heron + component: zookeeper template: metadata: labels: @@ -37,6 +41,20 @@ spec: component: zookeeper spec: + # Make sure multiple pods of ZK don't get scheduled on the + # same node, unless there are no other available nodes + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - zookeeper + topologyKey: "kubernetes.io/hostname" containers: - name: zookeeper image: heron/heron:latest diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1Controller.java similarity index 91% rename from heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java rename to heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1Controller.java index 2aea58d1f01..58ac941561e 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1beta1Controller.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/AppsV1Controller.java @@ -49,7 +49,8 @@ import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; -import io.kubernetes.client.apis.AppsV1beta1Api; +import io.kubernetes.client.apis.AppsV1Api; +import io.kubernetes.client.custom.Quantity; import io.kubernetes.client.models.V1Container; import io.kubernetes.client.models.V1ContainerPort; import io.kubernetes.client.models.V1DeleteOptions; @@ -64,22 +65,22 @@ import io.kubernetes.client.models.V1Toleration; import io.kubernetes.client.models.V1Volume; import io.kubernetes.client.models.V1VolumeMount; -import io.kubernetes.client.models.V1beta1StatefulSet; -import io.kubernetes.client.models.V1beta1StatefulSetSpec; +import io.kubernetes.client.models.V1StatefulSet; +import io.kubernetes.client.models.V1StatefulSetSpec; -public class AppsV1beta1Controller extends KubernetesController { +public class AppsV1Controller extends KubernetesController { private static final Logger LOG = - Logger.getLogger(AppsV1beta1Controller.class.getName()); + Logger.getLogger(AppsV1Controller.class.getName()); private static final String ENV_SHARD_ID = "SHARD_ID"; - private final AppsV1beta1Api client; + private final AppsV1Api client; - AppsV1beta1Controller(Config configuration, Config runtimeConfiguration) { + AppsV1Controller(Config configuration, Config runtimeConfiguration) { super(configuration, runtimeConfiguration); final ApiClient apiClient = new ApiClient().setBasePath(getKubernetesUri()); - client = new AppsV1beta1Api(apiClient); + client = new AppsV1Api(apiClient); } @Override @@ -97,7 +98,7 @@ boolean submit(PackingPlan packingPlan) { for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) { numberOfInstances = Math.max(numberOfInstances, containerPlan.getInstances().size()); } - final V1beta1StatefulSet statefulSet = createStatefulSet(containerResource, numberOfInstances); + final V1StatefulSet statefulSet = createStatefulSet(containerResource, numberOfInstances); try { final Response response = @@ -138,7 +139,7 @@ boolean restart(int shardId) { @Override public Set addContainers(Set containersToAdd) { - final V1beta1StatefulSet statefulSet; + final V1StatefulSet statefulSet; try { statefulSet = getStatefulSet(); } catch (ApiException ae) { @@ -148,7 +149,7 @@ boolean restart(int shardId) { final int currentContainerCount = statefulSet.getSpec().getReplicas(); final int newContainerCount = currentContainerCount + containersToAdd.size(); - final V1beta1StatefulSetSpec newSpec = new V1beta1StatefulSetSpec(); + final V1StatefulSetSpec newSpec = new V1StatefulSetSpec(); newSpec.setReplicas(newContainerCount); try { @@ -163,7 +164,7 @@ boolean restart(int shardId) { @Override public void removeContainers(Set containersToRemove) { - final V1beta1StatefulSet statefulSet; + final V1StatefulSet statefulSet; try { statefulSet = getStatefulSet(); } catch (ApiException ae) { @@ -173,7 +174,7 @@ public void removeContainers(Set containersToRemove) final int currentContainerCount = statefulSet.getSpec().getReplicas(); final int newContainerCount = currentContainerCount - containersToRemove.size(); - final V1beta1StatefulSetSpec newSpec = new V1beta1StatefulSetSpec(); + final V1StatefulSetSpec newSpec = new V1StatefulSetSpec(); newSpec.setReplicas(newContainerCount); try { @@ -184,7 +185,7 @@ public void removeContainers(Set containersToRemove) } } - private void doPatch(V1beta1StatefulSetSpec patchedSpec) throws ApiException { + private void doPatch(V1StatefulSetSpec patchedSpec) throws ApiException { final String body = String.format(JSON_PATCH_STATEFUL_SET_REPLICAS_FORMAT, patchedSpec.getReplicas().toString()); @@ -201,7 +202,7 @@ private Object deserialize(String jsonStr, Class targetClass) { return (new Gson()).fromJson(jsonStr, targetClass); } - V1beta1StatefulSet getStatefulSet() throws ApiException { + V1StatefulSet getStatefulSet() throws ApiException { return client.readNamespacedStatefulSet(getTopologyName(), getNamespace(), null, null, null); } @@ -269,11 +270,11 @@ private static String setShardIdEnvironmentVariableCommand() { } - private V1beta1StatefulSet createStatefulSet(Resource containerResource, int numberOfInstances) { + private V1StatefulSet createStatefulSet(Resource containerResource, int numberOfInstances) { final String topologyName = getTopologyName(); final Config runtimeConfiguration = getRuntimeConfiguration(); - final V1beta1StatefulSet statefulSet = new V1beta1StatefulSet(); + final V1StatefulSet statefulSet = new V1StatefulSet(); // setup stateful set metadata final V1ObjectMeta objectMeta = new V1ObjectMeta(); @@ -281,7 +282,7 @@ private V1beta1StatefulSet createStatefulSet(Resource containerResource, int num statefulSet.metadata(objectMeta); // create the stateful set spec - final V1beta1StatefulSetSpec statefulSetSpec = new V1beta1StatefulSetSpec(); + final V1StatefulSetSpec statefulSetSpec = new V1StatefulSetSpec(); statefulSetSpec.serviceName(topologyName); statefulSetSpec.setReplicas(Runtime.numContainers(runtimeConfiguration).intValue()); @@ -414,10 +415,10 @@ private V1Container getContainer(List executorCommand, Resource resource // set container resources final V1ResourceRequirements resourceRequirements = new V1ResourceRequirements(); - final Map requests = new HashMap<>(); + final Map requests = new HashMap<>(); requests.put(KubernetesConstants.MEMORY, - KubernetesUtils.Megabytes(resource.getRam())); - requests.put(KubernetesConstants.CPU, Double.toString(resource.getCpu())); + Quantity.fromString(KubernetesUtils.Megabytes(resource.getRam()))); + requests.put(KubernetesConstants.CPU, Quantity.fromString(Double.toString(roundDecimal(resource.getCpu(), 3)))); resourceRequirements.setRequests(requests); container.setResources(resourceRequirements); @@ -468,4 +469,9 @@ private void mountVolumeIfPresent(V1Container container) { container.volumeMounts(Collections.singletonList(mount)); } } + + public static double roundDecimal(double value, int places) { + double scale = Math.pow(10, places); + return Math.round(value * scale) / scale; + } } diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesScheduler.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesScheduler.java index bb6a919f605..71545beab3c 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesScheduler.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesScheduler.java @@ -50,7 +50,7 @@ public class KubernetesScheduler implements IScheduler, IScalable { private UpdateTopologyManager updateTopologyManager; protected KubernetesController getController() { - return new AppsV1beta1Controller(configuration, runtimeConfiguration); + return new AppsV1Controller(configuration, runtimeConfiguration); } @Override