Automated cherry pick of #915: Support configuring sysctls for Pods and enable net.*

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -45,6 +45,8 @@ spec:
  annotations:
 {{ toYaml .Values.pd.annotations | indent 6 }}
  {{- end }}
+ podSecurityContext:
+{{ toYaml .Values.pd.podSecurityContext | indent 6}}
  tikv:
  replicas: {{ .Values.tikv.replicas }}
  image: {{ .Values.tikv.image }}
@@ -68,6 +70,8 @@ spec:
 {{ toYaml .Values.tikv.annotations | indent 6 }}
  {{- end }}
  maxFailoverCount: {{ .Values.tikv.maxFailoverCount | default 3 }}
+ podSecurityContext:
+{{ toYaml .Values.tikv.podSecurityContext | indent 6}}
  tidb:
  replicas: {{ .Values.tidb.replicas }}
  image: {{ .Values.tidb.image }}
@@ -87,6 +91,8 @@ spec:
  annotations:
 {{ toYaml .Values.tidb.annotations | indent 6 }}
  {{- end }}
+ podSecurityContext:
+{{ toYaml .Values.tidb.podSecurityContext | indent 6}}
  binlogEnabled: {{ .Values.binlog.pump.create | default false }}
  maxFailoverCount: {{ .Values.tidb.maxFailoverCount | default 3 }}
  separateSlowLog: {{ .Values.tidb.separateSlowLog | default false }}

charts/tidb-cluster/values.yaml

@@ -154,6 +154,10 @@ pd:
  # effect: "NoSchedule"
  annotations: {}
 
+ # Specify the security context of PD Pod.
+ # refer to https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+ podSecurityContext: {}
+
 tikv:
  # Please refer to https://github.com/tikv/tikv/blob/master/etc/config-template.toml for the default
  # tikv configurations (change to the tags of your tikv version),
@@ -234,6 +238,10 @@ tikv:
  # maxFailoverCount is used to configure the maximum number of TiKV nodes that TiDB Operator can create when failover occurs.
  maxFailoverCount: 3
 
+ # Specify the security context of TiKV Pod.
+ # refer to https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+ podSecurityContext: {}
+
 tidb:
  # Please refer to https://github.com/pingcap/tidb/blob/master/config/config.toml.example for the default
  # tidb configurations(change to the tags of your tidb version),
@@ -290,6 +298,11 @@ tidb:
  # value: tidb
  # effect: "NoSchedule"
  annotations: {}
+
+ # Specify the security context of TiDB Pod.
+ # refer to https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+ podSecurityContext: {}
+
  maxFailoverCount: 3
  service:
  type: NodePort

deploy/aws/main.tf

@@ -3,7 +3,7 @@ provider "aws" {
 }
 
 locals {
- eks = module.tidb-operator.eks
+ eks  = module.tidb-operator.eks
  subnets = module.vpc.private_subnets
 }
 

deploy/modules/aws/tidb-cluster/data.tf

@@ -12,7 +12,7 @@ data "aws_ami" "eks_worker" {
 
 data "template_file" "userdata" {
  template = file("${path.module}/templates/userdata.sh.tpl")
- count = local.worker_group_count
+ count = length(local.tidb_cluster_worker_groups)
 
  vars = {
  cluster_name = var.eks.cluster_id

deploy/modules/aws/tidb-cluster/local.tf

@@ -40,13 +40,13 @@ locals {
 
  tidb_cluster_worker_groups = [
  {
- name  = "${var.cluster_name}-pd"
- key_name  = var.ssh_key_name
- instance_type  = var.pd_instance_type
- root_volume_size  = "50"
- public_ip  = false
+ name = "${var.cluster_name}-pd"
+ key_name = var.ssh_key_name
+ instance_type = var.pd_instance_type
+ root_volume_size = "50"
+ public_ip = false
  # the space separator is safe when the extra args is empty or prefixed by spaces (the same hereafter)
- kubelet_extra_args  = join(" ",
+ kubelet_extra_args = join(" ",
  [
  "--register-with-taints=dedicated=${var.cluster_name}-pd:NoSchedule",
  "--node-labels=dedicated=${var.cluster_name}-pd,pingcap.com/aws-local-ssd=true,zone=${local.aws_zone_getter}",
@@ -58,12 +58,12 @@ locals {
  # additional_userdata = file("userdata.sh")
  },
  {
- name  = "${var.cluster_name}-tikv"
- key_name  = var.ssh_key_name
- instance_type  = var.tikv_instance_type
- root_volume_size  = "50"
- public_ip  = false
- kubelet_extra_args  = join(" ",
+ name = "${var.cluster_name}-tikv"
+ key_name = var.ssh_key_name
+ instance_type = var.tikv_instance_type
+ root_volume_size = "50"
+ public_ip = false
+ kubelet_extra_args = join(" ",
  [
  "--register-with-taints=dedicated=${var.cluster_name}-tikv:NoSchedule",
  "--node-labels=dedicated=${var.cluster_name}-tikv,pingcap.com/aws-local-ssd=true,zone=${local.aws_zone_getter}",
@@ -77,14 +77,15 @@ locals {
  suspended_processes = ["ReplaceUnhealthy"]
  },
  {
- name  = "${var.cluster_name}-tidb"
- key_name  = var.ssh_key_name
- instance_type  = var.tidb_instance_type
- root_volume_type  = "gp2"
- root_volume_size  = "50"
- public_ip  = false
- kubelet_extra_args  = join(" ",
+ name = "${var.cluster_name}-tidb"
+ key_name = var.ssh_key_name
+ instance_type = var.tidb_instance_type
+ root_volume_type = "gp2"
+ root_volume_size = "50"
+ public_ip = false
+ kubelet_extra_args = join(" ",
  [
+ "--allowed-unsafe-sysctls=\\\"net.*\\\"",
  "--register-with-taints=dedicated=${var.cluster_name}-tidb:NoSchedule",
  "--node-labels=dedicated=${var.cluster_name}-tidb,zone=${local.aws_zone_getter}",
  lookup(var.group_kubelet_extra_args, "tidb", var.kubelet_extra_args)
@@ -94,20 +95,20 @@ locals {
  asg_max_size = var.tidb_count + 2
  },
  {
- name  = "${var.cluster_name}-monitor"
- key_name  = var.ssh_key_name
- instance_type  = var.monitor_instance_type
- root_volume_type  = "gp2"
- root_volume_size  = "50"
- public_ip  = false
- kubelet_extra_args  = join(" ",
+ name = "${var.cluster_name}-monitor"
+ key_name = var.ssh_key_name
+ instance_type = var.monitor_instance_type
+ root_volume_type = "gp2"
+ root_volume_size = "50"
+ public_ip = false
+ kubelet_extra_args = join(" ",
  [
  "--node-labels=zone=${local.aws_zone_getter}",
  lookup(var.group_kubelet_extra_args, "monitor", var.kubelet_extra_args)
  ]
  )
- asg_desired_capacity = 1
- asg_max_size = 3
+ asg_desired_capacity  = 1
+ asg_max_size  = 3
  }
  ]
 

deploy/modules/aws/tidb-cluster/templates/userdata.sh.tpl

@@ -10,6 +10,7 @@ root hard nofile 1000000
 root soft core unlimited
 root soft stack 10240
 EOF
+
 # config docker ulimit
 cp /usr/lib/systemd/system/docker.service /etc/systemd/system/docker.service
 sed -i 's/LimitNOFILE=infinity/LimitNOFILE=1048576/' /etc/systemd/system/docker.service

deploy/modules/aws/tidb-cluster/values/default.yaml

@@ -15,6 +15,17 @@ tidb:
  service.beta.kubernetes.io/aws-load-balancer-type: nlb
  service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true'
  separateSlowLog: true
+ config: |
+ [log]
+ level = "info"
+ [performance]
+ tcp-keep-alive = true
+ podSecurityContext:
+ sysctls:
+ - name: net.ipv4.tcp_keepalive_time
+ value: "300"
+ - name: net.ipv4.tcp_keepalive_intvl
+ value: "75"
 monitor:
  storage: 100Gi
  storageClassName: ebs-gp2

go.mod

@@ -113,7 +113,7 @@ require (
  k8s.io/apiserver v0.0.0-20190118115647-a748535592ba
  k8s.io/cli-runtime v0.0.0-20190118125240-caee4253d968
  k8s.io/client-go v0.0.0-20190115164855-701b91367003
- k8s.io/code-generator v0.0.0-20191109100332-a9a0d9c0b3aa
+ k8s.io/code-generator v0.0.0-20191114215150-2a85f169f05f
  k8s.io/klog v1.0.0
  k8s.io/kubernetes v1.12.5
  k8s.io/metrics v0.0.0-20190118124808-33c1aed8dc65 // indirect

go.sum

@@ -360,8 +360,8 @@ k8s.io/cli-runtime v0.0.0-20190118125240-caee4253d968 h1:VXLj8aMvJEo14Utv+knJDs0
 k8s.io/cli-runtime v0.0.0-20190118125240-caee4253d968/go.mod h1:qWnH3/b8sp/l7EvlDh7ulDU3UWA4P4N1NFbEEP791tM=
 k8s.io/client-go v0.0.0-20190115164855-701b91367003 h1:gQQC0U1hM6L808TYvGGO/5vhUisGw384axV7rqFUv04=
 k8s.io/client-go v0.0.0-20190115164855-701b91367003/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
-k8s.io/code-generator v0.0.0-20191109100332-a9a0d9c0b3aa h1:4feCF84yk6VEXdpOwOhwB1YIwobejEwKBzgHY0xa9Co=
-k8s.io/code-generator v0.0.0-20191109100332-a9a0d9c0b3aa/go.mod h1:fRFrKVixH946mn5PeglV2fvxbE86JesGi16bsWZ1xz4=
+k8s.io/code-generator v0.0.0-20191114215150-2a85f169f05f h1:sqMKZZhZNG3WgglCBOn9zRJrbEnCeJ5cP+4cv9awWsk=
+k8s.io/code-generator v0.0.0-20191114215150-2a85f169f05f/go.mod h1:Vh0irzg7dL9pFS4c8hFsali5txtbmse3MFS4zEH7Thg=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM=
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

pkg/apis/pingcap.com/v1alpha1/types.go

@@ -90,9 +90,9 @@ type TidbClusterSpec struct {
  TiKV TiKVSpec `json:"tikv,omitempty"`
  TiKVPromGateway TiKVPromGatewaySpec `json:"tikvPromGateway,omitempty"`
  // Services list non-headless services type used in TidbCluster
- Services []Service `json:"services,omitempty"`
- PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
- Timezone string `json:"timezone,omitempty"`
+ Services  []Service `json:"services,omitempty"`
+ PVReclaimPolicy  corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
+ Timezone  string `json:"timezone,omitempty"`
 }
 
 // TidbClusterStatus represents the current status of a tidb cluster.
@@ -106,27 +106,21 @@ type TidbClusterStatus struct {
 // PDSpec contains details of PD member
 type PDSpec struct {
  ContainerSpec
- Replicas int32 `json:"replicas"`
- Affinity *corev1.Affinity `json:"affinity,omitempty"`
- NodeSelector map[string]string `json:"nodeSelector,omitempty"`
- StorageClassName string `json:"storageClassName,omitempty"`
- Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
- Annotations map[string]string `json:"annotations,omitempty"`
+ PodAttributesSpec
+ Replicas int32 `json:"replicas"`
+ StorageClassName string `json:"storageClassName,omitempty"`
 }
 
 // TiDBSpec contains details of PD member
 type TiDBSpec struct {
  ContainerSpec
+ PodAttributesSpec
  Replicas int32 `json:"replicas"`
- Affinity *corev1.Affinity `json:"affinity,omitempty"`
- NodeSelector map[string]string `json:"nodeSelector,omitempty"`
  StorageClassName string `json:"storageClassName,omitempty"`
- Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
- Annotations map[string]string `json:"annotations,omitempty"`
  BinlogEnabled bool `json:"binlogEnabled,omitempty"`
- MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"`
  SeparateSlowLog bool `json:"separateSlowLog,omitempty"`
  SlowLogTailer TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"`
+ MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"`
 }
 
 // TiDBSlowLogTailerSpec represents an optional log tailer sidecar with TiDB
@@ -137,14 +131,11 @@ type TiDBSlowLogTailerSpec struct {
 // TiKVSpec contains details of TiKV members
 type TiKVSpec struct {
  ContainerSpec
- Privileged bool `json:"privileged,omitempty"`
- Replicas int32 `json:"replicas"`
- Affinity *corev1.Affinity `json:"affinity,omitempty"`
- NodeSelector map[string]string `json:"nodeSelector,omitempty"`
- StorageClassName string `json:"storageClassName,omitempty"`
- Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
- Annotations map[string]string `json:"annotations,omitempty"`
- MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"`
+ PodAttributesSpec
+ Replicas int32 `json:"replicas"`
+ Privileged bool `json:"privileged,omitempty"`
+ StorageClassName string `json:"storageClassName,omitempty"`
+ MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"`
 }
 
 // TiKVPromGatewaySpec runs as a sidecar with TiKVSpec
@@ -160,6 +151,15 @@ type ContainerSpec struct {
  Limits *ResourceRequirement `json:"limits,omitempty"`
 }
 
+// PodAttributesControlSpec is a spec of some general attributes of TiKV, TiDB and PD Pods
+type PodAttributesSpec struct {
+ Affinity *corev1.Affinity `json:"affinity,omitempty"`
+ NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+ Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+ Annotations map[string]string `json:"annotations,omitempty"`
+ PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
+}
+
 // Service represent service type used in TidbCluster
 type Service struct {
  Name string `json:"name,omitempty"`