From 54ecca3502a38fef5eee2eb630f93c15f6772973 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 Apr 2020 14:15:57 +0800 Subject: [PATCH 1/6] security: tikv encryption kms config --- docs/api-references/docs.md | 295 +++++++++++++++++- images/tidb-backup-manager/Dockerfile | 10 +- manifests/crd.yaml | 1 + .../pingcap/v1alpha1/openapi_generated.go | 7 +- pkg/apis/pingcap/v1alpha1/tikv_config.go | 71 +++++ .../pingcap/v1alpha1/zz_generated.deepcopy.go | 81 +++++ 6 files changed, 459 insertions(+), 6 deletions(-) diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 0f6e15a47a..52c0055f17 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -3677,7 +3677,150 @@ User -

GcsStorageProvider +

MasterKeyFileConfig +

+

+(Appears on: +TiKVMasterKeyConfig) +

+

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+method
+ +string + +		 +

Encrypyion method, use master key encryption data key +Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr +Optional: Default to plaintext +optional

+
+path
+ +string + +		 +

Text file containing the key in hex form, end with ‘\n’

+
+

MasterKeyKMSConfig +

+

+(Appears on: +TiKVMasterKeyConfig) +

+

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+key-id
+ +string + +		 +

AWS CMK key-id it can be find in AWS Console or use aws cli +This field is required

+
+access-key
+ +string + +		 +

AccessKey of AWS user, leave empty if using other authrization method +optional

+
+secret-access-key
+ +string + +		 +

SecretKey of AWS user, leave empty if using other authrization method +optional

+
+region
+ +string + +		 +

Region of this KMS key +Optional: Default to us-east-1 +optional

+
+endpoint
+ +string + +		 +

Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty +optional

+
+

MemberPhase +(string alias)

+

+(Appears on: +PDStatus, +PumpStatus, +TiDBStatus, +TiKVStatus) +

+

+

MemberPhase is the current state of member

+

+

MemberType +(string alias)

+

+

MemberType represents member type

+

+

MonitorComponentAccessor +

+

+

+

MonitorContainer

(Appears on: @@ -10358,6 +10501,19 @@ TiKVSecurityConfig (Optional) + + +encryption
+ + +TiKVEncryptionConfig + + + + +(Optional) + +

TiKVCoprocessorConfig @@ -10929,6 +11085,78 @@ TiKVTitanDBConfig +

TiKVEncryptionConfig +

+

+(Appears on: +TiKVConfig) +

+

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+method
+ +string + +		 +

Encrypyion method, use data key encryption raw rocksdb data +Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr +Optional: Default to plaintext +optional

+
+data-key-rotation-period
+ +string + +		 +

The frequency of datakey rotation, It managered by tikv +Optional: default to 7d +optional

+
+master-key
+ + +TiKVMasterKeyConfig + + +		 +

Master key config

+
+previous-master-key
+ + +TiKVMasterKeyConfig + + +		 +

Previous master key config +It used in master key rotation, the data key should decryption by previous master key and then encrypytion by new master key

+

TiKVFailureStore

@@ -11140,6 +11368,71 @@ string +

TiKVMasterKeyConfig +

+

+(Appears on: +TiKVEncryptionConfig) +

+

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+type
+ +string + +		 +

Use KMS encryption or use file encryption, possible values: kms, file +If set to kms, kms MasterKeyKMSConfig should be filled, if set to file MasterKeyFileConfig should be filled +optional

+
+MasterKeyFileConfig
+ + +MasterKeyFileConfig + + +		 +

+(Members of MasterKeyFileConfig are embedded into this type.) +

+

Master key file config +If the type set to file, this config should be filled

+
+MasterKeyKMSConfig
+ + +MasterKeyKMSConfig + + +		 +

+(Members of MasterKeyKMSConfig are embedded into this type.) +

+

Master key KMS config +If the type set to kms, this config should be filled

+

TiKVPDConfig

diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index f69b0e6573..a5bb422c31 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -2,6 +2,7 @@ FROM pingcap/tidb-enterprise-tools:latest ARG VERSION=v1.51.0 ARG SHUSH_VERSION=v1.4.0 ARG TOOLKIT_VERSION=v3.0.12 +ARG BR_TOOLKIT_VERSION=v4.0.0-rc RUN apk update && apk add ca-certificates RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \ @@ -10,11 +11,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 -RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ - && tar -xzf br-latest-linux-amd64.tar.gz \ - && mv bin/br /usr/local/bin \ +RUN wget -nv https://download.pingcap.org/tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ + && tar -xzf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ + && mv tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64/bin/br /usr/local/bin \ && chmod 755 /usr/local/bin/br \ - && rm -rf br-latest-linux-amd64.tar.gz + && rm -rf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64 \ + && rm -rf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \ && mv shush_linux_amd64 /usr/local/bin/shush \ diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 3b5a2f5693..41002ed962 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6117,6 +6117,7 @@ spec: to false optional' type: boolean type: object + encryption: {} gc: properties: "\tbatch-keys": diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 1a1b68acea..12c5a5d68a 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -4806,11 +4806,16 @@ func schema_pkg_apis_pingcap_v1alpha1_TiKVConfig(ref common.ReferenceCallback) c Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVSecurityConfig"), }, }, + "encryption": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVEncryptionConfig"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVCoprocessorConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVDbConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVGCConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVImportConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVPDConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftDBConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftstoreConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVReadPoolConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVSecurityConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVServerConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVStorageConfig"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVCoprocessorConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVDbConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVEncryptionConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVGCConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVImportConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVPDConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftDBConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftstoreConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVReadPoolConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVSecurityConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVServerConfig", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVStorageConfig"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/tikv_config.go b/pkg/apis/pingcap/v1alpha1/tikv_config.go index ec03c2fb15..2d6bbced84 100644 --- a/pkg/apis/pingcap/v1alpha1/tikv_config.go +++ b/pkg/apis/pingcap/v1alpha1/tikv_config.go @@ -51,6 +51,8 @@ type TiKVConfig struct { PD *TiKVPDConfig `json:"pd,omitempty" toml:"pd,omitempty"` // +optional Security *TiKVSecurityConfig `json:"security,omitempty" toml:"security,omitempty"` + // +optional + Encryption *TiKVEncryptionConfig `json:"encryption,omitempty" toml:"encryption,omitempty"` } // +k8s:openapi-gen=true @@ -742,3 +744,72 @@ type TiKVCoprocessorConfig struct { // optional RegionSplitKeys *int64 `json:"region-split-keys,omitempty" toml:"region-split-keys,omitempty"` } + +type TiKVEncryptionConfig struct { + // Encrypyion method, use data key encryption raw rocksdb data + // Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr + // Optional: Default to plaintext + // optional + Method string `json:"method,omitempty" toml:"method,omitempty"` + + // The frequency of datakey rotation, It managered by tikv + // Optional: default to 7d + // optional + DataKeyRotationPeriod string `json:"data-key-rotation-period,omitempty" toml:"data-key-rotation-period,omitempty"` + + // Master key config + MasterKey *TiKVMasterKeyConfig `json:"master-key,omitempty" toml:"master-key,omitempty"` + + // Previous master key config + // It used in master key rotation, the data key should decryption by previous master key and then encrypytion by new master key + PreviousMasterKey *TiKVMasterKeyConfig `json:"previous-master-key,omitempty" toml:"previoud-master-key,omitempty"` +} + +type TiKVMasterKeyConfig struct { + // Use KMS encryption or use file encryption, possible values: kms, file + // If set to kms, kms MasterKeyKMSConfig should be filled, if set to file MasterKeyFileConfig should be filled + // optional + Type string `json:"type,omitempty" toml:"type,omitempty"` + + // Master key file config + // If the type set to file, this config should be filled + MasterKeyFileConfig `json:",inline"` + + // Master key KMS config + // If the type set to kms, this config should be filled + MasterKeyKMSConfig `json:",inline"` +} + +type MasterKeyFileConfig struct { + // Encrypyion method, use master key encryption data key + // Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr + // Optional: Default to plaintext + // optional + Method string `json:"method,omitempty" toml:"method,omitempty"` + + // Text file containing the key in hex form, end with '\n' + Path string `json:"path" toml:"path"` +} + +type MasterKeyKMSConfig struct { + // AWS CMK key-id it can be find in AWS Console or use aws cli + // This field is required + KeyID string `json:"key-id" toml:"key-id` + + // AccessKey of AWS user, leave empty if using other authrization method + // optional + AccessKey string `json:"access-key,omitempty" toml:"access-key,omitempty"` + + // SecretKey of AWS user, leave empty if using other authrization method + // optional + SecretKey string `json:"secret-access-key,omitempty" toml:"access-key,omitempty"` + + // Region of this KMS key + // Optional: Default to us-east-1 + // optional + Region string `json:"region,omitempty" toml:"region,omitempty"` + + // Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty + // optional + Endpoint string `json:"endpoint,omitempty" toml:"endpoint,omitempty"` +} diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index a56247a7d6..d2be3759ea 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -1308,6 +1308,38 @@ func (in *Log) DeepCopy() *Log { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MasterKeyFileConfig) DeepCopyInto(out *MasterKeyFileConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MasterKeyFileConfig. +func (in *MasterKeyFileConfig) DeepCopy() *MasterKeyFileConfig { + if in == nil { + return nil + } + out := new(MasterKeyFileConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MasterKeyKMSConfig) DeepCopyInto(out *MasterKeyKMSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MasterKeyKMSConfig. +func (in *MasterKeyKMSConfig) DeepCopy() *MasterKeyKMSConfig { + if in == nil { + return nil + } + out := new(MasterKeyKMSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MonitorContainer) DeepCopyInto(out *MonitorContainer) { *out = *in @@ -3789,6 +3821,11 @@ func (in *TiKVConfig) DeepCopyInto(out *TiKVConfig) { *out = new(TiKVSecurityConfig) (*in).DeepCopyInto(*out) } + if in.Encryption != nil { + in, out := &in.Encryption, &out.Encryption + *out = new(TiKVEncryptionConfig) + (*in).DeepCopyInto(*out) + } return } @@ -3985,6 +4022,32 @@ func (in *TiKVDbConfig) DeepCopy() *TiKVDbConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TiKVEncryptionConfig) DeepCopyInto(out *TiKVEncryptionConfig) { + *out = *in + if in.MasterKey != nil { + in, out := &in.MasterKey, &out.MasterKey + *out = new(TiKVMasterKeyConfig) + **out = **in + } + if in.PreviousMasterKey != nil { + in, out := &in.PreviousMasterKey, &out.PreviousMasterKey + *out = new(TiKVMasterKeyConfig) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TiKVEncryptionConfig. +func (in *TiKVEncryptionConfig) DeepCopy() *TiKVEncryptionConfig { + if in == nil { + return nil + } + out := new(TiKVEncryptionConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TiKVFailureStore) DeepCopyInto(out *TiKVFailureStore) { *out = *in @@ -4064,6 +4127,24 @@ func (in *TiKVImportConfig) DeepCopy() *TiKVImportConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TiKVMasterKeyConfig) DeepCopyInto(out *TiKVMasterKeyConfig) { + *out = *in + out.MasterKeyFileConfig = in.MasterKeyFileConfig + out.MasterKeyKMSConfig = in.MasterKeyKMSConfig + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TiKVMasterKeyConfig. +func (in *TiKVMasterKeyConfig) DeepCopy() *TiKVMasterKeyConfig { + if in == nil { + return nil + } + out := new(TiKVMasterKeyConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TiKVPDConfig) DeepCopyInto(out *TiKVPDConfig) { *out = *in From 57e4f1e36f307757c2c1de1ed5da76c0cc3e19c4 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 Apr 2020 14:44:53 +0800 Subject: [PATCH 2/6] fix doc --- docs/api-references/docs.md | 256 ++++++++++++++++-------------------- 1 file changed, 112 insertions(+), 144 deletions(-) diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 52c0055f17..e2ee9cfbc8 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -3677,150 +3677,7 @@ User -

MasterKeyFileConfig -

-

-(Appears on: -TiKVMasterKeyConfig) -

-

-

- - - - - - - - - - - - - - - - - -
FieldDescription
-method
- -string - -		 -

Encrypyion method, use master key encryption data key -Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr -Optional: Default to plaintext -optional

-
-path
- -string - -		 -

Text file containing the key in hex form, end with ‘\n’

-
-

MasterKeyKMSConfig -

-

-(Appears on: -TiKVMasterKeyConfig) -

-

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-key-id
- -string - -		 -

AWS CMK key-id it can be find in AWS Console or use aws cli -This field is required

-
-access-key
- -string - -		 -

AccessKey of AWS user, leave empty if using other authrization method -optional

-
-secret-access-key
- -string - -		 -

SecretKey of AWS user, leave empty if using other authrization method -optional

-
-region
- -string - -		 -

Region of this KMS key -Optional: Default to us-east-1 -optional

-
-endpoint
- -string - -		 -

Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty -optional

-
-

MemberPhase -(string alias)

-

-(Appears on: -PDStatus, -PumpStatus, -TiDBStatus, -TiKVStatus) -

-

-

MemberPhase is the current state of member

-

-

MemberType -(string alias)

-

-

MemberType represents member type

-

-

MonitorComponentAccessor -

-

-

-

MonitorContainer +

GcsStorageProvider

(Appears on: @@ -4393,6 +4250,117 @@ uint32 +

MasterKeyFileConfig +

+

+(Appears on: +TiKVMasterKeyConfig) +

+

+

+ + + + + + + + + + + + + +
FieldDescription
+method
+ +string + +		 +

Encrypyion method, use master key encryption data key +Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr +Optional: Default to plaintext +optional

+
+

MasterKeyKMSConfig +

+

+(Appears on: +TiKVMasterKeyConfig) +

+

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+key-id
+ +string + +		 +

AWS CMK key-id it can be find in AWS Console or use aws cli +This field is required

+
+access-key
+ +string + +		 +

AccessKey of AWS user, leave empty if using other authrization method +optional

+
+secret-access-key
+ +string + +		 +

SecretKey of AWS user, leave empty if using other authrization method +optional

+
+region
+ +string + +		 +

Region of this KMS key +Optional: Default to us-east-1 +optional

+
+endpoint
+ +string + +		 +

Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty +optional

+

MemberPhase (string alias)

From 4146409c8d28a778a7f23df5720f5df42f8498d5 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 Apr 2020 14:52:43 +0800 Subject: [PATCH 3/6] fix tiny --- images/tidb-backup-manager/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index a5bb422c31..3d52061b9e 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -2,7 +2,6 @@ FROM pingcap/tidb-enterprise-tools:latest ARG VERSION=v1.51.0 ARG SHUSH_VERSION=v1.4.0 ARG TOOLKIT_VERSION=v3.0.12 -ARG BR_TOOLKIT_VERSION=v4.0.0-rc RUN apk update && apk add ca-certificates RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \ @@ -11,6 +10,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 +UN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ + && tar -xzf br-latest-linux-amd64.tar.gz \ + && mv bin/br /usr/local/bin \ + && chmod 755 /usr/local/bin/br \ + && rm -rf br-latest-linux-amd64.tar.gz + RUN wget -nv https://download.pingcap.org/tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ && tar -xzf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ && mv tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64/bin/br /usr/local/bin \ From 53dc3b6e56dab75a99c276addfc67228755a1173 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Fri, 10 Apr 2020 14:53:54 +0800 Subject: [PATCH 4/6] fix tiny --- images/tidb-backup-manager/Dockerfile | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index 3d52061b9e..f69b0e6573 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -10,19 +10,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 -UN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ +RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ && tar -xzf br-latest-linux-amd64.tar.gz \ && mv bin/br /usr/local/bin \ && chmod 755 /usr/local/bin/br \ && rm -rf br-latest-linux-amd64.tar.gz -RUN wget -nv https://download.pingcap.org/tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ - && tar -xzf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz \ - && mv tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64/bin/br /usr/local/bin \ - && chmod 755 /usr/local/bin/br \ - && rm -rf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64 \ - && rm -rf tidb-toolkit-${BR_TOOLKIT_VERSION}-linux-amd64.tar.gz - RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \ && mv shush_linux_amd64 /usr/local/bin/shush \ && chmod 755 /usr/local/bin/shush From e4498371a80970c3976c24ea622a573615d05e4d Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Mon, 13 Apr 2020 14:44:43 +0800 Subject: [PATCH 5/6] fix tiny --- pkg/apis/pingcap/v1alpha1/tikv_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/pingcap/v1alpha1/tikv_config.go b/pkg/apis/pingcap/v1alpha1/tikv_config.go index 2d6bbced84..82be0a4565 100644 --- a/pkg/apis/pingcap/v1alpha1/tikv_config.go +++ b/pkg/apis/pingcap/v1alpha1/tikv_config.go @@ -794,7 +794,7 @@ type MasterKeyFileConfig struct { type MasterKeyKMSConfig struct { // AWS CMK key-id it can be find in AWS Console or use aws cli // This field is required - KeyID string `json:"key-id" toml:"key-id` + KeyID string `json:"key-id" toml:"key-id"` // AccessKey of AWS user, leave empty if using other authrization method // optional From b960ede09ec7f87129ae20e86512445042d38bb6 Mon Sep 17 00:00:00 2001 From: shuijing198799 Date: Mon, 13 Apr 2020 16:08:10 +0800 Subject: [PATCH 6/6] address comment --- manifests/crd.yaml | 98 ++++++++- .../pingcap/v1alpha1/openapi_generated.go | 188 ++++++++++++++++++ pkg/apis/pingcap/v1alpha1/tikv_config.go | 4 + 3 files changed, 289 insertions(+), 1 deletion(-) diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 41002ed962..b4eb49031e 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6117,7 +6117,103 @@ spec: to false optional' type: boolean type: object - encryption: {} + encryption: + properties: + data-key-rotation-period: + description: 'The frequency of datakey rotation, It managered + by tikv Optional: default to 7d optional' + type: string + master-key: + properties: + access-key: + description: AccessKey of AWS user, leave empty if using + other authrization method optional + type: string + endpoint: + description: Used for KMS compatible KMS, such as Ceph, + minio, If use AWS, leave empty optional + type: string + key-id: + description: AWS CMK key-id it can be find in AWS Console + or use aws cli This field is required + type: string + method: + description: 'Encrypyion method, use master key encryption + data key Possible values: plaintext, aes128-ctr, aes192-ctr, + aes256-ctr Optional: Default to plaintext optional' + type: string + path: + description: |- + Text file containing the key in hex form, end with ' + ' + type: string + region: + description: 'Region of this KMS key Optional: Default + to us-east-1 optional' + type: string + secret-access-key: + description: SecretKey of AWS user, leave empty if using + other authrization method optional + type: string + type: + description: 'Use KMS encryption or use file encryption, + possible values: kms, file If set to kms, kms MasterKeyKMSConfig + should be filled, if set to file MasterKeyFileConfig + should be filled optional' + type: string + required: + - path + - key-id + type: object + method: + description: 'Encrypyion method, use data key encryption + raw rocksdb data Possible values: plaintext, aes128-ctr, + aes192-ctr, aes256-ctr Optional: Default to plaintext + optional' + type: string + previous-master-key: + properties: + access-key: + description: AccessKey of AWS user, leave empty if using + other authrization method optional + type: string + endpoint: + description: Used for KMS compatible KMS, such as Ceph, + minio, If use AWS, leave empty optional + type: string + key-id: + description: AWS CMK key-id it can be find in AWS Console + or use aws cli This field is required + type: string + method: + description: 'Encrypyion method, use master key encryption + data key Possible values: plaintext, aes128-ctr, aes192-ctr, + aes256-ctr Optional: Default to plaintext optional' + type: string + path: + description: |- + Text file containing the key in hex form, end with ' + ' + type: string + region: + description: 'Region of this KMS key Optional: Default + to us-east-1 optional' + type: string + secret-access-key: + description: SecretKey of AWS user, leave empty if using + other authrization method optional + type: string + type: + description: 'Use KMS encryption or use file encryption, + possible values: kms, file If set to kms, kms MasterKeyKMSConfig + should be filled, if set to file MasterKeyFileConfig + should be filled optional' + type: string + required: + - path + - key-id + type: object + type: object gc: properties: "\tbatch-keys": diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 12c5a5d68a..6dc513a64e 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -47,6 +47,8 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.HelperSpec": schema_pkg_apis_pingcap_v1alpha1_HelperSpec(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.IsolationRead": schema_pkg_apis_pingcap_v1alpha1_IsolationRead(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Log": schema_pkg_apis_pingcap_v1alpha1_Log(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MasterKeyFileConfig": schema_pkg_apis_pingcap_v1alpha1_MasterKeyFileConfig(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MasterKeyKMSConfig": schema_pkg_apis_pingcap_v1alpha1_MasterKeyKMSConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MonitorContainer": schema_pkg_apis_pingcap_v1alpha1_MonitorContainer(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.OpenTracing": schema_pkg_apis_pingcap_v1alpha1_OpenTracing(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.OpenTracingReporter": schema_pkg_apis_pingcap_v1alpha1_OpenTracingReporter(ref), @@ -93,8 +95,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVCoprocessorConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVCoprocessorConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVCoprocessorReadPoolConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVCoprocessorReadPoolConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVDbConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVDbConfig(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVEncryptionConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVEncryptionConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVGCConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVGCConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVImportConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVImportConfig(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVMasterKeyConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVMasterKeyConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVPDConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVPDConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftDBConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVRaftDBConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVRaftstoreConfig": schema_pkg_apis_pingcap_v1alpha1_TiKVRaftstoreConfig(ref), @@ -1482,6 +1486,81 @@ func schema_pkg_apis_pingcap_v1alpha1_Log(ref common.ReferenceCallback) common.O } } +func schema_pkg_apis_pingcap_v1alpha1_MasterKeyFileConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "method": { + SchemaProps: spec.SchemaProps{ + Description: "Encrypyion method, use master key encryption data key Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr Optional: Default to plaintext optional", + Type: []string{"string"}, + Format: "", + }, + }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "Text file containing the key in hex form, end with '\n'", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"path"}, + }, + }, + } +} + +func schema_pkg_apis_pingcap_v1alpha1_MasterKeyKMSConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key-id": { + SchemaProps: spec.SchemaProps{ + Description: "AWS CMK key-id it can be find in AWS Console or use aws cli This field is required", + Type: []string{"string"}, + Format: "", + }, + }, + "access-key": { + SchemaProps: spec.SchemaProps{ + Description: "AccessKey of AWS user, leave empty if using other authrization method optional", + Type: []string{"string"}, + Format: "", + }, + }, + "secret-access-key": { + SchemaProps: spec.SchemaProps{ + Description: "SecretKey of AWS user, leave empty if using other authrization method optional", + Type: []string{"string"}, + Format: "", + }, + }, + "region": { + SchemaProps: spec.SchemaProps{ + Description: "Region of this KMS key Optional: Default to us-east-1 optional", + Type: []string{"string"}, + Format: "", + }, + }, + "endpoint": { + SchemaProps: spec.SchemaProps{ + Description: "Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty optional", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"key-id"}, + }, + }, + } +} + func schema_pkg_apis_pingcap_v1alpha1_MonitorContainer(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5129,6 +5208,46 @@ func schema_pkg_apis_pingcap_v1alpha1_TiKVDbConfig(ref common.ReferenceCallback) } } +func schema_pkg_apis_pingcap_v1alpha1_TiKVEncryptionConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "method": { + SchemaProps: spec.SchemaProps{ + Description: "Encrypyion method, use data key encryption raw rocksdb data Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr Optional: Default to plaintext optional", + Type: []string{"string"}, + Format: "", + }, + }, + "data-key-rotation-period": { + SchemaProps: spec.SchemaProps{ + Description: "The frequency of datakey rotation, It managered by tikv Optional: default to 7d optional", + Type: []string{"string"}, + Format: "", + }, + }, + "master-key": { + SchemaProps: spec.SchemaProps{ + Description: "Master key config", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVMasterKeyConfig"), + }, + }, + "previous-master-key": { + SchemaProps: spec.SchemaProps{ + Description: "Previous master key config It used in master key rotation, the data key should decryption by previous master key and then encrypytion by new master key", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVMasterKeyConfig"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVMasterKeyConfig"}, + } +} + func schema_pkg_apis_pingcap_v1alpha1_TiKVGCConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5220,6 +5339,75 @@ func schema_pkg_apis_pingcap_v1alpha1_TiKVImportConfig(ref common.ReferenceCallb } } +func schema_pkg_apis_pingcap_v1alpha1_TiKVMasterKeyConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "Use KMS encryption or use file encryption, possible values: kms, file If set to kms, kms MasterKeyKMSConfig should be filled, if set to file MasterKeyFileConfig should be filled optional", + Type: []string{"string"}, + Format: "", + }, + }, + "method": { + SchemaProps: spec.SchemaProps{ + Description: "Encrypyion method, use master key encryption data key Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr Optional: Default to plaintext optional", + Type: []string{"string"}, + Format: "", + }, + }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "Text file containing the key in hex form, end with '\n'", + Type: []string{"string"}, + Format: "", + }, + }, + "key-id": { + SchemaProps: spec.SchemaProps{ + Description: "AWS CMK key-id it can be find in AWS Console or use aws cli This field is required", + Type: []string{"string"}, + Format: "", + }, + }, + "access-key": { + SchemaProps: spec.SchemaProps{ + Description: "AccessKey of AWS user, leave empty if using other authrization method optional", + Type: []string{"string"}, + Format: "", + }, + }, + "secret-access-key": { + SchemaProps: spec.SchemaProps{ + Description: "SecretKey of AWS user, leave empty if using other authrization method optional", + Type: []string{"string"}, + Format: "", + }, + }, + "region": { + SchemaProps: spec.SchemaProps{ + Description: "Region of this KMS key Optional: Default to us-east-1 optional", + Type: []string{"string"}, + Format: "", + }, + }, + "endpoint": { + SchemaProps: spec.SchemaProps{ + Description: "Used for KMS compatible KMS, such as Ceph, minio, If use AWS, leave empty optional", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"path", "key-id"}, + }, + }, + } +} + func schema_pkg_apis_pingcap_v1alpha1_TiKVPDConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/apis/pingcap/v1alpha1/tikv_config.go b/pkg/apis/pingcap/v1alpha1/tikv_config.go index 82be0a4565..0257367d87 100644 --- a/pkg/apis/pingcap/v1alpha1/tikv_config.go +++ b/pkg/apis/pingcap/v1alpha1/tikv_config.go @@ -745,6 +745,7 @@ type TiKVCoprocessorConfig struct { RegionSplitKeys *int64 `json:"region-split-keys,omitempty" toml:"region-split-keys,omitempty"` } +// +k8s:openapi-gen=true type TiKVEncryptionConfig struct { // Encrypyion method, use data key encryption raw rocksdb data // Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr @@ -765,6 +766,7 @@ type TiKVEncryptionConfig struct { PreviousMasterKey *TiKVMasterKeyConfig `json:"previous-master-key,omitempty" toml:"previoud-master-key,omitempty"` } +// +k8s:openapi-gen=true type TiKVMasterKeyConfig struct { // Use KMS encryption or use file encryption, possible values: kms, file // If set to kms, kms MasterKeyKMSConfig should be filled, if set to file MasterKeyFileConfig should be filled @@ -780,6 +782,7 @@ type TiKVMasterKeyConfig struct { MasterKeyKMSConfig `json:",inline"` } +// +k8s:openapi-gen=true type MasterKeyFileConfig struct { // Encrypyion method, use master key encryption data key // Possible values: plaintext, aes128-ctr, aes192-ctr, aes256-ctr @@ -791,6 +794,7 @@ type MasterKeyFileConfig struct { Path string `json:"path" toml:"path"` } +// +k8s:openapi-gen=true type MasterKeyKMSConfig struct { // AWS CMK key-id it can be find in AWS Console or use aws cli // This field is required