Merge branch 'master' into bindingNormSQL

br/pkg/storage/parse_test.go

@@ -69,14 +69,15 @@ func TestCreateStorage(t *testing.T) {
  require.Equal(t, "TestKey", s3.SseKmsKeyId)
 
  // special character in access keys
- s, err = ParseBackend(`s3://bucket4/prefix/path?access-key=NXN7IPIOSAAKDEEOLMAF&secret-access-key=nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw`, nil)
+ s, err = ParseBackend(`s3://bucket4/prefix/path?access-key=NXN7IPIOSAAKDEEOLMAF&secret-access-key=nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw&session-token=FQoDYXdzEPP//////////wEaDPv5GPAhRW8pw6/nsiKsAZu7sZDCXPtEBEurxmvyV1r+nWy1I4VPbdIJV+iDnotwS3PKIyj+yDnOeigMf2yp9y2Dg9D7r51vWUyUQQfceZi9/8Ghy38RcOnWImhNdVP5zl1zh85FHz6ytePo+puHZwfTkuAQHj38gy6VF/14GU17qDcPTfjhbETGqEmh8QX6xfmWlO0ZrTmsAo4ZHav8yzbbl3oYdCLICOjMhOO1oY+B/DiURk3ZLPjaXyoo2Iql2QU=`, nil)
  require.NoError(t, err)
  s3 = s.GetS3()
  require.NotNil(t, s3)
  require.Equal(t, "bucket4", s3.Bucket)
  require.Equal(t, "prefix/path", s3.Prefix)
  require.Equal(t, "NXN7IPIOSAAKDEEOLMAF", s3.AccessKey)
  require.Equal(t, "nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw", s3.SecretAccessKey)
+ require.Equal(t, "FQoDYXdzEPP//////////wEaDPv5GPAhRW8pw6/nsiKsAZu7sZDCXPtEBEurxmvyV1r+nWy1I4VPbdIJV+iDnotwS3PKIyj+yDnOeigMf2yp9y2Dg9D7r51vWUyUQQfceZi9/8Ghy38RcOnWImhNdVP5zl1zh85FHz6ytePo+puHZwfTkuAQHj38gy6VF/14GU17qDcPTfjhbETGqEmh8QX6xfmWlO0ZrTmsAo4ZHav8yzbbl3oYdCLICOjMhOO1oY+B/DiURk3ZLPjaXyoo2Iql2QU=", s3.SessionToken)
  require.True(t, s3.ForcePathStyle)
 
  // parse role ARN and external ID

br/pkg/storage/s3.go

@@ -140,6 +140,7 @@ type S3BackendOptions struct {
  ACL string `json:"acl" toml:"acl"`
  AccessKey string `json:"access-key" toml:"access-key"`
  SecretAccessKey string `json:"secret-access-key" toml:"secret-access-key"`
+ SessionToken string `json:"session-token" toml:"session-token"`
  Provider string `json:"provider" toml:"provider"`
  ForcePathStyle bool `json:"force-path-style" toml:"force-path-style"`
  UseAccelerateEndpoint bool `json:"use-accelerate-endpoint" toml:"use-accelerate-endpoint"`
@@ -184,6 +185,7 @@ func (options *S3BackendOptions) Apply(s3 *backuppb.S3) error {
  s3.Acl = options.ACL
  s3.AccessKey = options.AccessKey
  s3.SecretAccessKey = options.SecretAccessKey
+ s3.SessionToken = options.SessionToken
  s3.ForcePathStyle = options.ForcePathStyle
  s3.RoleArn = options.RoleARN
  s3.ExternalId = options.ExternalID
@@ -262,7 +264,7 @@ func NewS3StorageForTest(svc s3iface.S3API, options *backuppb.S3) *S3Storage {
 // auto access without ak / sk.
 func autoNewCred(qs *backuppb.S3) (cred *credentials.Credentials, err error) {
  if qs.AccessKey != "" && qs.SecretAccessKey != "" {
- return credentials.NewStaticCredentials(qs.AccessKey, qs.SecretAccessKey, ""), nil
+ return credentials.NewStaticCredentials(qs.AccessKey, qs.SecretAccessKey, qs.SessionToken), nil
  }
  endpoint := qs.Endpoint
  // if endpoint is empty,return no error and run default(aws) follow.
@@ -330,6 +332,7 @@ func NewS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
  // Clear the credentials if exists so that they will not be sent to TiKV
  backend.AccessKey = ""
  backend.SecretAccessKey = ""
+ backend.SessionToken = ""
  } else if ses.Config.Credentials != nil {
  if qs.AccessKey == "" || qs.SecretAccessKey == "" {
  v, cerr := ses.Config.Credentials.Get()
@@ -338,6 +341,7 @@ func NewS3Storage(backend *backuppb.S3, opts *ExternalStorageOptions) (obj *S3St
  }
  backend.AccessKey = v.AccessKeyID
  backend.SecretAccessKey = v.SecretAccessKey
+ backend.SessionToken = v.SessionToken
  }
  }
 

br/pkg/storage/s3_test.go

@@ -144,6 +144,7 @@ func TestApplyUpdate(t *testing.T) {
  if test.setEnv {
  require.NoError(t, os.Setenv("AWS_ACCESS_KEY_ID", "ab"))
  require.NoError(t, os.Setenv("AWS_SECRET_ACCESS_KEY", "cd"))
+ require.NoError(t, os.Setenv("AWS_SESSION_TOKEN", "ef"))
  }
  u, err := ParseBackend("s3://bucket/prefix/", &BackendOptions{S3: test.options})
  require.NoError(t, err)
@@ -260,11 +261,13 @@ func TestApplyUpdate(t *testing.T) {
  Region: "us-west-2",
  AccessKey: "ab",
  SecretAccessKey: "cd",
+ SessionToken: "ef",
  },
  s3: &backuppb.S3{
  Region: "us-west-2",
  AccessKey: "ab",
  SecretAccessKey: "cd",
+ SessionToken: "ef",
  Bucket: "bucket",
  Prefix: "prefix",
  },
@@ -354,6 +357,7 @@ func TestS3Storage(t *testing.T) {
  Endpoint: s.URL,
  AccessKey: "ab",
  SecretAccessKey: "cd",
+ SessionToken: "ef",
  Bucket: "bucket",
  Prefix: "prefix",
  ForcePathStyle: true,
@@ -1112,10 +1116,12 @@ func TestWalkDirWithEmptyPrefix(t *testing.T) {
 func TestSendCreds(t *testing.T) {
  accessKey := "ab"
  secretAccessKey := "cd"
+ sessionToken := "ef"
  backendOpt := BackendOptions{
  S3: S3BackendOptions{
  AccessKey: accessKey,
  SecretAccessKey: secretAccessKey,
+ SessionToken: sessionToken,
  },
  }
  backend, err := ParseBackend("s3://bucket/prefix/", &backendOpt)
@@ -1128,12 +1134,15 @@ func TestSendCreds(t *testing.T) {
  sentAccessKey := backend.GetS3().AccessKey
  require.Equal(t, accessKey, sentAccessKey)
  sentSecretAccessKey := backend.GetS3().SecretAccessKey
- require.Equal(t, sentSecretAccessKey, sentSecretAccessKey)
+ require.Equal(t, secretAccessKey, sentSecretAccessKey)
+ sentSessionToken := backend.GetS3().SessionToken
+ require.Equal(t, sessionToken, sentSessionToken)
 
  backendOpt = BackendOptions{
  S3: S3BackendOptions{
  AccessKey: accessKey,
  SecretAccessKey: secretAccessKey,
+ SessionToken: sessionToken,
  },
  }
  backend, err = ParseBackend("s3://bucket/prefix/", &backendOpt)
@@ -1147,6 +1156,8 @@ func TestSendCreds(t *testing.T) {
  require.Equal(t, "", sentAccessKey)
  sentSecretAccessKey = backend.GetS3().SecretAccessKey
  require.Equal(t, "", sentSecretAccessKey)
+ sentSessionToken = backend.GetS3().SessionToken
+ require.Equal(t, "", sentSessionToken)
 }
 
 func TestObjectLock(t *testing.T) {