diff --git a/br/pkg/logutil/logging.go b/br/pkg/logutil/logging.go index 1727325b84d84..525227f9f2b38 100644 --- a/br/pkg/logutil/logging.go +++ b/br/pkg/logutil/logging.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "regexp" "strings" "github.com/google/uuid" @@ -20,6 +21,15 @@ import ( "go.uber.org/zap/zapcore" ) +var ( + reAccessKey = regexp.MustCompile(`access_key:\"[^\"]*\"`) + reSecretAccessKey = regexp.MustCompile(`secret_access_key:\"[^\"]*\"`) + reSharedKey = regexp.MustCompile(`shared_key:\"[^\"]*\"`) + reCredentialsBlob = regexp.MustCompile(`credentials_blob:\"[^\"]*\"`) + reAccessSig = regexp.MustCompile(`access_sig:\"[^\"]*\"`) + reEncryptKey = regexp.MustCompile(`encryption_key:<.*?>`) +) + // AbbreviatedArrayMarshaler abbreviates an array of elements. type AbbreviatedArrayMarshaler []string @@ -340,3 +350,25 @@ func (b HexBytes) String() string { func (b HexBytes) MarshalJSON() ([]byte, error) { return json.Marshal(hex.EncodeToString(b)) } + +// TaskInfoRedacted is a wrapper of backup.StreamBackupTaskInfo to redact sensitive information +type TaskInfoRedacted struct { + Info *backuppb.StreamBackupTaskInfo +} + +func (TaskInfoRedacted) redact(input string) string { + // Replace the matched fields with redacted versions + output := reAccessKey.ReplaceAllString(input, `access_key:"[REDACTED]"`) + output = reSecretAccessKey.ReplaceAllString(output, `secret_access_key:"[REDACTED]"`) + output = reSharedKey.ReplaceAllString(output, `shared_key:"[REDACTED]"`) + output = reCredentialsBlob.ReplaceAllString(output, `CredentialsBlob:"[REDACTED]"`) + output = reAccessSig.ReplaceAllString(output, `access_sig:"[REDACTED]"`) + output = reEncryptKey.ReplaceAllString(output, `encryption_key:<[REDACTED]>`) + + return output +} + +// String returns the redacted string of the task info +func (t TaskInfoRedacted) String() string { + return t.redact(t.Info.String()) +} diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel index b08caa57d7f6f..ef4ca75797449 100644 --- a/br/pkg/streamhelper/BUILD.bazel +++ b/br/pkg/streamhelper/BUILD.bazel @@ -68,7 +68,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 32, + shard_count = 33, deps = [ ":streamhelper", "//br/pkg/errors", diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go index d82a88233833b..0222ecd9570ac 100644 --- a/br/pkg/streamhelper/advancer.go +++ b/br/pkg/streamhelper/advancer.go @@ -437,7 +437,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error if err != nil { log.Warn("failed to upload service GC safepoint, skipping.", logutil.ShortError(err)) } - log.Info("added event", zap.Stringer("task", e.Info), + log.Info("added event", zap.Stringer("task", logutil.TaskInfoRedacted{Info: e.Info}), zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange)), zap.Uint64("current-checkpoint", p)) case EventDel: utils.LogBackupTaskCountDec() diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go index 58deb6301afb8..dc7988524dbb1 100644 --- a/br/pkg/streamhelper/advancer_test.go +++ b/br/pkg/streamhelper/advancer_test.go @@ -15,6 +15,7 @@ import ( backup "github.com/pingcap/kvproto/pkg/brpb" logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/streamhelper" "github.com/pingcap/tidb/br/pkg/streamhelper/config" "github.com/pingcap/tidb/br/pkg/streamhelper/spans" @@ -824,3 +825,48 @@ func TestSubscriptionPanic(t *testing.T) { cancel() wg.Wait() } + +func TestRedactBackend(t *testing.T) { + info := new(backup.StreamBackupTaskInfo) + info.Name = "test" + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_S3{ + S3: &backup.S3{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + AccessKey: "12abCD!@#[]{}?/\\", + SecretAccessKey: "12abCD!@#[]{}?/\\", + }, + }, + } + + redacted := logutil.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) + + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_Gcs{ + Gcs: &backup.GCS{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + CredentialsBlob: "12abCD!@#[]{}?/\\", + }, + }, + } + redacted = logutil.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) + + info.Storage = &backup.StorageBackend{ + Backend: &backup.StorageBackend_AzureBlobStorage{ + AzureBlobStorage: &backup.AzureBlobStorage{ + Endpoint: "http://", + Bucket: "test", + Prefix: "test", + SharedKey: "12abCD!@#[]{}?/\\", + }, + }, + } + redacted = logutil.TaskInfoRedacted{Info: info} + require.Equal(t, "storage: > name:\"test\" ", redacted.String()) +}