br: redact ak/sk in logging (#55622) (#55777)

close #55273
pingcap · Nov 11, 2024 · 3929a07 · 3929a07
1 parent 6336e2f
commit 3929a07
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 2 deletions.
diff --git a/br/pkg/logutil/logging.go b/br/pkg/logutil/logging.go
@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/google/uuid"
@@ -20,6 +21,15 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
+var (
+	reAccessKey       = regexp.MustCompile(`access_key:\"[^\"]*\"`)
+	reSecretAccessKey = regexp.MustCompile(`secret_access_key:\"[^\"]*\"`)
+	reSharedKey       = regexp.MustCompile(`shared_key:\"[^\"]*\"`)
+	reCredentialsBlob = regexp.MustCompile(`credentials_blob:\"[^\"]*\"`)
+	reAccessSig       = regexp.MustCompile(`access_sig:\"[^\"]*\"`)
+	reEncryptKey      = regexp.MustCompile(`encryption_key:<.*?>`)
+)
+
 // AbbreviatedArrayMarshaler abbreviates an array of elements.
 type AbbreviatedArrayMarshaler []string
 
@@ -340,3 +350,25 @@ func (b HexBytes) String() string {
 func (b HexBytes) MarshalJSON() ([]byte, error) {
 	return json.Marshal(hex.EncodeToString(b))
 }
+
+// TaskInfoRedacted is a wrapper of backup.StreamBackupTaskInfo to redact sensitive information
+type TaskInfoRedacted struct {
+	Info *backuppb.StreamBackupTaskInfo
+}
+
+func (TaskInfoRedacted) redact(input string) string {
+	// Replace the matched fields with redacted versions
+	output := reAccessKey.ReplaceAllString(input, `access_key:"[REDACTED]"`)
+	output = reSecretAccessKey.ReplaceAllString(output, `secret_access_key:"[REDACTED]"`)
+	output = reSharedKey.ReplaceAllString(output, `shared_key:"[REDACTED]"`)
+	output = reCredentialsBlob.ReplaceAllString(output, `CredentialsBlob:"[REDACTED]"`)
+	output = reAccessSig.ReplaceAllString(output, `access_sig:"[REDACTED]"`)
+	output = reEncryptKey.ReplaceAllString(output, `encryption_key:<[REDACTED]>`)
+
+	return output
+}
+
+// String returns the redacted string of the task info
+func (t TaskInfoRedacted) String() string {
+	return t.redact(t.Info.String())
+}
diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel
@@ -68,7 +68,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 32,
+    shard_count = 33,
     deps = [
         ":streamhelper",
         "//br/pkg/errors",

diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go
@@ -437,7 +437,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error
 		if err != nil {
 			log.Warn("failed to upload service GC safepoint, skipping.", logutil.ShortError(err))
 		}
-		log.Info("added event", zap.Stringer("task", e.Info),
+		log.Info("added event", zap.Stringer("task", logutil.TaskInfoRedacted{Info: e.Info}),
 			zap.Stringer("ranges", logutil.StringifyKeys(c.taskRange)), zap.Uint64("current-checkpoint", p))
 	case EventDel:
 		utils.LogBackupTaskCountDec()

diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go
@@ -15,6 +15,7 @@ import (
 	backup "github.com/pingcap/kvproto/pkg/brpb"
 	logbackup "github.com/pingcap/kvproto/pkg/logbackuppb"
 	"github.com/pingcap/log"
+	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/streamhelper"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/config"
 	"github.com/pingcap/tidb/br/pkg/streamhelper/spans"
@@ -824,3 +825,48 @@ func TestSubscriptionPanic(t *testing.T) {
 	cancel()
 	wg.Wait()
 }
+
+func TestRedactBackend(t *testing.T) {
+	info := new(backup.StreamBackupTaskInfo)
+	info.Name = "test"
+	info.Storage = &backup.StorageBackend{
+		Backend: &backup.StorageBackend_S3{
+			S3: &backup.S3{
+				Endpoint:        "http://",
+				Bucket:          "test",
+				Prefix:          "test",
+				AccessKey:       "12abCD!@#[]{}?/\\",
+				SecretAccessKey: "12abCD!@#[]{}?/\\",
+			},
+		},
+	}
+
+	redacted := logutil.TaskInfoRedacted{Info: info}
+	require.Equal(t, "storage:<s3:<endpoint:\"http://\" bucket:\"test\" prefix:\"test\" access_key:\"[REDACTED]\" secret_access_key:\"[REDACTED]\" > > name:\"test\" ", redacted.String())
+
+	info.Storage = &backup.StorageBackend{
+		Backend: &backup.StorageBackend_Gcs{
+			Gcs: &backup.GCS{
+				Endpoint:        "http://",
+				Bucket:          "test",
+				Prefix:          "test",
+				CredentialsBlob: "12abCD!@#[]{}?/\\",
+			},
+		},
+	}
+	redacted = logutil.TaskInfoRedacted{Info: info}
+	require.Equal(t, "storage:<gcs:<endpoint:\"http://\" bucket:\"test\" prefix:\"test\" CredentialsBlob:\"[REDACTED]\" > > name:\"test\" ", redacted.String())
+
+	info.Storage = &backup.StorageBackend{
+		Backend: &backup.StorageBackend_AzureBlobStorage{
+			AzureBlobStorage: &backup.AzureBlobStorage{
+				Endpoint:  "http://",
+				Bucket:    "test",
+				Prefix:    "test",
+				SharedKey: "12abCD!@#[]{}?/\\",
+			},
+		},
+	}
+	redacted = logutil.TaskInfoRedacted{Info: info}
+	require.Equal(t, "storage:<azure_blob_storage:<endpoint:\"http://\" bucket:\"test\" prefix:\"test\" shared_key:\"[REDACTED]\" > > name:\"test\" ", redacted.String())
+}