From 7cbebd2051cf296ef873bc9c928f48d784a79da1 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 25 Nov 2020 02:36:13 -0600 Subject: [PATCH] storage/: handle special character in URL (#614) * handle special character in URL * Update tests/br_s3/run.sh Co-authored-by: kennytm * clear data * try to fix * add comment and a unit test Co-authored-by: kennytm --- pkg/storage/parse.go | 4 ++ pkg/storage/parse_test.go | 10 ++++ tests/br_s3/run.sh | 119 ++++++++++++++++++++------------------ 3 files changed, 78 insertions(+), 55 deletions(-) diff --git a/pkg/storage/parse.go b/pkg/storage/parse.go index 5a7a09601..06fa6713c 100644 --- a/pkg/storage/parse.go +++ b/pkg/storage/parse.go @@ -29,6 +29,10 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "empty store is not allowed") } + // https://github.com/pingcap/br/issues/603 + // In aws the secret key may contain '/+=' and '+' has a special meaning in URL. + // Replace "+" by "%2B" here to avoid this problem. + rawURL = strings.ReplaceAll(rawURL, "+", "%2B") u, err := url.Parse(rawURL) if err != nil { return nil, errors.Trace(err) diff --git a/pkg/storage/parse_test.go b/pkg/storage/parse_test.go index 34d6537b7..6fe38695d 100644 --- a/pkg/storage/parse_test.go +++ b/pkg/storage/parse_test.go @@ -67,6 +67,16 @@ func (r *testStorageSuite) TestCreateStorage(c *C) { c.Assert(s3.Sse, Equals, "aws:kms") c.Assert(s3.SseKmsKeyId, Equals, "TestKey") + // special character in access keys + s, err = ParseBackend(`s3://bucket4/prefix/path?access-key=NXN7IPIOSAAKDEEOLMAF&secret-access-key=nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw`, nil) + c.Assert(err, IsNil) + s3 = s.GetS3() + c.Assert(s3, NotNil) + c.Assert(s3.Bucket, Equals, "bucket4") + c.Assert(s3.Prefix, Equals, "prefix/path") + c.Assert(s3.AccessKey, Equals, "NXN7IPIOSAAKDEEOLMAF") + c.Assert(s3.SecretAccessKey, Equals, "nREY/7Dt+PaIbYKrKlEEMMF/ExCiJEX=XMLPUANw") + gcsOpt := &BackendOptions{ GCS: GCSBackendOptions{ Endpoint: "https://gcs.example.com/", diff --git a/tests/br_s3/run.sh b/tests/br_s3/run.sh index 8b2cdace0..e2ed29cc6 100755 --- a/tests/br_s3/run.sh +++ b/tests/br_s3/run.sh @@ -19,8 +19,8 @@ TABLE="usertable" DB_COUNT=3 # start the s3 server -export MINIO_ACCESS_KEY=brs3accesskey -export MINIO_SECRET_KEY=brs3secretkey +export MINIO_ACCESS_KEY='KEXI7MANNASOPDLAOIEF' +export MINIO_SECRET_KEY='MaKYxEGDInMPtEYECXRJLU+FPNKb/wAX/MElir7E' export MINIO_BROWSER=off export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY @@ -44,73 +44,82 @@ stop_minio() { } trap stop_minio EXIT -s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://mybucket - # Fill in the database for i in $(seq $DB_COUNT); do run_sql "CREATE DATABASE $DB${i};" go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i} done +S3_KEY="" +for p in $(seq 2); do + s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://mybucket -for i in $(seq $DB_COUNT); do - row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') -done + for i in $(seq $DB_COUNT); do + row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') + done -# backup full -echo "backup start..." -BACKUP_LOG="backup.log" -rm -f $BACKUP_LOG -unset BR_LOG_TO_TERM -run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT" \ - --log-file $BACKUP_LOG || \ - ( cat $BACKUP_LOG && BR_LOG_TO_TERM=1 && exit 1 ) -cat $BACKUP_LOG -BR_LOG_TO_TERM=1 + # backup full + echo "backup start..." + BACKUP_LOG="backup.log" + rm -f $BACKUP_LOG + unset BR_LOG_TO_TERM + run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \ + --log-file $BACKUP_LOG || \ + ( cat $BACKUP_LOG && BR_LOG_TO_TERM=1 && exit 1 ) + cat $BACKUP_LOG + BR_LOG_TO_TERM=1 -if grep -i $MINIO_SECRET_KEY $BACKUP_LOG; then - echo "Secret key logged in log. Please remove them." - exit 1 -fi + if grep -i $MINIO_SECRET_KEY $BACKUP_LOG; then + echo "Secret key logged in log. Please remove them." + exit 1 + fi -for i in $(seq $DB_COUNT); do - run_sql "DROP DATABASE $DB${i};" -done + for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" + done -# restore full -echo "restore start..." -RESTORE_LOG="restore.log" -rm -f $RESTORE_LOG -unset BR_LOG_TO_TERM -run_br restore full -s "s3://mybucket/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \ - --log-file $RESTORE_LOG || \ - ( cat $RESTORE_LOG && BR_LOG_TO_TERM=1 && exit 1 ) -cat $RESTORE_LOG -BR_LOG_TO_TERM=1 + # restore full + echo "restore start..." + RESTORE_LOG="restore.log" + rm -f $RESTORE_LOG + unset BR_LOG_TO_TERM + run_br restore full -s "s3://mybucket/$DB?$S3_KEY" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \ + --log-file $RESTORE_LOG || \ + ( cat $RESTORE_LOG && BR_LOG_TO_TERM=1 && exit 1 ) + cat $RESTORE_LOG + BR_LOG_TO_TERM=1 -if grep -i $MINIO_SECRET_KEY $RESTORE_LOG; then - echo "Secret key logged in log. Please remove them." - exit 1 -fi + if grep -i $MINIO_SECRET_KEY $RESTORE_LOG; then + echo "Secret key logged in log. Please remove them." + exit 1 + fi -for i in $(seq $DB_COUNT); do - row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') -done + for i in $(seq $DB_COUNT); do + row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') + done -fail=false -for i in $(seq $DB_COUNT); do - if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then - fail=true - echo "TEST: [$TEST_NAME] fail on database $DB${i}" - fi - echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" -done + fail=false + for i in $(seq $DB_COUNT); do + if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB${i}" + fi + echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" + done + + if $fail; then + echo "TEST: [$TEST_NAME] failed!" + exit 1 + else + echo "TEST: [$TEST_NAME] successed!" + fi -if $fail; then - echo "TEST: [$TEST_NAME] failed!" - exit 1 -else - echo "TEST: [$TEST_NAME] successed!" -fi + # prepare for next test + S3_KEY="&access-key=$MINIO_ACCESS_KEY&secret-access-key=$MINIO_SECRET_KEY" + export AWS_ACCESS_KEY_ID="" + export AWS_SECRET_ACCESS_KEY="" + rm -rf "$TEST_DIR/$DB" + mkdir -p "$TEST_DIR/$DB" +done for i in $(seq $DB_COUNT); do run_sql "DROP DATABASE $DB${i};"