dumpling: compress supports snappy/zstd (#38910)

ref #38514
pingcap · Nov 14, 2022 · 78d1905 · 78d1905
1 parent a8f56ee
commit 78d1905
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 17 deletions.
diff --git a/dumpling/export/config.go b/dumpling/export/config.go
@@ -291,7 +291,7 @@ func (*Config) DefineFlags(flags *pflag.FlagSet) {
  _ = flags.MarkHidden(flagReadTimeout)
  flags.Bool(flagTransactionalConsistency, true, "Only support transactional consistency")
  _ = flags.MarkHidden(flagTransactionalConsistency)
- flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'no-compression' now")
+ flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'snappy', 'zstd', 'no-compression' now")
 }
 
 // ParseFromFlags parses dumpling's export.Config from flags
@@ -596,6 +596,10 @@ func ParseCompressType(compressType string) (storage.CompressType, error) {
  return storage.NoCompression, nil
  case "gzip", "gz":
  return storage.Gzip, nil
+ case "snappy":
+ return storage.Snappy, nil
+ case "zstd", "zst":
+ return storage.Zstd, nil
  default:
  return storage.NoCompression, errors.Errorf("unknown compress type %s", compressType)
  }

diff --git a/dumpling/export/writer_util.go b/dumpling/export/writer_util.go
@@ -591,6 +591,10 @@ func compressFileSuffix(compressType storage.CompressType) string {
  return ""
  case storage.Gzip:
  return ".gz"
+ case storage.Snappy:
+ return ".snappy"
+ case storage.Zstd:
+ return ".zst"
  default:
  return ""
  }

diff --git a/dumpling/install.sh b/dumpling/install.sh
@@ -15,3 +15,11 @@ chmod a+x bin/minio
 
 wget https://dl.minio.io/client/mc/release/linux-amd64/mc -O bin/mc
 chmod a+x bin/mc
+
+go get github.com/ma6174/snappy
+go install github.com/ma6174/snappy
+
+wget https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz
+tar xvfz zstd-1.5.2.tar.gz
+cd zstd-1.5.2
+make
diff --git a/dumpling/tests/s3/run.sh b/dumpling/tests/s3/run.sh
@@ -80,21 +80,46 @@ echo "1st records count is ${cnt}"
 # run dumpling with compress option
 export DUMPLING_OUTPUT_DIR=s3://mybucket/dump-compress
 
-run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress "gzip"
-
-mkdir -p "${HOST_DIR}/compress"
-
-bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.gz "${HOST_DIR}/compress/s3-schema-create.sql.gz"
-bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.gz "${HOST_DIR}/compress/s3.t-schema.sql.gz"
-bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.gz "${HOST_DIR}/compress/s3.t.000000000.sql.gz"
-
-gzip "${HOST_DIR}/compress/s3-schema-create.sql.gz" -d
-diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
-
-gzip "${HOST_DIR}/compress/s3.t-schema.sql.gz" -d
-diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
-
-gzip "${HOST_DIR}/compress/s3.t.000000000.sql.gz" -d
-diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+for compressType in "gzip" "snappy" "zst"
+do
+ echo -e "\nstart ${compressType} test"
+ run_dumpling --s3.endpoint="http://$S3_ENDPOINT/" --compress ${compressType}
+
+ mkdir -p "${HOST_DIR}/compress"
+
+ case $compressType in
+ "gzip")
+ suffix="gz"
+ binary=gzip
+ ;;
+
+ "snappy")
+ suffix="snappy"
+ binary=snappy
+ ;;
+
+ "zst")
+ suffix="zst"
+ binary=zstd
+ ;;
+ esac
+
+ bin/mc cp minio/mybucket/dump-compress/s3-schema-create.sql.${suffix} "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}"
+ bin/mc cp minio/mybucket/dump-compress/s3.t-schema.sql.${suffix} "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}"
+ bin/mc cp minio/mybucket/dump-compress/s3.t.000000000.sql.${suffix} "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}"
+
+ ${binary} -d "${HOST_DIR}/compress/s3-schema-create.sql.${suffix}"
+ diff "${HOST_DIR}/local/s3-schema-create.sql" "${HOST_DIR}/compress/s3-schema-create.sql"
+
+ ${binary} -d "${HOST_DIR}/compress/s3.t-schema.sql.${suffix}"
+ diff "${HOST_DIR}/local/s3.t-schema.sql" "${HOST_DIR}/compress/s3.t-schema.sql"
+
+ ${binary} -d "${HOST_DIR}/compress/s3.t.000000000.sql.${suffix}"
+ diff "${HOST_DIR}/local/s3.t.000000000.sql" "${HOST_DIR}/compress/s3.t.000000000.sql"
+
+ rm "${HOST_DIR}/compress/s3-schema-create.sql"
+ rm "${HOST_DIR}/compress/s3.t-schema.sql"
+ rm "${HOST_DIR}/compress/s3.t.000000000.sql"
+done
 
 run_sql "drop database if exists \`$DB_NAME\`;"