From 0ed76120d6e0d29b7dc1625ac573da0dad5ba4e8 Mon Sep 17 00:00:00 2001 From: kl7sn Date: Wed, 27 Sep 2023 14:51:14 +0800 Subject: [PATCH 1/3] test: slog put Signed-off-by: kl7sn --- aws.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws.go b/aws.go index 09a3dd9..10e4dc4 100644 --- a/aws.go +++ b/aws.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "log/slog" "strings" "time" @@ -296,6 +297,7 @@ func (a *S3) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map } encoding := a.compressor.ContentEncoding() input.ContentEncoding = &encoding + slog.Debug("compress", slog.String("key", key), slog.String("compressor", encoding)) } } From f80bd420242dcd15b8423e77d443dbc8098313f3 Mon Sep 17 00:00:00 2001 From: kl7sn Date: Wed, 27 Sep 2023 18:59:06 +0800 Subject: [PATCH 2/3] fix: seek Signed-off-by: kl7sn --- aws.go | 15 +++++-------- aws_test.go | 57 +++++++++++++++++++++++++----------------------- client.go | 2 ++ compress.go | 35 +++++++++++++++++++++++------ compress_test.go | 4 ++-- oss.go | 2 +- 6 files changed, 69 insertions(+), 46 deletions(-) diff --git a/aws.go b/aws.go index 10e4dc4..7c8f8e9 100644 --- a/aws.go +++ b/aws.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "log/slog" "strings" "time" @@ -216,7 +215,7 @@ func (a *S3) GetAndDecompress(ctx context.Context, key string) (string, error) { return "", errors.New("GetAndDecompress only supports snappy for now, got " + *compressor) } - rawBytes, err := ioutil.ReadAll(body) + rawBytes, err := io.ReadAll(body) if err != nil { return "", err } @@ -225,7 +224,7 @@ func (a *S3) GetAndDecompress(ctx context.Context, key string) (string, error) { if err != nil { if errors.Is(err, snappy.ErrCorrupt) { reader := snappy.NewReader(bytes.NewReader(rawBytes)) - data, err := ioutil.ReadAll(reader) + data, err := io.ReadAll(reader) if err != nil { return "", err } @@ -259,12 +258,10 @@ func (a *S3) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map if err != nil { return err } - putOptions := DefaultPutOptions() for _, opt := range options { opt(putOptions) } - input := &s3.PutObjectInput{ Body: reader, Bucket: aws.String(bucketName), @@ -284,20 +281,20 @@ func (a *S3) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map if putOptions.expires != nil { input.Expires = putOptions.expires } - if a.compressor != nil { - l, err := GetReaderLength(input.Body) + wrapReader, l, err := WrapReader(input.Body) if err != nil { return err } if l > a.cfg.CompressLimit { - input.Body, err = a.compressor.Compress(input.Body) + input.Body, _, err = a.compressor.Compress(wrapReader) if err != nil { return err } encoding := a.compressor.ContentEncoding() input.ContentEncoding = &encoding - slog.Debug("compress", slog.String("key", key), slog.String("compressor", encoding)) + // input.SetContentLength(clen) + // meta["length"] = fmt.Sprintf("%d", clen) } } diff --git a/aws_test.go b/aws_test.go index 0f1d94c..217a787 100644 --- a/aws_test.go +++ b/aws_test.go @@ -16,8 +16,8 @@ import ( ) const ( - S3Guid = "test123" - S3Content = "aaaaaa" + S3Guid = "myguid-gzip" + S3Content = `[[20,"修复流程","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"evEo\"|32:1|direction:\"ltr\""],[20,"修复前，提前叫米哈游可以的话做一下文档副本备份","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"oP2N\"|direction:\"ltr\""],[20,"\n","24:\"isMR\"|direction:\"ltr\""],[20,"先确认线上环境SDK版本！！！","26:\"89312954\"|8:1|inline-dir:\"ltr\""],[20,"\n","24:\"j3PH\"|direction:\"ltr\""],[20,"对应的版本号：","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"OMek\"|direction:\"ltr\""],[20,"sdk-3.8"],[20,"\n","24:\"rFlw\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"svc-sdk-entrypoint: "],[20,"1f5ec12","26:\"89312954\""],[20,"\n","24:\"lTX7\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"svc-history: 73aa408"],[20,"\n","24:\"DM9n\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"ulc9\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"sdk-3.12"],[20,"\n","24:\"Ug4O\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"svc-sdk-entrypoint: 093f1b8"],[20,"\n","24:\"mKaf\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"svc-history: 28431d2"],[20,"\n","24:\"JqvD\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"pSgu\"|direction:\"ltr\""],[20,"如果脚本执行有问题，代码在：","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"cru6\"|direction:\"ltr\""],[20,"svc-sdk-entrypoint: cmd/sdktools/internal/commands/user_dup_fix.go"],[20,"\n","24:\"2Dns\"|36:150|41:\"89312954\"|direction:\"ltr\""],[20,"svc-chistory: cmd/cmd.go runFixTool()方法"],[20,"\n","24:\"pRNF\"|36:150|41:\"89312954\"|direction:\"ltr\""],[20,"\n","24:\"9Jx8\"|direction:\"ltr\""],[20,"0 备份 endpoint_users 表","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"G9Al\"|32:2|direction:\"ltr\""],[20,"备份表（只有一张","26:\"69036605\"|inline-dir:\"ltr\""],[20,"表","26:\"67194766\"|inline-dir:\"ltr\""],[20,"）","26:\"69036605\"|inline-dir:\"ltr\""],[20,"\n","24:\"31k6\"|direction:\"ltr\""],[20,"/data/pkg/mysqldump  -h$mysql_host -u$mysql_user -p$mysql_passwd -P$mysql_port --set-gtid-purged=OFF --default-character-set=utf8mb4 --opt -c --flush-logs --single-transaction --master-data=2 --max_allowed_packet=1G  $db $table > table.sql"],[20,"\n","24:\"gQfd\"|36:177|41:\"69036605\"|direction:\"ltr\""],[20,"如果提示没开Binlogging","26:\"69036605\"|inline-dir:\"ltr\""],[20,"\n","24:\"LjIJ\"|direction:\"ltr\""],[20,"/data/pkg/mysqldump  -h$mysql_host -u$mysql_user -p$mysql_passwd -P$mysql_port --set-gtid-purged=OFF --default-character-set=utf8mb4 --master-data=2 --max_allowed_packet=1G  $db $table > table.sql"],[20,"\n","24:\"DvIB\"|36:177|41:\"69036605\"|direction:\"ltr\""],[20,"\n","24:\"n9Ze\"|direction:\"ltr\""],[20,"1 获取重复ID用户","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"C0K4\"|32:2|direction:\"ltr\""],[20,"svc-sdk-entrypoint","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"容器中，执行","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"GpF8\"|direction:\"ltr\""],[20,"./sdk-tools user-dup-find --dsn 'sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local'"],[20,"\n","24:\"K9te\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"获取重复","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"id","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"的用户","26:\"89312954\"|inline-dir:\"ltr\""],[20,"，返回 JSON，需要米哈游确认脏数据","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"4rLX\"|direction:\"ltr\""],[20,"\n","24:\"7GcX\"|direction:\"ltr\""],[20,"-------------+----------------------------------+--------------------+---------------------+--------+"],[20,"\n","24:\"xBxz\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"| 10000000040 | KwMj4gjAqCxv8SOU5uBUp5Vw5xww6a3z | plat_qa_test01 | 2023-06-17 07:50:34 |      1 |"],[20,"\n","24:\"nyNs\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"| 10000000041 | KwMj4gjAqCxv8SOU5uBUp5Vw5xww6a3z | Plat_qa_test01     | 2022-05-17 12:37:50 |      1 |"],[20,"\n","24:\"fpqE\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"| 10000000042 | KwMj4gjAqCxv8SOU5uBUp5Vw5xww6a3z | yufan.yang         | 2022-08-29 11:36:09 |      1 |"],[20,"\n","24:\"dpqR\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"| 10000000043 | KwMj4gjAqCxv8SOU5uBUp5Vw5xww6a3z | Yufan.yang | 2023-09-26 06:16:57 |     -1 |"],[20,"\n","24:\"3j60\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"uadJ\"|direction:\"ltr\""],[20,"2 删除重复用户","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"WdCO\"|32:2|direction:\"ltr\""],[20,"假如查询到以上 4 条数据，米哈游确认我们要删除的是 ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"10000000041，10000000043","26:\"67194766\""],[20,"\n","24:\"sIb8\"|direction:\"ltr\""],[20,"svc-sdk-entrypoint容器中，执行","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"c3eL\"|direction:\"ltr\""],[20,"潜在风险：可能没有修改表字段类型权限","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"jbEU\"|blockquote:true|direction:\"ltr\""],[20,"./sdk-tools user-dup-fix --dsn 'sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local' --del-uids 10000000041 --del-uids 10000000043"],[20,"\n","24:\"q402\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"删除重复用户，并修改表结构，将","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"general_bin","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"改成","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"general_ci","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"gnzd\"|direction:\"ltr\""],[20,"\n","24:\"MsLQ\"|direction:\"ltr\""],[20,"3 确认表 collate","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"YGf2\"|32:2|direction:\"ltr\""],[20,"查看表的","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"collate：","26:\"89312954\"|inline-dir:\"ltr\""],[20,"（确认表结构是否修改成功）","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"iKAb\"|direction:\"ltr\""],[20,"select table_name, table_collation from information_schema.tables where table_schema = 'svc_sdk_entrypoint' and table_name = 'endpoint_users';"],[20,"\n","24:\"dWXw\"|36:150|41:\"89312954\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"6dYf\"|direction:\"ltr\""],[20,"4 获取影响文件 GUID","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"0s53\"|32:2|direction:\"ltr\""],[20,"找到被删除用户相关的 guid，这里使用 ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"10000000041，10000000043","26:\"67194766\""],[20,"\n","24:\"PraN\"|direction:\"ltr\""],[20,"svc-history容器中，执行","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"CmyO\"|direction:\"ltr\""],[20,"./svc-history fixTool --uids 10000000041,10000000043 --queryNum 10000"],[20,"\n","24:\"Rcqi\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"获取这些用户影响的文件id","26:\"89312954\"|inline-dir:\"ltr\""],[20,"（关注查询过程中是否有性能问题，如果有从库，最好使用从库进行查询）该操作会读取全部数据，每次查询 queryNum 数量，这是 like 查询","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"lhdI\"|direction:\"ltr\""],[20,"\n","24:\"1AgI\"|direction:\"ltr\""],[20,"获取到的数据如下","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"fUoL\"|direction:\"ltr\""],[20,"["],[20,"\n","24:\"gYje\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"    {"],[20,"\n","24:\"44RK\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"        \"10000000041\": \"B2xGm0G1W1RmHZAU\""],[20,"\n","24:\"tKgx\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"    },"],[20,"\n","24:\"L7bi\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"    {"],[20,"\n","24:\"LrTf\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"        \"10000000041\": \"iurbdIOwM8t6ex5J\""],[20,"\n","24:\"EUif\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"    }"],[20,"\n","24:\"3oUn\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"]"],[20,"\n","24:\"QyyK\"|36:63|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"SV43\"|direction:\"ltr\""],[20,"5 修复 OSS 数据","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"taHP\"|32:2|direction:\"ltr\""],[20,"进入 ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"ee-tools","26:\"89312954\"|inline-dir:\"ltr\""],[20," ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"容器","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"IyFW\"|direction:\"ltr\""],[20,"\n","24:\"vZv9\"|direction:\"ltr\""],[20,"修改确认配置文件 /data/config/default.json ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"hx3o\"|direction:\"ltr\""],[20,{"gallery":"https://uploader.shimo.im/f/RiNoQ25U1VqRvM9L.png!thumbnail"},"26:\"89312954\"|29:0|30:0|3:\"1664\"|4:\"auto\"|crop:\"\"|frame:\"none\"|line-inline:\"GNTE\"|ori-height:\"1234\"|ori-width:\"1664\""],[20,"\n","24:\"wIVM\"|direction:\"ltr\""],[20,"\n","24:\"otMB\"|direction:\"ltr\""],[20,"脚本执行前，会在oss中生成备份的file-content，名称为 {guid}-bakcup-{timestamp}","26:\"89312954\"|inline-dir:\"ltr\""],[20,"\n","24:\"O04E\"|direction:\"ltr\""],[20,"\n","24:\"s1dM\"|direction:\"ltr\""],[20,"修改成功后，逐个文档执行 ","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"cNqB\"|direction:\"ltr\""],[20,"\n","24:\"3JEN\"|direction:\"ltr\""],[20,"node sdk_fix_modoc_content_uid.js 文档guid 错误uid 正确uid","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"4alp\"|direction:\"ltr\""],[20,"\n","24:\"FKDr\"|blockquote:true|direction:\"ltr\""],[20,"该脚本如果执行异常需要王珂珂协助","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"Ut5C\"|blockquote:true|direction:\"ltr\""],[20,"node sdk_fix_modoc_content_uid.js B2xGm0G1W1RmHZAU 10000000041 10000000042"],[20,"\n","24:\"jL7D\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"node sdk_fix_modoc_content_uid.js iurbdIOwM8t6ex5J 10000000041 10000000042"],[20,"\n","24:\"IgyN\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"NJnG\"|direction:\"ltr\""],[20,"\n","24:\"5oCq\"|direction:\"ltr\""],[20,"其他","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"y1on\"|32:1|direction:\"ltr\""],[20,"\n","24:\"vM6I\"|direction:\"ltr\""],[20,"获取客户方文件 ID","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"nlQT\"|32:2|direction:\"ltr\""],[20,"如果客户需要获取我们修改了哪些文件，可以 svc-sdk-entrypoint容器中，执行 --guids 参数内容来自修复 流程第4æ­¥","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"xCNo\"|direction:\"ltr\""],[20,"./sdk-tools user-dup-files --dsn 'sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local' --guids B2xGm0G1W1RmHZAU --guids iurbdIOwM8t6ex5J"],[20,"\n","24:\"EjYv\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"XpWw\"|direction:\"ltr\""],[20,"\n","24:\"4mCo\"|direction:\"ltr\""],[20,"2 3 步骤内部具体流程","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"Tw4C\"|direction:\"ltr\""],[20,"SELECT DISTINCT a.*"],[20,"\n","24:\"3eZ7\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"FROM endpoint_users a"],[20,"\n","24:\"RZh6\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"JOIN endpoint_users b ON a.provider_user_id COLLATE utf8mb4_general_ci = b.provider_user_id COLLATE utf8mb4_general_ci"],[20,"\n","24:\"j1Q8\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"WHERE a.provider_user_id <> b.provider_user_id;"],[20,"\n","24:\"Klmn\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"n6CP\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"v8P0\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"DELETE FROM endpoint_users WHERE id IN (10000002247, 10000005364, 10000005365, 10000005366, 10000004991);"],[20,"\n","24:\"vtgK\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"7T0y\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"ALTER TABLE endpoint_users "],[20,"\n","24:\"Zk8j\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"MODIFY COLUMN app_id varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL AFTER id,"],[20,"\n","24:\"qGap\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"MODIFY COLUMN provider_user_id varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL AFTER app_id,"],[20,"\n","24:\"il9E\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"COLLATE = utf8mb4_general_ci;"],[20,"\n","24:\"ErMC\"|36:150|41:\"67194766\"|direction:\"ltr\""],[20,"\n","24:\"pKYp\""],[20,"dsn：","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"hufd\"|direction:\"ltr\""],[20,"sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"MYfv\"|direction:\"ltr\""],[20,"\n","24:\"Ehvz\""],[20,"删除重复数据、修改表结构","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"uwII\"|direction:\"ltr\""],[20,"./sdk-tools user-dup-find --dsn 'sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local'"],[20,"\n","24:\"LeA1\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"yca7\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"./sdk-tools user-dup-fix --dsn 'sm_mysql:mysql_Aa123456.@(mysql-master)/svc_sdk_entrypoint?charset=utf8mb4&parseTime=True&loc=Local' --del-uids 10000000041 --del-uids 10000000043"],[20,"\n","24:\"rbVb\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"pzBd\""],[20,"\n","24:\"MUSU\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"./svc-history fixTool --uids 10000002247,10000005364,10000005365,10000005366,10000004991 --queryNum 10000"],[20,"\n","24:\"UTpA\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"BdWk\""],[20,"// 客户方的文章id","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"s0yp\"|direction:\"ltr\""],[20,"select provider_file_id as mhy_id, history_guid as shimo_id from endpoint_files where history_guid in ('B2xGm0G1W1RmHZAU', 'iurbdIOwM8t6ex5J', 'fw0NFg3JHtWmRC6H', 'GhwkDsr6rLqAFf8g', 'u3FatjfeGNGPBikT', 'TflrbD9Vuemfy0Pc', 'V8g9EET4ZUVyH9cF');","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"AwYT\"|direction:\"ltr\""],[20,"\n","24:\"KiQr\""],[20,"\n","24:\"kone\""],[20,"// 修复file-content","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"7L6t\"|direction:\"ltr\""],[20,"node sdk_fix_modoc_content_uid.js u3FatjfeGNGPBikT 10000005364 10000004525","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"b8lq\"|direction:\"ltr\""],[20,"node sdk_fix_modoc_content_uid.js u3FatjfeGNGPBikT 10000005365 10000005321","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"EcJa\"|direction:\"ltr\""],[20,"\n","24:\"Oamp\""],[20,"--------+------------------+","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"ybvS\"|direction:\"ltr\""],[20,"| mhy_id | shimo_id         |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"gDk0\"|direction:\"ltr\""],[20,"+--------+------------------+","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"7CU6\"|direction:\"ltr\""],[20,"| 178001 | B2xGm0G1W1RmHZAU |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"R1il\"|direction:\"ltr\""],[20,"| 559750 | GhwkDsr6rLqAFf8g |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"899T\"|direction:\"ltr\""],[20,"| 500181 | TflrbD9Vuemfy0Pc |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"pQgX\"|direction:\"ltr\""],[20,"| 345626 | V8g9EET4ZUVyH9cF |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"wjXP\"|direction:\"ltr\""],[20,"| 499866 | fw0NFg3JHtWmRC6H |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"B5ZD\"|direction:\"ltr\""],[20,"| 272286 | iurbdIOwM8t6ex5J |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"cuXg\"|direction:\"ltr\""],[20,"| 572691 | u3FatjfeGNGPBikT |","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"HYQd\"|direction:\"ltr\""],[20,"\n","24:\"H567\""],[20,"private-toolbox 配置文件存档","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"Ri9J\"|direction:\"ltr\""],[20,"mhyçš„minio为minio-new-service","26:\"67194766\"|inline-dir:\"ltr\""],[20,"\n","24:\"rFC1\"|direction:\"ltr\""],[20,"\n","24:\"UWfy\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"{"],[20,"\n","24:\"CmEl\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  \"mysql\": {"],[20,"\n","24:\"4yT8\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"host\": \"mysql-master\","],[20,"\n","24:\"qrvf\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"user\": \"sm_mysql\","],[20,"\n","24:\"n8dl\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"password\": \"mysql_Aa123456.\","],[20,"\n","24:\"jq2r\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"port\": 3306"],[20,"\n","24:\"h0K8\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  },"],[20,"\n","24:\"7XO0\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  \"redis\": {"],[20,"\n","24:\"AAQq\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"host\": \"\","],[20,"\n","24:\"O7tL\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"port\": 6379,"],[20,"\n","24:\"v4M9\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"password\": \"\""],[20,"\n","24:\"NsLL\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  },"],[20,"\n","24:\"kpld\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  \"mongo\": {"],[20,"\n","24:\"ikh0\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"docHistory\": \"mongodb://your_mongo_user:your_mongo_password@server_host:3717/doc_history\""],[20,"\n","24:\"63Xp\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  },"],[20,"\n","24:\"jLFl\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  \"storage\": {"],[20,"\n","24:\"gCRK\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"type\": \"aws\","],[20,"\n","24:\"C5YJ\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"aws\": {"],[20,"\n","24:\"VEMk\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"accessKeyId\": \"minio-shimo\","],[20,"\n","24:\"iBm5\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"secretAccessKey\": \"minio-shimo2019\","],[20,"\n","24:\"OqHV\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"region\": \"cn-north-1\","],[20,"\n","24:\"Ufqj\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"endpoint\": \"http://minio-service:9000\","],[20,"\n","24:\"1Hqy\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"s3ForcePathStyle\": true"],[20,"\n","24:\"Nrdt\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    },"],[20,"\n","24:\"XfRI\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"oss\": {},"],[20,"\n","24:\"xIZF\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    \"buckets\": {"],[20,"\n","24:\"ElT3\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"fileContent\": \"file-contents\","],[20,"\n","24:\"KTjX\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"fileSnapshots\": \"file-snapshots\","],[20,"\n","24:\"Ht1G\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"sheetHistories\": \"sheet-histories\","],[20,"\n","24:\"uKrm\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"docHistory\": \"svc-doc-history\","],[20,"\n","24:\"WygN\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"      \"composePayloads\": \"compose-payloads\""],[20,"\n","24:\"HjLK\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"    }"],[20,"\n","24:\"cKEp\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"  }"],[20,"\n","24:\"LBO3\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"}"],[20,"\n","24:\"jUEN\"|36:150|41:\"67194766\"|42:\"true\"|direction:\"ltr\""],[20,"\n","24:\"7LuJ\""],[20,"\n","24:\"NrtP\"|direction:\"ltr\""]]` S3ExpectLength = 6 S3ExpectHead = 1 @@ -40,10 +40,13 @@ accessKeyID = "%s" accessKeySecret = "%s" endpoint = "%s" bucket = "%s" -s3ForcePathStyle = false +s3ForcePathStyle = true region = "%s" ssl = false -shards = ["stuvwxyz0123456789", "abcdefghijklmnopqr"] +shards = [] +enableCompressor = true +compressType = "gzip" +compressLimit = 0 ` confs = fmt.Sprintf(confs, os.Getenv("AK_ID"), os.Getenv("AK_SECRET"), os.Getenv("ENDPOINT"), os.Getenv("BUCKET"), os.Getenv("REGION")) if err := econf.LoadFromReader(strings.NewReader(confs), toml.Unmarshal); err != nil { @@ -87,9 +90,30 @@ func TestS3_Put(t *testing.T) { t.Fail() } - err = awsCmp.Put(ctx, S3Guid, bytes.NewReader([]byte(S3Content)), meta) + // err = awsCmp.Put(ctx, S3Guid, bytes.NewReader([]byte(S3Content)), meta) + // if err != nil { + // t.Log("aws put error", err) + // t.Fail() + // } +} + +func TestS3_GetWithMeta(t *testing.T) { + ctx := context.TODO() + attributes := make([]string, 0) + attributes = append(attributes, "head") + res, meta, err := awsCmp.GetWithMeta(ctx, S3Guid, attributes) if err != nil { - t.Log("aws put error", err) + t.Fatal("aws get content as reader fail, err:", err) + } + defer res.Close() + byteRes, _ := ioutil.ReadAll(res) + if string(byteRes) != S3Content { + t.Fatal("aws get as reader, readAll error") + } + + head, err := strconv.Atoi(meta["head"]) + if err != nil || head != S3ExpectHead { + t.Log("aws get head fail, res:", res, "err:", err) t.Fail() } } @@ -170,27 +194,6 @@ func TestS3_Get(t *testing.T) { } } -func TestS3_GetWithMeta(t *testing.T) { - ctx := context.TODO() - attributes := make([]string, 0) - attributes = append(attributes, "head") - res, meta, err := awsCmp.GetWithMeta(ctx, S3Guid, attributes) - if err != nil { - t.Fatal("aws get content as reader fail, err:", err) - } - defer res.Close() - byteRes, _ := ioutil.ReadAll(res) - if string(byteRes) != S3Content { - t.Fatal("aws get as reader, readAll error") - } - - head, err := strconv.Atoi(meta["head"]) - if err != nil || head != S3ExpectHead { - t.Log("aws get head fail, res:", res, "err:", err) - t.Fail() - } -} - // compressed content func TestS3_GetAndDecompress(t *testing.T) { ctx := context.TODO() diff --git a/client.go b/client.go index c17b4d0..b0b5680 100644 --- a/client.go +++ b/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "log/slog" "net/http" "net/http/httptrace" "strings" @@ -111,6 +112,7 @@ func newStorage(name string, cfg *BucketConfig, logger *elog.Component) (Client, } if cfg.Debug { config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody | aws.LogDebugWithSigning) + slog.Default().Enabled(context.Background(), slog.LevelDebug) } config.HTTPClient = &http.Client{ diff --git a/compress.go b/compress.go index 9935aed..20da861 100644 --- a/compress.go +++ b/compress.go @@ -20,19 +20,34 @@ func Register(comp Compressor) { } type Compressor interface { - Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, err error) + // Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, err error) + Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, len int64, err error) ContentEncoding() string } type GzipCompressor struct { } -func (g *GzipCompressor) Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, err error) { - return &gzipReadSeeker{ - reader: reader, - }, nil +func (g *GzipCompressor) Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, len int64, err error) { + var buffer bytes.Buffer + gzipWriter := gzip.NewWriter(&buffer) + _, err = io.Copy(gzipWriter, reader) + if err != nil { + return nil, 0, err + } + err = gzipWriter.Close() + if err != nil { + return nil, 0, err + } + return bytes.NewReader(buffer.Bytes()), int64(buffer.Len()), nil } +// func (g *GzipCompressor) Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, err error) { +// return &gzipReadSeeker{ +// reader: reader, +// }, nil +// } + func (g *GzipCompressor) ContentEncoding() string { return compressTypeGzip } @@ -74,13 +89,20 @@ func (crs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) { var DefaultGzipCompressor = &GzipCompressor{} +func WrapReader(reader io.ReadSeeker) (io.ReadSeeker, int64, error) { + all, err := io.ReadAll(reader) + if err != nil { + return nil, 0, err + } + return bytes.NewReader(all), int64(len(all)), nil +} + func GetReaderLength(reader io.ReadSeeker) (int64, error) { // 保存当前的读写位置 originalPos, err := reader.Seek(0, io.SeekCurrent) if err != nil { return 0, err } - // 移动到文件末尾以获取字节长度 length, err := reader.Seek(0, io.SeekEnd) if err != nil { @@ -91,6 +113,5 @@ func GetReaderLength(reader io.ReadSeeker) (int64, error) { if err != nil { return 0, err } - return length, nil } diff --git a/compress_test.go b/compress_test.go index a380f19..ef8bd18 100644 --- a/compress_test.go +++ b/compress_test.go @@ -17,7 +17,7 @@ func TestCompress_gzip(t *testing.T) { if err != nil { panic(err) } - reader, err := DefaultGzipCompressor.Compress(source) + reader, _, err := DefaultGzipCompressor.Compress(source) if err != nil { panic(err) } @@ -62,7 +62,7 @@ func TestGetLength(t *testing.T) { } t.Logf("length %d", length) - seeker, _ := DefaultGzipCompressor.Compress(source) + seeker, _, _ := DefaultGzipCompressor.Compress(source) targetPath := os.Getenv("target_path") _, err = os.Stat(targetPath) if err == nil { diff --git a/oss.go b/oss.go index b8b09ef..7840f13 100644 --- a/oss.go +++ b/oss.go @@ -261,7 +261,7 @@ func (ossClient *OSS) Put(ctx context.Context, key string, reader io.ReadSeeker, return err } if l > ossClient.cfg.CompressLimit { - reader, err = ossClient.compressor.Compress(reader) + reader, _, err = ossClient.compressor.Compress(reader) if err != nil { return err } From 93af576be2bf20b7e1a90afc0129a7e36f5b981a Mon Sep 17 00:00:00 2001 From: kl7sn Date: Wed, 27 Sep 2023 22:37:08 +0800 Subject: [PATCH 3/3] fix: input.body set Signed-off-by: kl7sn --- aws.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws.go b/aws.go index 7c8f8e9..07be990 100644 --- a/aws.go +++ b/aws.go @@ -293,8 +293,8 @@ func (a *S3) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map } encoding := a.compressor.ContentEncoding() input.ContentEncoding = &encoding - // input.SetContentLength(clen) - // meta["length"] = fmt.Sprintf("%d", clen) + } else { + input.Body = wrapReader } }