From 54f83eb56b4d5825fa39ac959a722bef04d8232b Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Tue, 18 Jul 2023 10:35:16 +0800 Subject: [PATCH 1/5] This is an automated cherry-pick of #45374 Signed-off-by: ti-chi-bot --- dumpling/export/ir_impl.go | 1 + dumpling/export/metadata.go | 9 ++++-- dumpling/export/writer.go | 17 ++++++++++- dumpling/export/writer_test.go | 22 ++++++++++++++ dumpling/export/writer_util.go | 20 +++++++++---- dumpling/tests/basic/run.sh | 55 ++++++++++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 10 deletions(-) diff --git a/dumpling/export/ir_impl.go b/dumpling/export/ir_impl.go index bca821f612623..3f3f7f9e416c4 100644 --- a/dumpling/export/ir_impl.go +++ b/dumpling/export/ir_impl.go @@ -353,6 +353,7 @@ func newMultiQueriesChunk(queries []string, colLength int) *multiQueriesChunk { func (td *multiQueriesChunk) Start(tctx *tcontext.Context, conn *sql.Conn) error { td.tctx = tctx td.conn = conn + td.SQLRowIter = nil return nil } diff --git a/dumpling/export/metadata.go b/dumpling/export/metadata.go index 7d2cf53128688..4adbe2f91626b 100644 --- a/dumpling/export/metadata.go +++ b/dumpling/export/metadata.go @@ -219,9 +219,12 @@ func (m *globalMetadata) writeGlobalMetaData() error { if err != nil { return err } - defer tearDown(m.tctx) - - return write(m.tctx, fileWriter, m.String()) + err = write(m.tctx, fileWriter, m.String()) + tearDownErr := tearDown(m.tctx) + if err == nil { + return tearDownErr + } + return err } func getValidStr(str []string, idx int) string { diff --git a/dumpling/export/writer.go b/dumpling/export/writer.go index e64c067c807b7..eecff89746abe 100644 --- a/dumpling/export/writer.go +++ b/dumpling/export/writer.go @@ -237,10 +237,13 @@ func (w *Writer) tryToWriteTableData(tctx *tcontext.Context, meta TableMeta, ir for { fileWriter, tearDown := buildInterceptFileWriter(tctx, w.extStorage, fileName, conf.CompressType) n, err := format.WriteInsert(tctx, conf, meta, ir, fileWriter, w.metrics) - tearDown(tctx) + tearDownErr := tearDown(tctx) if err != nil { return err } + if tearDownErr != nil { + return tearDownErr + } if w, ok := fileWriter.(*InterceptFileWriter); ok && !w.SomethingIsWritten { break @@ -275,6 +278,7 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E if err != nil { return errors.Trace(err) } +<<<<<<< HEAD defer tearDown(tctx) return WriteMeta(tctx, &metaData{ @@ -283,7 +287,18 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E specCmts: []string{ "/*!40101 SET NAMES binary*/;", }, +======= + err = WriteMeta(tctx, &metaData{ + target: target, + metaSQL: metaSQL, + specCmts: getSpecialComments(w.conf.ServerInfo.ServerType), +>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) }, fileWriter) + tearDownErr := tearDown(tctx) + if err == nil { + return tearDownErr + } + return err } type outputFileNamer struct { diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go index b5f54f3debcb9..9811706c34d89 100644 --- a/dumpling/export/writer_test.go +++ b/dumpling/export/writer_test.go @@ -12,6 +12,11 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" +<<<<<<< HEAD +======= + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/br/pkg/version" +>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) tcontext "github.com/pingcap/tidb/dumpling/context" "github.com/pingcap/tidb/util/promutil" "github.com/stretchr/testify/require" @@ -70,7 +75,17 @@ func TestWriteTableMeta(t *testing.T) { require.NoError(t, err) bytes, err := ioutil.ReadFile(p) require.NoError(t, err) +<<<<<<< HEAD require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) +======= + require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) + + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile", "return(true)")) + defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=return(true)") + + err = writer.WriteTableMeta("test", "t", "CREATE TABLE t (a INT)") + require.ErrorContains(t, err, "injected error: fail to close meta file") +>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) } func TestWriteViewMeta(t *testing.T) { @@ -137,6 +152,13 @@ func TestWriteTableData(t *testing.T) { "(3,'male','john@mail.com','020-1256','healthy'),\n" + "(4,'female','sarah@mail.com','020-1235','healthy');\n" require.Equal(t, expected, string(bytes)) + + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile", "return(true)")) + defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=return(true)") + + tableIR = newMockTableIR("test", "employee", data, specCmts, colTypes) + err = writer.WriteTableData(tableIR, tableIR, 0) + require.ErrorContains(t, err, "injected error: fail to close data file") } func TestWriteTableDataWithFileSize(t *testing.T) { diff --git a/dumpling/export/writer_util.go b/dumpling/export/writer_util.go index 809f82e59c9b5..fe7672f4f0015 100644 --- a/dumpling/export/writer_util.go +++ b/dumpling/export/writer_util.go @@ -451,7 +451,7 @@ func writeBytes(tctx *tcontext.Context, writer storage.ExternalFileWriter, p []b return errors.Trace(err) } -func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(ctx context.Context), error) { +func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(ctx context.Context) error, error) { fileName += compressFileSuffix(compressType) fullPath := s.URI() + "/" + fileName writer, err := storage.WithCompression(s, compressType).Create(tctx, fileName) @@ -462,20 +462,24 @@ func buildFileWriter(tctx *tcontext.Context, s storage.ExternalStorage, fileName return nil, nil, errors.Trace(err) } tctx.L().Debug("opened file", zap.String("path", fullPath)) - tearDownRoutine := func(ctx context.Context) { + tearDownRoutine := func(ctx context.Context) error { err := writer.Close(ctx) + failpoint.Inject("FailToCloseMetaFile", func(_ failpoint.Value) { + err = errors.New("injected error: fail to close meta file") + }) if err == nil { - return + return nil } err = errors.Trace(err) tctx.L().Warn("fail to close file", zap.String("path", fullPath), zap.Error(err)) + return err } return writer, tearDownRoutine, nil } -func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(context.Context)) { +func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage, fileName string, compressType storage.CompressType) (storage.ExternalFileWriter, func(context.Context) error) { fileName += compressFileSuffix(compressType) var writer storage.ExternalFileWriter fullPath := s.URI() + "/" + fileName @@ -497,17 +501,21 @@ func buildInterceptFileWriter(pCtx *tcontext.Context, s storage.ExternalStorage, } fileWriter.initRoutine = initRoutine - tearDownRoutine := func(ctx context.Context) { + tearDownRoutine := func(ctx context.Context) error { if writer == nil { - return + return nil } pCtx.L().Debug("tear down lazy file writer...", zap.String("path", fullPath)) err := writer.Close(ctx) + failpoint.Inject("FailToCloseDataFile", func(_ failpoint.Value) { + err = errors.New("injected error: fail to close data file") + }) if err != nil { pCtx.L().Warn("fail to close file", zap.String("path", fullPath), zap.Error(err)) } + return err } return fileWriter, tearDownRoutine } diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index 11a738805275b..aa4171389943c 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -128,3 +128,58 @@ run_dumpling --consistency lock -B "$DB_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling cnt=$(grep -w "$DB_NAME" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}-schema-create.sql|wc -l) echo "records count is ${cnt}" [ "$cnt" = 1 ] +<<<<<<< HEAD +======= + +echo "Test for recording network usage." +run_sql "drop database if exists test_db;" +run_sql "create database test_db;" +run_sql "create table test_db.test_table (a int primary key);" +run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);" + +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/SetIOTotalBytes=return(1)" +run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log +cnt=$(grep "IOTotalBytes=" ${DUMPLING_OUTPUT_DIR}/dumpling.log | grep -v "IOTotalBytes=0" | wc -l) +[ "$cnt" -ge 1 ] + +echo "Test for failing to close meta/data file" +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=1*return" +rm ${DUMPLING_OUTPUT_DIR}/dumpling.log +set +e +run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log +set -e +cnt=$(grep -w "dump failed error stack info" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +[ "$cnt" -ge 1 ] + +# dumpling retry will make it succeed +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=1*return" +export DUMPLING_TEST_PORT=4000 +run_sql "drop database if exists test_db;" +run_sql "create database test_db;" +run_sql "create table test_db.test_table (a int primary key);" +run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);" +rm ${DUMPLING_OUTPUT_DIR}/dumpling.log +set +e +run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log +set -e +cnt=$(grep -w "dump data successfully" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +[ "$cnt" -ge 1 ] +cnt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/test_db.test_table.000000000.sql|wc -l) +echo "records count is ${cnt}" +[ "$cnt" -eq 8 ] + +export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=5*return" +rm ${DUMPLING_OUTPUT_DIR}/dumpling.log +set +e +run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log +set -e +cnt=$(grep -w "dump failed error stack info" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l) +[ "$cnt" -ge 1 ] + +echo "Test for empty query result, should success." +run_sql "drop database if exists test_db;" +run_sql "create database test_db;" +run_sql "create table test_db.test_table (a int primary key);" +export GO_FAILPOINTS="" +run_dumpling --sql "select * from test_db.test_table" --filetype csv > ${DUMPLING_OUTPUT_DIR}/dumpling.log +>>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) From 751d1751abf72da71571831517eab473316ac756 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 10 Aug 2023 15:03:02 +0800 Subject: [PATCH 2/5] fix code --- dumpling/export/writer.go | 14 +++----------- dumpling/export/writer_test.go | 7 ------- dumpling/tests/basic/run.sh | 3 --- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/dumpling/export/writer.go b/dumpling/export/writer.go index eecff89746abe..3f7e2f1ad0c24 100644 --- a/dumpling/export/writer.go +++ b/dumpling/export/writer.go @@ -278,21 +278,13 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E if err != nil { return errors.Trace(err) } -<<<<<<< HEAD - defer tearDown(tctx) - return WriteMeta(tctx, &metaData{ - target: target, - metaSQL: metaSQL, - specCmts: []string{ - "/*!40101 SET NAMES binary*/;", - }, -======= err = WriteMeta(tctx, &metaData{ target: target, metaSQL: metaSQL, - specCmts: getSpecialComments(w.conf.ServerInfo.ServerType), ->>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) + specCmts: []string{ + "/*!40101 SET NAMES binary*/;", + }, }, fileWriter) tearDownErr := tearDown(tctx) if err == nil { diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go index 9811706c34d89..67398dbfa64b3 100644 --- a/dumpling/export/writer_test.go +++ b/dumpling/export/writer_test.go @@ -12,11 +12,8 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" -<<<<<<< HEAD -======= "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/version" ->>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) tcontext "github.com/pingcap/tidb/dumpling/context" "github.com/pingcap/tidb/util/promutil" "github.com/stretchr/testify/require" @@ -75,17 +72,13 @@ func TestWriteTableMeta(t *testing.T) { require.NoError(t, err) bytes, err := ioutil.ReadFile(p) require.NoError(t, err) -<<<<<<< HEAD require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) -======= - require.Equal(t, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile", "return(true)")) defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=return(true)") err = writer.WriteTableMeta("test", "t", "CREATE TABLE t (a INT)") require.ErrorContains(t, err, "injected error: fail to close meta file") ->>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) } func TestWriteViewMeta(t *testing.T) { diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index aa4171389943c..d4d643c00d53b 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -128,8 +128,6 @@ run_dumpling --consistency lock -B "$DB_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling cnt=$(grep -w "$DB_NAME" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}-schema-create.sql|wc -l) echo "records count is ${cnt}" [ "$cnt" = 1 ] -<<<<<<< HEAD -======= echo "Test for recording network usage." run_sql "drop database if exists test_db;" @@ -182,4 +180,3 @@ run_sql "create database test_db;" run_sql "create table test_db.test_table (a int primary key);" export GO_FAILPOINTS="" run_dumpling --sql "select * from test_db.test_table" --filetype csv > ${DUMPLING_OUTPUT_DIR}/dumpling.log ->>>>>>> aca44298814 (dumpling: fix dumpling ignore file writer close error (#45374)) From 8f68913b76ac0266bf3984e8ea2dfbd0480b3154 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 10 Aug 2023 15:11:19 +0800 Subject: [PATCH 3/5] fix code --- dumpling/tests/basic/run.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/dumpling/tests/basic/run.sh b/dumpling/tests/basic/run.sh index d4d643c00d53b..2134c4f3c3594 100644 --- a/dumpling/tests/basic/run.sh +++ b/dumpling/tests/basic/run.sh @@ -129,17 +129,6 @@ cnt=$(grep -w "$DB_NAME" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}-schema-create.sql|wc echo "records count is ${cnt}" [ "$cnt" = 1 ] -echo "Test for recording network usage." -run_sql "drop database if exists test_db;" -run_sql "create database test_db;" -run_sql "create table test_db.test_table (a int primary key);" -run_sql "insert into test_db.test_table values (1),(2),(3),(4),(5),(6),(7),(8);" - -export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/SetIOTotalBytes=return(1)" -run_dumpling -B "test_db" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log -cnt=$(grep "IOTotalBytes=" ${DUMPLING_OUTPUT_DIR}/dumpling.log | grep -v "IOTotalBytes=0" | wc -l) -[ "$cnt" -ge 1 ] - echo "Test for failing to close meta/data file" export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=1*return" rm ${DUMPLING_OUTPUT_DIR}/dumpling.log From 8fc1ca60180eda5f00461a063f6048024ee3b2eb Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 10 Aug 2023 16:23:55 +0800 Subject: [PATCH 4/5] fix lint --- dumpling/export/writer.go | 4 ++-- dumpling/export/writer_test.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dumpling/export/writer.go b/dumpling/export/writer.go index 3f7e2f1ad0c24..291b5fcc4ac1d 100644 --- a/dumpling/export/writer.go +++ b/dumpling/export/writer.go @@ -280,8 +280,8 @@ func writeMetaToFile(tctx *tcontext.Context, target, metaSQL string, s storage.E } err = WriteMeta(tctx, &metaData{ - target: target, - metaSQL: metaSQL, + target: target, + metaSQL: metaSQL, specCmts: []string{ "/*!40101 SET NAMES binary*/;", }, diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go index 67398dbfa64b3..60cb8a6417e29 100644 --- a/dumpling/export/writer_test.go +++ b/dumpling/export/writer_test.go @@ -13,7 +13,6 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/br/pkg/version" tcontext "github.com/pingcap/tidb/dumpling/context" "github.com/pingcap/tidb/util/promutil" "github.com/stretchr/testify/require" From 3d9f8e1424df1a1942efd859a431c40af4ab5d85 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 16 Aug 2023 14:35:59 +0800 Subject: [PATCH 5/5] fix failpoint --- dumpling/export/writer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go index 60cb8a6417e29..458a6540353dd 100644 --- a/dumpling/export/writer_test.go +++ b/dumpling/export/writer_test.go @@ -74,7 +74,7 @@ func TestWriteTableMeta(t *testing.T) { require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile", "return(true)")) - defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile=return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseMetaFile") err = writer.WriteTableMeta("test", "t", "CREATE TABLE t (a INT)") require.ErrorContains(t, err, "injected error: fail to close meta file") @@ -146,7 +146,7 @@ func TestWriteTableData(t *testing.T) { require.Equal(t, expected, string(bytes)) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile", "return(true)")) - defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile=return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/dumpling/export/FailToCloseDataFile") tableIR = newMockTableIR("test", "employee", data, specCmts, colTypes) err = writer.WriteTableData(tableIR, tableIR, 0)