From e6ccc0282ed288786c30786a6817034a75c6fc39 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Fri, 7 Jul 2023 21:27:21 +0800 Subject: [PATCH 01/14] fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 38 ++++++++---- br/pkg/lightning/backend/kv/base_test.go | 73 ++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 br/pkg/lightning/backend/kv/base_test.go diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index a9185bc48166a..872f95427343f 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -299,17 +299,35 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo row = row[j : j+1] } - e.logger.Error("kv convert failed", - zap.Array("original", RowArrayMarshaller(row)), - zap.Int("originalCol", j), - zap.String("colName", colInfo.Name.O), - zap.Stringer("colType", &colInfo.FieldType), - log.ShortError(err), - ) + var rowPrefix string + var originalPrefix string + if len(row[0].GetString()) >= 512*1024 { + rowPrefix = row[0].GetString()[0:1024] + originalPrefix = original.GetString()[0:1024] + + e.logger.Error("kv convert failed", + zap.String("original", rowPrefix), + zap.Int("originalCol", j), + zap.String("colName", colInfo.Name.O), + zap.Stringer("colType", &colInfo.FieldType), + log.ShortError(err), + ) + e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), + zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), + zap.Int("columnID", j+1)) + } else { + e.logger.Error("kv convert failed", + zap.Array("original", RowArrayMarshaller(row)), + zap.Int("originalCol", j), + zap.String("colName", colInfo.Name.O), + zap.Stringer("colType", &colInfo.FieldType), + log.ShortError(err), + ) + e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), + zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), + zap.Int("columnID", j+1)) + } - e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), - zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), - zap.Int("columnID", j+1)) return errors.Annotatef( err, "failed to cast value as %s for column `%s` (#%d)", &colInfo.FieldType, colInfo.Name.O, j+1, diff --git a/br/pkg/lightning/backend/kv/base_test.go b/br/pkg/lightning/backend/kv/base_test.go new file mode 100644 index 0000000000000..fee5698ac5069 --- /dev/null +++ b/br/pkg/lightning/backend/kv/base_test.go @@ -0,0 +1,73 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv_test + +import ( + "fmt" + "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" + "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" + "github.com/pingcap/tidb/br/pkg/lightning/log" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/types" + "github.com/stretchr/testify/require" + "os" + "testing" +) + +func TestLogKVConvertFailed(t *testing.T) { + dir := "/temp.txt" + dir = fmt.Sprintf("%s%s", t.TempDir(), dir) + logCfg := &log.Config{File: dir, FileMaxSize: 1} + err := log.InitLogger(logCfg, "info") + require.NoError(t, err) + msg := "logger is initialized" + log.L().Info(msg) + + c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} + cols := []*model.ColumnInfo{c1} + tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} + var tbl table.Table + tbl, err = tables.TableFromMeta(kv.NewPanickingAllocators(0), tblInfo) + require.NoError(t, err) + + var baseKVEncoder *kv.BaseKVEncoder + baseKVEncoder, err = kv.NewBaseKVEncoder(&encode.EncodingConfig{ + Table: tbl, + SessionOptions: encode.SessionOptions{ + SQLMode: mysql.ModeStrictAllTables, + Timestamp: 1234567890, + }, + Logger: log.L(), + }) + newString := "test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test" + for i := 0; i < 10000; i++ { + newString = newString + "_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test" + } + newDatum := types.NewStringDatum(newString) + rows := []types.Datum{} + for i := 0; i <= 10; i++ { + rows = append(rows, newDatum) + } + err = baseKVEncoder.LogKVConvertFailed(rows, 6, c1, err) + require.NoError(t, err) + + var content []byte + content, err = os.ReadFile(dir) + require.NoError(t, err) + require.LessOrEqual(t, 500, len(string(content))) +} From 76697f88fcbf2abc8d51c4c12a4b6584c359517b Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Fri, 7 Jul 2023 22:19:17 +0800 Subject: [PATCH 02/14] modify the list of unit tests in BUILD.bazel --- br/pkg/lightning/backend/kv/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index 006ff7ea876ef..765c49c4bf7b7 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -48,6 +48,7 @@ go_test( name = "kv_test", timeout = "short", srcs = [ + "base_test" "kv2sql_test.go", "session_internal_test.go", "session_test.go", From a0618380458f459ac837e14ae779c17d9e13c818 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Fri, 7 Jul 2023 22:23:03 +0800 Subject: [PATCH 03/14] modify the list of unit tests in BUILD.bazel --- br/pkg/lightning/backend/kv/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index 765c49c4bf7b7..f995874a96c66 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -48,7 +48,7 @@ go_test( name = "kv_test", timeout = "short", srcs = [ - "base_test" + "base_test", "kv2sql_test.go", "session_internal_test.go", "session_test.go", From 88bac434aaeea93c8a61ae6ad3b0466b692a65d1 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Fri, 7 Jul 2023 22:34:36 +0800 Subject: [PATCH 04/14] modify the list of unit tests in BUILD.bazel --- br/pkg/lightning/backend/kv/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index f995874a96c66..11ed55d8cc45a 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -48,7 +48,7 @@ go_test( name = "kv_test", timeout = "short", srcs = [ - "base_test", + "base_test.go", "kv2sql_test.go", "session_internal_test.go", "session_test.go", From 7441b1599d89c972fe1ecb3a8f45b9a056ecc6f2 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 15:31:21 +0800 Subject: [PATCH 05/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 57 ++++++++++-------------- br/pkg/lightning/backend/kv/base_test.go | 14 +++--- 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 872f95427343f..5f5d9df5c6733 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -77,6 +77,7 @@ var kindStr = [...]string{ // MarshalLogArray implements the zapcore.ArrayMarshaler interface func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) error { + var totalLength = 0 for _, datum := range row { kind := datum.Kind() var str string @@ -93,13 +94,21 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro if err != nil { return err } + if len(str) > 512*1024 { + str = str[0:1024] + } + totalLength += len(str) } - if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { - enc.AddString("kind", kindStr[kind]) - enc.AddString("val", redact.String(str)) + if totalLength < 512*1024 { + if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { + enc.AddString("kind", kindStr[kind]) + enc.AddString("val", redact.String(str)) + return nil + })); err != nil { + return err + } + } else { return nil - })); err != nil { - return err } } return nil @@ -299,35 +308,17 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo row = row[j : j+1] } - var rowPrefix string - var originalPrefix string - if len(row[0].GetString()) >= 512*1024 { - rowPrefix = row[0].GetString()[0:1024] - originalPrefix = original.GetString()[0:1024] - - e.logger.Error("kv convert failed", - zap.String("original", rowPrefix), - zap.Int("originalCol", j), - zap.String("colName", colInfo.Name.O), - zap.Stringer("colType", &colInfo.FieldType), - log.ShortError(err), - ) - e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), - zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), - zap.Int("columnID", j+1)) - } else { - e.logger.Error("kv convert failed", - zap.Array("original", RowArrayMarshaller(row)), - zap.Int("originalCol", j), - zap.String("colName", colInfo.Name.O), - zap.Stringer("colType", &colInfo.FieldType), - log.ShortError(err), - ) - e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), - zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), - zap.Int("columnID", j+1)) - } + e.logger.Error("kv convert failed", + zap.Array("original", RowArrayMarshaller(row)), + zap.Int("originalCol", j), + zap.String("colName", colInfo.Name.O), + zap.Stringer("colType", &colInfo.FieldType), + log.ShortError(err), + ) + e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), + zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), + zap.Int("columnID", j+1)) return errors.Annotatef( err, "failed to cast value as %s for column `%s` (#%d)", &colInfo.FieldType, colInfo.Name.O, j+1, diff --git a/br/pkg/lightning/backend/kv/base_test.go b/br/pkg/lightning/backend/kv/base_test.go index fee5698ac5069..b1a6a37590094 100644 --- a/br/pkg/lightning/backend/kv/base_test.go +++ b/br/pkg/lightning/backend/kv/base_test.go @@ -16,6 +16,10 @@ package kv_test import ( "fmt" + "os" + "strings" + "testing" + "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/br/pkg/lightning/log" @@ -25,8 +29,6 @@ import ( "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/stretchr/testify/require" - "os" - "testing" ) func TestLogKVConvertFailed(t *testing.T) { @@ -54,11 +56,11 @@ func TestLogKVConvertFailed(t *testing.T) { }, Logger: log.L(), }) - newString := "test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test" - for i := 0; i < 10000; i++ { - newString = newString + "_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test_test" + var newString strings.Builder + for i := 0; i < 100000; i++ { + newString.WriteString("test_test_test_test_") } - newDatum := types.NewStringDatum(newString) + newDatum := types.NewStringDatum(newString.String()) rows := []types.Datum{} for i := 0; i <= 10; i++ { rows = append(rows, newDatum) From ee24ec616a2bff1626f749b1e5df22c7aa786b89 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 16:18:59 +0800 Subject: [PATCH 06/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 5f5d9df5c6733..7c063b6ad0210 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -316,9 +316,17 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo log.ShortError(err), ) - e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), - zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), - zap.Int("columnID", j+1)) + var originalPrefix string + if len(row[0].GetString()) >= 512*1024 { + originalPrefix = original.GetString()[0:1024] + e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), + zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), + zap.Int("columnID", j+1)) + } else { + e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", original.GetValue()), + zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), + zap.Int("columnID", j+1)) + } return errors.Annotatef( err, "failed to cast value as %s for column `%s` (#%d)", &colInfo.FieldType, colInfo.Name.O, j+1, From 336d2577f40ce080562bd9eaa26aded1e941af1a Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 16:36:57 +0800 Subject: [PATCH 07/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 7c063b6ad0210..bae113506ed09 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -99,16 +99,15 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro } totalLength += len(str) } - if totalLength < 512*1024 { - if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { - enc.AddString("kind", kindStr[kind]) - enc.AddString("val", redact.String(str)) - return nil - })); err != nil { - return err - } - } else { + if totalLength >= 512*1024 { + return nil + } + if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { + enc.AddString("kind", kindStr[kind]) + enc.AddString("val", redact.String(str)) return nil + })); err != nil { + return err } } return nil From 5825bbf73075e4d1324ecdc72476aac9c0d562ae Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 16:49:55 +0800 Subject: [PATCH 08/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/base_test.go b/br/pkg/lightning/backend/kv/base_test.go index b1a6a37590094..e86feeaa1ebf0 100644 --- a/br/pkg/lightning/backend/kv/base_test.go +++ b/br/pkg/lightning/backend/kv/base_test.go @@ -40,7 +40,10 @@ func TestLogKVConvertFailed(t *testing.T) { msg := "logger is initialized" log.L().Info(msg) - c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} + modelName := model.NewCIStr("c1") + modelState := model.StatePublic + modelFieldType := *types.NewFieldType(mysql.TypeTiny) + c1 := &model.ColumnInfo{ID: 1, Name: modelName, State: modelState, Offset: 0, FieldType: modelFieldType} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} var tbl table.Table From ac9cdb282eacc96d2b5bf2b38f7ac77d04aee809 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 17:46:29 +0800 Subject: [PATCH 09/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index 11ed55d8cc45a..d9ad7497ca579 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -57,7 +57,7 @@ go_test( embed = [":kv"], flaky = True, race = "on", - shard_count = 18, + shard_count = 19, deps = [ "//br/pkg/lightning/backend/encode", "//br/pkg/lightning/common", From 0db14ec4fa387b6a820550bf894ae815afbf6cb8 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Mon, 10 Jul 2023 21:52:07 +0800 Subject: [PATCH 10/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 12 ++++++------ br/pkg/lightning/backend/kv/base_test.go | 21 +++++++++------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index bae113506ed09..5e741c4efedef 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -94,11 +94,11 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro if err != nil { return err } - if len(str) > 512*1024 { - str = str[0:1024] - } - totalLength += len(str) } + if len(str) > 512*1024 { + str = str[0:1024] + " (truncated)" + } + totalLength += len(str) if totalLength >= 512*1024 { return nil } @@ -315,8 +315,8 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo log.ShortError(err), ) - var originalPrefix string - if len(row[0].GetString()) >= 512*1024 { + if len(original.GetString()) >= 512*1024 { + var originalPrefix string originalPrefix = original.GetString()[0:1024] e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), diff --git a/br/pkg/lightning/backend/kv/base_test.go b/br/pkg/lightning/backend/kv/base_test.go index e86feeaa1ebf0..d02956e4016f0 100644 --- a/br/pkg/lightning/backend/kv/base_test.go +++ b/br/pkg/lightning/backend/kv/base_test.go @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kv_test +package kv import ( - "fmt" "os" + "path/filepath" "strings" "testing" "github.com/pingcap/tidb/br/pkg/lightning/backend/encode" - "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" @@ -32,13 +31,10 @@ import ( ) func TestLogKVConvertFailed(t *testing.T) { - dir := "/temp.txt" - dir = fmt.Sprintf("%s%s", t.TempDir(), dir) - logCfg := &log.Config{File: dir, FileMaxSize: 1} + tempPath := filepath.Join(t.TempDir(), "/temp.txt") + logCfg := &log.Config{File: tempPath, FileMaxSize: 1} err := log.InitLogger(logCfg, "info") require.NoError(t, err) - msg := "logger is initialized" - log.L().Info(msg) modelName := model.NewCIStr("c1") modelState := model.StatePublic @@ -47,11 +43,11 @@ func TestLogKVConvertFailed(t *testing.T) { cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} var tbl table.Table - tbl, err = tables.TableFromMeta(kv.NewPanickingAllocators(0), tblInfo) + tbl, err = tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) require.NoError(t, err) - var baseKVEncoder *kv.BaseKVEncoder - baseKVEncoder, err = kv.NewBaseKVEncoder(&encode.EncodingConfig{ + var baseKVEncoder *BaseKVEncoder + baseKVEncoder, err = NewBaseKVEncoder(&encode.EncodingConfig{ Table: tbl, SessionOptions: encode.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, @@ -72,7 +68,8 @@ func TestLogKVConvertFailed(t *testing.T) { require.NoError(t, err) var content []byte - content, err = os.ReadFile(dir) + content, err = os.ReadFile(tempPath) require.NoError(t, err) require.LessOrEqual(t, 500, len(string(content))) + require.NotContains(t, content, "exceeds maximum file size") } From 5104276cae4033d469dd918e7a9dffb1135f99b8 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Tue, 11 Jul 2023 13:26:06 +0800 Subject: [PATCH 11/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 5e741c4efedef..430d8b26ad920 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -316,8 +316,7 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo ) if len(original.GetString()) >= 512*1024 { - var originalPrefix string - originalPrefix = original.GetString()[0:1024] + originalPrefix := original.GetString()[0:1024] e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), zap.Int("columnID", j+1)) From 5805bff592565b4966769113e99b252b4d2ad367 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Tue, 11 Jul 2023 14:22:11 +0800 Subject: [PATCH 12/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 430d8b26ad920..0ba0f46098bdc 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -100,6 +100,7 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro } totalLength += len(str) if totalLength >= 512*1024 { + encoder.AppendString("The row has been truncated, and the log has exited early.") return nil } if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { @@ -316,7 +317,7 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo ) if len(original.GetString()) >= 512*1024 { - originalPrefix := original.GetString()[0:1024] + originalPrefix := original.GetString()[0:1024] + " (truncated)" e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), zap.Int("columnID", j+1)) From 585941fd7ef9fd7ac1fb575fa022d023f6ed7d35 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Thu, 13 Jul 2023 15:27:49 +0800 Subject: [PATCH 13/14] lightning: fix lightning failed to log encoding error --- br/pkg/lightning/backend/kv/base.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index 0ba0f46098bdc..ebc50d9eaff09 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -37,6 +37,10 @@ import ( "go.uber.org/zap/zapcore" ) +const ( + maxFileSize = 512 * 1024 +) + // ExtraHandleColumnInfo is the column info of extra handle column. var ExtraHandleColumnInfo = model.NewExtraHandleColInfo() @@ -95,11 +99,11 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro return err } } - if len(str) > 512*1024 { + if len(str) > maxFileSize { str = str[0:1024] + " (truncated)" } totalLength += len(str) - if totalLength >= 512*1024 { + if totalLength >= maxFileSize { encoder.AppendString("The row has been truncated, and the log has exited early.") return nil } @@ -316,7 +320,7 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo log.ShortError(err), ) - if len(original.GetString()) >= 512*1024 { + if len(original.GetString()) >= maxFileSize { originalPrefix := original.GetString()[0:1024] + " (truncated)" e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O), From 7aa255ed7d84e8182c56fee25581341d53170254 Mon Sep 17 00:00:00 2001 From: lyzx2001 Date: Fri, 14 Jul 2023 11:10:33 +0800 Subject: [PATCH 14/14] lightning: fix lightning failed to log encoding error of ill-formatted row --- br/pkg/lightning/backend/kv/base.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/br/pkg/lightning/backend/kv/base.go b/br/pkg/lightning/backend/kv/base.go index ebc50d9eaff09..dece47edc9788 100644 --- a/br/pkg/lightning/backend/kv/base.go +++ b/br/pkg/lightning/backend/kv/base.go @@ -38,7 +38,7 @@ import ( ) const ( - maxFileSize = 512 * 1024 + maxLogLength = 512 * 1024 ) // ExtraHandleColumnInfo is the column info of extra handle column. @@ -99,11 +99,11 @@ func (row RowArrayMarshaller) MarshalLogArray(encoder zapcore.ArrayEncoder) erro return err } } - if len(str) > maxFileSize { + if len(str) > maxLogLength { str = str[0:1024] + " (truncated)" } totalLength += len(str) - if totalLength >= maxFileSize { + if totalLength >= maxLogLength { encoder.AppendString("The row has been truncated, and the log has exited early.") return nil } @@ -320,7 +320,7 @@ func (e *BaseKVEncoder) LogKVConvertFailed(row []types.Datum, j int, colInfo *mo log.ShortError(err), ) - if len(original.GetString()) >= maxFileSize { + if len(original.GetString()) >= maxLogLength { originalPrefix := original.GetString()[0:1024] + " (truncated)" e.logger.Error("failed to convert kv value", logutil.RedactAny("origVal", originalPrefix), zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O),