From e01d6c92c1f37b89eab423b13242974fa01d6af1 Mon Sep 17 00:00:00 2001 From: erwadba Date: Tue, 7 Mar 2023 14:37:46 +0800 Subject: [PATCH 1/2] use hex to compare blob data --- pkg/dbutil/types.go | 10 ++++ sync_diff_inspector/utils/utils.go | 78 +++++++++++++++++++----------- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/pkg/dbutil/types.go b/pkg/dbutil/types.go index d13ba9e07..0ccdb6731 100644 --- a/pkg/dbutil/types.go +++ b/pkg/dbutil/types.go @@ -31,3 +31,13 @@ func IsTimeTypeAndNeedDecode(tp byte) bool { } return false } + +// IsBLOBType returns true if tp is Blob type +func IsBLOBType(tp byte) bool { + switch tp { + case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: + return true + } + + return false +} diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index 5f5031730..b2ab6b47f 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -184,7 +184,11 @@ func GenerateReplaceDML(data map[string]*dbutil.ColumnData, table *model.TableIn } if NeedQuotes(col.FieldType.GetType()) { - values = append(values, fmt.Sprintf("'%s'", strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1))) + if dbutil.IsBLOBType(col.FieldType.GetType()) { + values = append(values, fmt.Sprintf("x'%x'", data[col.Name.O].Data)) + } else { + values = append(values, fmt.Sprintf("'%s'", strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1))) + } } else { values = append(values, string(data[col.Name.O].Data)) } @@ -217,7 +221,11 @@ func GenerateReplaceDMLWithAnnotation(source, target map[string]*dbutil.ColumnDa value1 = "NULL" } else { if NeedQuotes(col.FieldType.GetType()) { - value1 = fmt.Sprintf("'%s'", strings.Replace(string(data1.Data), "'", "\\'", -1)) + if dbutil.IsBLOBType(col.FieldType.GetType()) { + value1 = fmt.Sprintf("x'%x'", data1.Data) + } else { + value1 = fmt.Sprintf("'%s'", strings.Replace(string(data1.Data), "'", "\\'", -1)) + } } else { value1 = string(data1.Data) } @@ -238,7 +246,11 @@ func GenerateReplaceDMLWithAnnotation(source, target map[string]*dbutil.ColumnDa values2 = append(values2, "NULL") } else { if NeedQuotes(col.FieldType.GetType()) { - values2 = append(values2, fmt.Sprintf("'%s'", strings.Replace(string(data2.Data), "'", "\\'", -1))) + if dbutil.IsBLOBType(col.FieldType.GetType()) { + values2 = append(values2, fmt.Sprintf("x'%x'", data1.Data)) + } else { + values2 = append(values2, fmt.Sprintf("'%s'", strings.Replace(string(data2.Data), "'", "\\'", -1))) + } } else { values2 = append(values2, string(data2.Data)) } @@ -274,7 +286,11 @@ func GenerateDeleteDML(data map[string]*dbutil.ColumnData, table *model.TableInf } if NeedQuotes(col.FieldType.GetType()) { - kvs = append(kvs, fmt.Sprintf("%s = '%s'", dbutil.ColumnName(col.Name.O), strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1))) + if dbutil.IsBLOBType(col.FieldType.GetType()) { + kvs = append(kvs, fmt.Sprintf("%s = x'%x'", dbutil.ColumnName(col.Name.O), data[col.Name.O].Data)) + } else { + kvs = append(kvs, fmt.Sprintf("%s = '%s'", dbutil.ColumnName(col.Name.O), strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1))) + } } else { kvs = append(kvs, fmt.Sprintf("%s = %s", dbutil.ColumnName(col.Name.O), string(data[col.Name.O].Data))) } @@ -538,10 +554,13 @@ func CompareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols, columns } str1 = string(data1.Data) str2 = string(data2.Data) - if column.FieldType.GetType() == mysql.TypeFloat || column.FieldType.GetType() == mysql.TypeDouble { - if data1.IsNull && data2.IsNull { + if data1.IsNull && data2.IsNull { + if str1 == str2 { continue - } else if !data1.IsNull && !data2.IsNull { + } + } else if !data1.IsNull && !data2.IsNull { + switch column.FieldType.GetType() { + case mysql.TypeFloat, mysql.TypeDouble: num1, err1 := strconv.ParseFloat(str1, 64) num2, err2 := strconv.ParseFloat(str2, 64) if err1 != nil || err2 != nil { @@ -551,26 +570,27 @@ func CompareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols, columns if math.Abs(num1-num2) <= 1e-6 { continue } - } - } else if column.FieldType.GetType() == mysql.TypeJSON { - if (str1 == str2) || (data1.IsNull && data2.IsNull) { - continue - } - var v1, v2 any - err := json.Unmarshal(data1.Data, &v1) - if err != nil { - return false, 0, errors.Errorf("unmarshal json %s failed, error %v", str1, err) - } - err = json.Unmarshal(data2.Data, &v2) - if err != nil { - return false, 0, errors.Errorf("unmarshal json %s failed, error %v", str2, err) - } - if reflect.DeepEqual(v1, v2) { - continue - } - } else { - if (str1 == str2) && (data1.IsNull == data2.IsNull) { - continue + case mysql.TypeJSON: + var v1, v2 any + err := json.Unmarshal(data1.Data, &v1) + if err != nil { + return false, 0, errors.Errorf("unmarshal json %s failed, error %v", str1, err) + } + err = json.Unmarshal(data2.Data, &v2) + if err != nil { + return false, 0, errors.Errorf("unmarshal json %s failed, error %v", str2, err) + } + if reflect.DeepEqual(v1, v2) { + continue + } + case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: + if fmt.Sprintf("%x", data1.Data) == fmt.Sprintf("%x", data2.Data) { + continue + } + default: + if str1 == str2 { + continue + } } } @@ -597,6 +617,10 @@ func CompareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols, columns if NeedQuotes(col.FieldType.GetType()) { strData1 := string(data1.Data) strData2 := string(data2.Data) + if dbutil.IsBLOBType(col.FieldType.GetType()) { + strData1 = fmt.Sprintf("%x", data1.Data) + strData2 = fmt.Sprintf("%x", data2.Data) + } if len(strData1) == len(strData2) && strData1 == strData2 { continue From b1004b047082cde2601f2444c74ad31bb25081e8 Mon Sep 17 00:00:00 2001 From: erwadba Date: Wed, 15 Mar 2023 14:06:20 +0800 Subject: [PATCH 2/2] check str1 == str2 first --- sync_diff_inspector/utils/utils.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index b2ab6b47f..c59d75e08 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -555,10 +555,11 @@ func CompareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols, columns str1 = string(data1.Data) str2 = string(data2.Data) if data1.IsNull && data2.IsNull { + continue + } else if !data1.IsNull && !data2.IsNull { if str1 == str2 { continue } - } else if !data1.IsNull && !data2.IsNull { switch column.FieldType.GetType() { case mysql.TypeFloat, mysql.TypeDouble: num1, err1 := strconv.ParseFloat(str1, 64) @@ -587,10 +588,6 @@ func CompareData(map1, map2 map[string]*dbutil.ColumnData, orderKeyCols, columns if fmt.Sprintf("%x", data1.Data) == fmt.Sprintf("%x", data2.Data) { continue } - default: - if str1 == str2 { - continue - } } }