lightning: specify collation when parquet value to string datum (#38391)

close #38351
pingcap · Oct 17, 2022 · 68305e9 · 68305e9
1 parent 17fac8b
commit 68305e9
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 9 deletions.
diff --git a/br/pkg/lightning/mydump/parquet_parser.go b/br/pkg/lightning/mydump/parquet_parser.go
@@ -458,7 +458,7 @@ func setDatumByString(d *types.Datum, v string, meta *parquet.SchemaElement) {
 		ts = ts.UTC()
 		v = ts.Format(utcTimeLayout)
 	}
-	d.SetString(v, "")
+	d.SetString(v, "utf8mb4_bin")
 }
 
 func binaryToDecimalStr(rawBytes []byte, scale int) string {
@@ -515,20 +515,20 @@ func setDatumByInt(d *types.Datum, v int64, meta *parquet.SchemaElement) error {
 		}
 		val := fmt.Sprintf("%0*d", minLen, v)
 		dotIndex := len(val) - int(*meta.Scale)
-		d.SetString(val[:dotIndex]+"."+val[dotIndex:], "")
+		d.SetString(val[:dotIndex]+"."+val[dotIndex:], "utf8mb4_bin")
 	case logicalType.DATE != nil:
 		dateStr := time.Unix(v*86400, 0).Format("2006-01-02")
-		d.SetString(dateStr, "")
+		d.SetString(dateStr, "utf8mb4_bin")
 	case logicalType.TIMESTAMP != nil:
 		// convert all timestamp types (datetime/timestamp) to string
 		timeStr := formatTime(v, logicalType.TIMESTAMP.Unit, timeLayout,
 			utcTimeLayout, logicalType.TIMESTAMP.IsAdjustedToUTC)
-		d.SetString(timeStr, "")
+		d.SetString(timeStr, "utf8mb4_bin")
 	case logicalType.TIME != nil:
 		// convert all timestamp types (datetime/timestamp) to string
 		timeStr := formatTime(v, logicalType.TIME.Unit, "15:04:05.999999", "15:04:05.999999Z",
 			logicalType.TIME.IsAdjustedToUTC)
-		d.SetString(timeStr, "")
+		d.SetString(timeStr, "utf8mb4_bin")
 	default:
 		d.SetInt64(v)
 	}

diff --git a/br/pkg/lightning/mydump/parquet_parser_test.go b/br/pkg/lightning/mydump/parquet_parser_test.go
@@ -54,7 +54,7 @@ func TestParquetParser(t *testing.T) {
 	verifyRow := func(i int) {
 		require.Equal(t, int64(i+1), reader.lastRow.RowID)
 		require.Len(t, reader.lastRow.Row, 2)
-		require.Equal(t, types.NewCollationStringDatum(strconv.Itoa(i), ""), reader.lastRow.Row[0])
+		require.Equal(t, types.NewCollationStringDatum(strconv.Itoa(i), "utf8mb4_bin"), reader.lastRow.Row[0])
 		require.Equal(t, types.NewIntDatum(int64(i)), reader.lastRow.Row[1])
 	}
 

diff --git a/br/pkg/lightning/restore/get_pre_info_test.go b/br/pkg/lightning/restore/get_pre_info_test.go
@@ -349,15 +349,15 @@ INSERT INTO db01.tbl01 (ival, sval) VALUES (444, 'ddd');`
 			ExpectFirstRowDatums: [][]types.Datum{
 				{
 					types.NewIntDatum(1),
-					types.NewCollationStringDatum("name_1", ""),
+					types.NewCollationStringDatum("name_1", "utf8mb4_bin"),
 				},
 				{
 					types.NewIntDatum(2),
-					types.NewCollationStringDatum("name_2", ""),
+					types.NewCollationStringDatum("name_2", "utf8mb4_bin"),
 				},
 				{
 					types.NewIntDatum(3),
-					types.NewCollationStringDatum("name_3", ""),
+					types.NewCollationStringDatum("name_3", "utf8mb4_bin"),
 				},
 			},
 			ExpectColumns: []string{"id", "name"},