diff --git a/executor/analyze.go b/executor/analyze.go
index a155684377cf8..12106f89a59d4 100755
--- a/executor/analyze.go
+++ b/executor/analyze.go
@@ -1210,7 +1210,7 @@ type analyzeIndexIncrementalExec struct {
 
 func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult {
 	startPos := idxExec.oldHist.GetUpper(idxExec.oldHist.Len() - 1)
-	values, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns))
+	values, _, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns))
 	if err != nil {
 		return analyzeResult{Err: err, job: idxExec.job}
 	}
diff --git a/statistics/feedback.go b/statistics/feedback.go
index 013b9c2d6f9b2..24cf07fc912d1 100644
--- a/statistics/feedback.go
+++ b/statistics/feedback.go
@@ -120,11 +120,11 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) {
 		if isIndex {
 			var err error
 			// As we do not know the origin length, just use a custom value here.
-			lowVal, err = codec.DecodeRange(low.GetBytes(), 4)
+			lowVal, _, err = codec.DecodeRange(low.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			highVal, err = codec.DecodeRange(high.GetBytes(), 4)
+			highVal, _, err = codec.DecodeRange(high.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
@@ -831,7 +831,7 @@ func ConvertDatumsType(vals []types.Datum, ft *types.FieldType, loc *time.Locati
 }
 
 func decodeColumnBounds(data []byte, ft *types.FieldType) ([]types.Datum, error) {
-	vals, err := codec.DecodeRange(data, 1)
+	vals, _, err := codec.DecodeRange(data, 1)
 	if err != nil {
 		return nil, err
 	}
diff --git a/statistics/histogram.go b/statistics/histogram.go
index 503ce178ce362..fead832b38231 100644
--- a/statistics/histogram.go
+++ b/statistics/histogram.go
@@ -226,9 +226,10 @@ func ValueToString(value *types.Datum, idxCols int) (string, error) {
 	if idxCols == 0 {
 		return value.ToString()
 	}
-	decodedVals, err := codec.DecodeRange(value.GetBytes(), idxCols)
-	if err != nil {
-		return "", errors.Trace(err)
+	// Ignore the error and treat remaining part that cannot decode successfully as bytes.
+	decodedVals, remained, _ := codec.DecodeRange(value.GetBytes(), idxCols)
+	if len(remained) > 0 {
+		decodedVals = append(decodedVals, types.NewBytesDatum(remained))
 	}
 	str, err := types.DatumsToString(decodedVals, true)
 	return str, err
diff --git a/statistics/histogram_test.go b/statistics/histogram_test.go
index 1ce1682076217..93ebe0d4fe241 100644
--- a/statistics/histogram_test.go
+++ b/statistics/histogram_test.go
@@ -130,3 +130,14 @@ func (s *testStatisticsSuite) TestTruncateHistogram(c *C) {
 	newHist = hist.TruncateHistogram(0)
 	c.Assert(newHist.Len(), Equals, 0)
 }
+
+func (s *testStatisticsSuite) TestValueToString4InvalidKey(c *C) {
+	bytes, err := codec.EncodeKey(nil, nil, types.NewDatum(1), types.NewDatum(0.5))
+	c.Assert(err, IsNil)
+	// Append invalid flag.
+	bytes = append(bytes, 20)
+	datum := types.NewDatum(bytes)
+	res, err := ValueToString(&datum, 3)
+	c.Assert(err, IsNil)
+	c.Assert(res, Equals, "(1, 0.5, \x14)")
+}
diff --git a/util/codec/codec.go b/util/codec/codec.go
index 9ba94cc3dac1c..9296bd7a7b567 100644
--- a/util/codec/codec.go
+++ b/util/codec/codec.go
@@ -424,9 +424,9 @@ func Decode(b []byte, size int) ([]types.Datum, error) {
 
 // DecodeRange decodes the range values from a byte slice that generated by EncodeKey.
 // It handles some special values like `MinNotNull` and `MaxValueDatum`.
-func DecodeRange(b []byte, size int) ([]types.Datum, error) {
+func DecodeRange(b []byte, size int) ([]types.Datum, []byte, error) {
 	if len(b) < 1 {
-		return nil, errors.New("invalid encoded key: length of key is zero")
+		return nil, b, errors.New("invalid encoded key: length of key is zero")
 	}
 
 	var (
@@ -438,7 +438,7 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		var d types.Datum
 		b, d, err = DecodeOne(b)
 		if err != nil {
-			return nil, errors.Trace(err)
+			return values, b, errors.Trace(err)
 		}
 		values = append(values, d)
 	}
@@ -453,10 +453,10 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		case maxFlag, maxFlag + 1:
 			values = append(values, types.MaxValueDatum())
 		default:
-			return nil, errors.Errorf("invalid encoded key flag %v", b[0])
+			return values, b, errors.Errorf("invalid encoded key flag %v", b[0])
 		}
 	}
-	return values, nil
+	return values, nil, nil
 }
 
 // DecodeOne decodes on datum from a byte slice generated with EncodeKey or EncodeValue.
diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go
index f23b23e7f3774..755ed032896e0 100644
--- a/util/codec/codec_test.go
+++ b/util/codec/codec_test.go
@@ -1012,14 +1012,14 @@ func chunkForTest(c *C, sc *stmtctx.StatementContext, datums []types.Datum, tps
 }
 
 func (s *testCodecSuite) TestDecodeRange(c *C) {
-	_, err := DecodeRange(nil, 0)
+	_, _, err := DecodeRange(nil, 0)
 	c.Assert(err, NotNil)
 
 	datums := types.MakeDatums(1, "abc", 1.1, []byte("def"))
 	rowData, err := EncodeValue(nil, nil, datums...)
 	c.Assert(err, IsNil)
 
-	datums1, err := DecodeRange(rowData, len(datums))
+	datums1, _, err := DecodeRange(rowData, len(datums))
 	c.Assert(err, IsNil)
 	for i, datum := range datums1 {
 		cmp, err := datum.CompareDatum(nil, &datums[i])
@@ -1029,7 +1029,7 @@ func (s *testCodecSuite) TestDecodeRange(c *C) {
 
 	for _, b := range []byte{NilFlag, bytesFlag, maxFlag, maxFlag + 1} {
 		newData := append(rowData, b)
-		_, err := DecodeRange(newData, len(datums)+1)
+		_, _, err := DecodeRange(newData, len(datums)+1)
 		c.Assert(err, IsNil)
 	}
 }