diff --git a/statistics/feedback.go b/statistics/feedback.go
index 87260f93e5036..3d1aa2c6d2786 100644
--- a/statistics/feedback.go
+++ b/statistics/feedback.go
@@ -112,11 +112,11 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) {
 		if isIndex {
 			var err error
 			// As we do not know the origin length, just use a custom value here.
-			lowVal, err = codec.DecodeRange(low.GetBytes(), 4)
+			lowVal, _, err = codec.DecodeRange(low.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
-			highVal, err = codec.DecodeRange(high.GetBytes(), 4)
+			highVal, _, err = codec.DecodeRange(high.GetBytes(), 4)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
@@ -819,7 +819,7 @@ func convertDatumsType(vals []types.Datum, ft *types.FieldType, loc *time.Locati
 }
 
 func decodeColumnBounds(data []byte, ft *types.FieldType) ([]types.Datum, error) {
-	vals, err := codec.DecodeRange(data, 1)
+	vals, _, err := codec.DecodeRange(data, 1)
 	if err != nil {
 		return nil, err
 	}
diff --git a/statistics/histogram.go b/statistics/histogram.go
index 9369f32b3761e..2444c3ca4591c 100644
--- a/statistics/histogram.go
+++ b/statistics/histogram.go
@@ -349,9 +349,10 @@ func ValueToString(value *types.Datum, idxCols int) (string, error) {
 	if idxCols == 0 {
 		return value.ToString()
 	}
-	decodedVals, err := codec.DecodeRange(value.GetBytes(), idxCols)
-	if err != nil {
-		return "", errors.Trace(err)
+	// Treat remaining part that cannot decode successfully as bytes.
+	decodedVals, remained, err := codec.DecodeRange(value.GetBytes(), idxCols)
+	if err != nil && len(remained) > 0 {
+		decodedVals = append(decodedVals, types.NewBytesDatum(remained))
 	}
 	str, err := types.DatumsToString(decodedVals, true)
 	if err != nil {
diff --git a/statistics/statistics_test.go b/statistics/statistics_test.go
index 17bd22d79465a..ee7410a793b60 100644
--- a/statistics/statistics_test.go
+++ b/statistics/statistics_test.go
@@ -668,3 +668,14 @@ func (s *testStatisticsSuite) TestIndexRanges(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(int(count), Equals, 0)
 }
+
+func (s *testStatisticsSuite) TestValueToString4InvalidKey(c *C) {
+	bytes, err := codec.EncodeKey(nil, nil, types.NewDatum(1), types.NewDatum(0.5))
+	c.Assert(err, IsNil)
+	// Append invalid flag.
+	bytes = append(bytes, 20)
+	datum := types.NewDatum(bytes)
+	res, err := ValueToString(&datum, 3)
+	c.Assert(err, IsNil)
+	c.Assert(res, Equals, "(1, 0.5, \x14)")
+}
diff --git a/util/codec/codec.go b/util/codec/codec.go
index c6e95a44a9b24..30225fdaf66ca 100644
--- a/util/codec/codec.go
+++ b/util/codec/codec.go
@@ -322,9 +322,9 @@ func Decode(b []byte, size int) ([]types.Datum, error) {
 
 // DecodeRange decodes the range values from a byte slice that generated by EncodeKey.
 // It handles some special values like `MinNotNull` and `MaxValueDatum`.
-func DecodeRange(b []byte, size int) ([]types.Datum, error) {
+func DecodeRange(b []byte, size int) ([]types.Datum, []byte, error) {
 	if len(b) < 1 {
-		return nil, errors.New("invalid encoded key: length of key is zero")
+		return nil, b, errors.New("invalid encoded key: length of key is zero")
 	}
 
 	var (
@@ -336,7 +336,7 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		var d types.Datum
 		b, d, err = DecodeOne(b)
 		if err != nil {
-			return nil, errors.Trace(err)
+			return values, b, errors.Trace(err)
 		}
 		values = append(values, d)
 	}
@@ -351,10 +351,10 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) {
 		case maxFlag, maxFlag + 1:
 			values = append(values, types.MaxValueDatum())
 		default:
-			return nil, errors.Errorf("invalid encoded key flag %v", b[0])
+			return values, b, errors.Errorf("invalid encoded key flag %v", b[0])
 		}
 	}
-	return values, nil
+	return values, nil, nil
 }
 
 // DecodeOne decodes on datum from a byte slice generated with EncodeKey or EncodeValue.
diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go
index 86679c9052bdd..4c9250e731f8d 100644
--- a/util/codec/codec_test.go
+++ b/util/codec/codec_test.go
@@ -20,8 +20,8 @@ import (
 	"time"
 
 	. "github.com/pingcap/check"
-	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
+	"github.com/pingcap/tidb/mysql"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/types/json"