diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go
index b6a0da7018612..b743a21b824e2 100644
--- a/ddl/db_integration_test.go
+++ b/ddl/db_integration_test.go
@@ -172,25 +172,47 @@ func (s *testIntegrationSuite) TestNullGeneratedColumn(c *C) {
 
 func (s *testIntegrationSuite) TestChangingCharsetToUtf8(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t1(a varchar(20) charset utf8)")
+	tk.MustExec("insert into t1 values (?)", "t1_value")
 
-	tk.MustExec("USE test")
-	tk.MustExec("create table t(a char(10) charset latin1)")
-	tk.MustExec("alter table t modify column a char(10) charset latin1")
-	tk.MustExec("alter table t modify column a char(10) charset utf8")
-	tk.MustExec("alter table t modify column a char(10) charset utf8mb4")
-	rs, err := tk.Exec("alter table t modify column a char(10) charset utf8mb4 collate utf8bin")
+	tk.MustExec("alter table t1 modify column a varchar(20) charset utf8mb4")
+	tk.MustQuery("select * from t1;").Check(testkit.Rows("t1_value"))
+
+	tk.MustExec("create table t(a varchar(20) charset latin1)")
+	tk.MustExec("insert into t values (?)", "t_value")
+
+	tk.MustExec("alter table t modify column a varchar(20) charset latin1")
+	tk.MustQuery("select * from t;").Check(testkit.Rows("t_value"))
+
+	rs, err := tk.Exec("alter table t modify column a varchar(20) charset utf8")
 	if rs != nil {
 		rs.Close()
 	}
 	c.Assert(err, NotNil)
-	tk.MustExec("alter table t modify column a char(10) charset utf8mb4 collate utf8mb4_bin")
-	rs, err = tk.Exec("alter table t modify column a char(10) charset utf8 collate utf8_bin")
+	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from latin1 to utf8")
+	rs, err = tk.Exec("alter table t modify column a varchar(20) charset utf8mb4")
 	if rs != nil {
 		rs.Close()
 	}
+	c.Assert(err, NotNil)
+	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from latin1 to utf8mb4")
 
+	rs, err = tk.Exec("alter table t modify column a varchar(20) charset utf8mb4 collate utf8bin")
+	if rs != nil {
+		rs.Close()
+	}
+	c.Assert(err, NotNil)
+	rs, err = tk.Exec("alter table t modify column a varchar(20) charset utf8 collate utf8_bin")
+	if rs != nil {
+		rs.Close()
+	}
+	c.Assert(err, NotNil)
+	rs, err = tk.Exec("alter table t modify column a varchar(20) charset utf8mb4 collate utf8mb4_general_ci")
+	if rs != nil {
+		rs.Close()
+	}
 	c.Assert(err, NotNil)
-	tk.MustExec("alter table t modify column a char(10) charset utf8mb4 collate utf8mb4_general_ci")
 }
 
 func (s *testIntegrationSuite) TestChangingTableCharset(c *C) {
@@ -203,21 +225,11 @@ func (s *testIntegrationSuite) TestChangingTableCharset(c *C) {
 		rs.Close()
 	}
 	c.Assert(err.Error(), Equals, "Unknown charset gbk")
-	tk.MustExec("alter table t charset utf8")
-	tk.MustExec("alter table t charset utf8 collate utf8_bin")
 	rs, err = tk.Exec("alter table t charset utf8 collate latin1_bin")
 	if rs != nil {
 		rs.Close()
 	}
 	c.Assert(err, NotNil)
-	tk.MustExec("alter table t charset utf8mb4")
-	tk.MustExec("alter table t charset utf8mb4 collate utf8mb4_bin")
-
-	rs, err = tk.Exec("alter table t charset utf8 collate utf8_bin")
-	if rs != nil {
-		rs.Close()
-	}
-	c.Assert(err, NotNil)
 }
 
 func (s *testIntegrationSuite) TestCaseInsensitiveCharsetAndCollate(c *C) {
diff --git a/ddl/db_test.go b/ddl/db_test.go
index 2b4c44749978c..e8abf7cdaf3d6 100644
--- a/ddl/db_test.go
+++ b/ddl/db_test.go
@@ -2051,6 +2051,27 @@ func (s *testDBSuite) testRenameTable(c *C, sql string, isAlterTable bool) {
 	s.tk.MustExec("drop database test1")
 }
 
+func (s *testDBSuite) TestCheckConvertToCharacter(c *C) {
+	s.tk = testkit.NewTestKit(c, s.store)
+	s.tk.MustExec("use test")
+	s.tk.MustExec("drop table if exists t")
+	defer s.tk.MustExec("drop table t")
+	s.tk.MustExec("create table t(a varchar(10) charset binary);")
+	ctx := s.tk.Se.(sessionctx.Context)
+	is := domain.GetDomain(ctx).InfoSchema()
+	t, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	rs, err := s.tk.Exec("alter table t modify column a varchar(10) charset utf8 collate utf8_bin")
+	c.Assert(err, NotNil)
+	rs, err = s.tk.Exec("alter table t modify column a varchar(10) charset utf8mb4 collate utf8mb4_bin")
+	c.Assert(err, NotNil)
+	rs, err = s.tk.Exec("alter table t modify column a varchar(10) charset latin collate latin1_bin")
+	c.Assert(err, NotNil)
+	if rs != nil {
+		rs.Close()
+	}
+	c.Assert(t.Cols()[0].Charset, Equals, "binary")
+}
+
 func (s *testDBSuite) TestRenameMultiTables(c *C) {
 	s.tk = testkit.NewTestKit(c, s.store)
 	s.tk.MustExec("use test")
@@ -3662,26 +3683,30 @@ func (s *testDBSuite) TestAlterTableCharset(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(t.Meta().Collate, Equals, defCollate)
 
-	tk.MustExec("alter table t charset utf8")
+	rs, err := tk.Exec("alter table t charset utf8")
+	if rs != nil {
+		rs.Close()
+	}
+	c.Assert(err, NotNil)
 	is = domain.GetDomain(ctx).InfoSchema()
 	t, err = is.TableByName(model.NewCIStr("test_charset"), model.NewCIStr("t"))
-	c.Assert(t.Meta().Charset, Equals, "utf8")
-	defCollate, err = charset.GetDefaultCollation("utf8")
+	c.Assert(t.Meta().Charset, Equals, "latin1")
+	defCollate, err = charset.GetDefaultCollation("latin1")
 	c.Assert(err, IsNil)
 	c.Assert(t.Meta().Collate, Equals, defCollate)
 
-	tk.MustExec("alter table t charset utf8mb4 collate utf8mb4_general_ci")
+	rs, err = tk.Exec("alter table t charset utf8mb4 collate utf8mb4_general_ci")
 	is = domain.GetDomain(ctx).InfoSchema()
 	t, err = is.TableByName(model.NewCIStr("test_charset"), model.NewCIStr("t"))
-	c.Assert(t.Meta().Charset, Equals, "utf8mb4")
-	c.Assert(t.Meta().Collate, Equals, "utf8mb4_general_ci")
+	c.Assert(t.Meta().Charset, Equals, "latin1")
+	c.Assert(t.Meta().Collate, Equals, "latin1_bin")
 
-	rs, err := tk.Exec("alter table t charset utf8")
+	rs, err = tk.Exec("alter table t charset utf8")
 	if rs != nil {
 		rs.Close()
 	}
 
-	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from utf8mb4 to utf8")
+	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from latin1 to utf8")
 }
 
 func (s *testDBSuite) TestAlterColumnCharset(c *C) {
@@ -3702,40 +3727,40 @@ func (s *testDBSuite) TestAlterColumnCharset(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(col.Collate, Equals, defCollate)
 
-	tk.MustExec("alter table t modify column a char(10) charset utf8")
+	_, err = tk.Exec("alter table t modify column a char(10) charset utf8")
 	is = domain.GetDomain(ctx).InfoSchema()
 	t, err = is.TableByName(model.NewCIStr("test_charset"), model.NewCIStr("t"))
 	c.Assert(err, IsNil)
 	col = model.FindColumnInfo(t.Meta().Columns, "a")
 	c.Assert(col, NotNil)
-	c.Assert(col.Charset, Equals, "utf8")
-	defCollate, err = charset.GetDefaultCollation("utf8")
+	c.Assert(col.Charset, Equals, "latin1")
+	defCollate, err = charset.GetDefaultCollation("latin1")
 	c.Assert(err, IsNil)
 	c.Assert(col.Collate, Equals, defCollate)
 
-	tk.MustExec("alter table t modify column a char(10) charset utf8 collate utf8_general_ci")
+	_, err = tk.Exec("alter table t modify column a char(10) charset utf8 collate utf8_general_ci")
 	is = domain.GetDomain(ctx).InfoSchema()
 	t, err = is.TableByName(model.NewCIStr("test_charset"), model.NewCIStr("t"))
 	c.Assert(err, IsNil)
 	col = model.FindColumnInfo(t.Meta().Columns, "a")
 	c.Assert(col, NotNil)
-	c.Assert(col.Charset, Equals, "utf8")
-	c.Assert(col.Collate, Equals, "utf8_general_ci")
+	c.Assert(col.Charset, Equals, "latin1")
+	c.Assert(col.Collate, Equals, "latin1_bin")
 
-	tk.MustExec("alter table t modify column a char(10) charset utf8mb4 collate utf8mb4_general_ci")
+	_, err = tk.Exec("alter table t modify column a char(10) charset utf8mb4 collate utf8mb4_general_ci")
 	is = domain.GetDomain(ctx).InfoSchema()
 	t, err = is.TableByName(model.NewCIStr("test_charset"), model.NewCIStr("t"))
 	c.Assert(err, IsNil)
 	col = model.FindColumnInfo(t.Meta().Columns, "a")
 	c.Assert(col, NotNil)
-	c.Assert(col.Charset, Equals, "utf8mb4")
-	c.Assert(col.Collate, Equals, "utf8mb4_general_ci")
+	c.Assert(col.Charset, Equals, "latin1")
+	c.Assert(col.Collate, Equals, "latin1_bin")
 
 	rs, err := tk.Exec("alter table t modify column a char(10) charset utf8")
 	if rs != nil {
 		rs.Close()
 	}
-	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from utf8mb4 to utf8")
+	c.Assert(err.Error(), Equals, "[ddl:210]unsupported modify charset from latin1 to utf8")
 }
 
 func (s *testDBSuite) TestDropSchemaWithPartitionTable(c *C) {
diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go
index c839b5922d994..16015d6076730 100644
--- a/ddl/ddl_api.go
+++ b/ddl/ddl_api.go
@@ -1547,10 +1547,8 @@ func modifiableCharsetAndCollation(toCharset, toCollate, origCharset, origCollat
 	if !charset.ValidCharsetAndCollation(toCharset, toCollate) {
 		return ErrUnknownCharacterSet.GenWithStackByArgs(toCharset, toCollate)
 	}
-
-	if toCharset == charset.CharsetUTF8MB4 || (toCharset == charset.CharsetUTF8 && origCharset != charset.CharsetUTF8MB4) {
-		// TiDB treats all the data as utf8mb4, so we support changing the charset to utf8mb4.
-		// And not allow to change utf8mb4 to utf8.
+	if toCharset == charset.CharsetUTF8MB4 && origCharset == charset.CharsetUTF8 {
+		// TiDB only allow utf8 to be changed to utf8mb4.
 		return nil
 	}