planner: prune hash partition should consider unsigned flag (#27098) (#…

…27164)
pingcap · Sep 14, 2021 · aba11c5 · aba11c5
1 parent 7f2f1a7
commit aba11c5
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 2 deletions.
diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go
@@ -68,6 +68,7 @@ func (s *testPartitionPruneSuit) TestHashPartitionPruner(c *C) {
 	tk.MustExec("create table t7(a int, b int) partition by hash(a + b) partitions 10;")
 	tk.MustExec("create table t8(a int, b int) partition by hash(a) partitions 6;")
 	tk.MustExec("create table t9(a bit(1) default null, b int(11) default null) partition by hash(a) partitions 3;") //issue #22619
+	tk.MustExec("create table t10(a bigint unsigned) partition BY hash (a);")
 
 	var input []string
 	var output []struct {

diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go
@@ -183,7 +183,13 @@ func (s *partitionProcessor) findUsedPartitions(ctx sessionctx.Context, tbl tabl
 				if r.HighExclude {
 					posHigh--
 				}
-				rangeScalar := float64(posHigh) - float64(posLow) // use float64 to avoid integer overflow
+
+				var rangeScalar float64
+				if mysql.HasUnsignedFlag(col.RetType.Flag) {
+					rangeScalar = float64(uint64(posHigh)) - float64(uint64(posLow)) // use float64 to avoid integer overflow
+				} else {
+					rangeScalar = float64(posHigh) - float64(posLow) // use float64 to avoid integer overflow
+				}
 
 				// if range is less than the number of partitions, there will be unused partitions we can prune out.
 				if rangeScalar < float64(numPartitions) && !highIsNull && !lowIsNull {

diff --git a/planner/core/testdata/partition_pruner_in.json b/planner/core/testdata/partition_pruner_in.json
@@ -29,7 +29,8 @@
       "explain format = 'brief' select * from t8 where a between 5 and 12",
       "explain format = 'brief' select * from t8 where (a <= 10 and a >= 8) or (a <= 13 and a >= 11) or (a <= 16 and a >= 14)",
       "explain format = 'brief' select * from t8 where a < 12 and a > 9",
-      "explain format = 'brief' select * from t9"
+      "explain format = 'brief' select * from t9",
+      "explain format = 'brief' select * from t10 where a between 0 AND 15218001646226433652"
     ]
   },
   {

diff --git a/planner/core/testdata/partition_pruner_out.json b/planner/core/testdata/partition_pruner_out.json
@@ -222,6 +222,14 @@
           "TableReader 10000.00 root partition:p0,p1 data:TableFullScan",
           "└─TableFullScan 10000.00 cop[tikv] table:t9 keep order:false, stats:pseudo"
         ]
+      },
+      {
+        "SQL": "explain format = 'brief' select * from t10 where a between 0 AND 15218001646226433652",
+        "Result": [
+          "TableReader 250.00 root partition:all data:Selection",
+          "└─Selection 250.00 cop[tikv]  ge(test_partition.t10.a, 0), le(test_partition.t10.a, 15218001646226433652)",
+          "  └─TableFullScan 10000.00 cop[tikv] table:t10 keep order:false, stats:pseudo"
+        ]
       }
     ]
   },