*: Support List default partition (#41537)

close #20679

br/tests/lightning_partitioned-table/data/partitioned.defaultlist-schema.sql

@@ -0,0 +1 @@
+create table `defaultlist` (a int, b varchar(16), KEY key_b (`b`)) partition by list(a) (partition p1 values in (1,4,8),partition p2 values in (32,default), partition p3 values in (262144,65536));

br/tests/lightning_partitioned-table/data/partitioned.defaultlist.sql

@@ -0,0 +1,10 @@
+insert into defaultlist values
+    (268435456, '268435456'),
+    (1, 'adgagdadgagag'),
+    (262144, 'gadgagaha'),
+    (32, '32'),
+    (4, 'hahaha'),
+    (65536, 'luck dog'),
+    (8388608, 'heyhey'),
+    (0, '999');
+

br/tests/lightning_partitioned-table/data/partitioned.list-schema.sql

@@ -0,0 +1 @@
+create table `list` (a int, b varchar(16), KEY key_b (`b`)) partition by list(a) (partition p1 values in (1,4,8),partition p2 values in (32,8388608,268435456), partition p3 values in (262144,0,65536));

br/tests/lightning_partitioned-table/data/partitioned.list.sql

@@ -0,0 +1,10 @@
+insert into list values
+    (268435456, '268435456'),
+    (1, 'adgagdadgagag'),
+    (262144, 'gadgagaha'),
+    (32, '32'),
+    (4, 'hahaha'),
+    (65536, 'luck dog'),
+    (8388608, 'heyhey'),
+    (0, '999');
+

br/tests/lightning_partitioned-table/data/partitioned.range-schema.sql

@@ -0,0 +1 @@
+create table `range` (a int, b varchar(16), KEY key_b (`b`)) partition by range(a) (partition pNeg values less than (0), partition pMax values less than (maxvalue));

br/tests/lightning_partitioned-table/data/partitioned.range.sql

@@ -0,0 +1,10 @@
+insert into range values
+    (268435456, '268435456'),
+    (1, 'adgagdadgagag'),
+    (262144, 'gadgagaha'),
+    (32, '32'),
+    (4, 'hahaha'),
+    (65536, 'luck dog'),
+    (8388608, 'heyhey'),
+    (0, '999');
+

br/tests/lightning_partitioned-table/run.sh

@@ -25,6 +25,8 @@ for BACKEND in tidb local; do
 
     run_sql 'DROP DATABASE IF EXISTS partitioned;'
 
+    # DEFAULT List partitioning needs to be enabled for defaultlist table
+    run_sql 'set @@global.tidb_enable_default_list_partition=1;'
     run_lightning --backend $BACKEND
 
     run_sql 'SELECT count(1), sum(a) FROM partitioned.a;'
@@ -33,4 +35,25 @@ for BACKEND in tidb local; do
 
     run_sql "SHOW TABLE STATUS FROM partitioned WHERE name = 'a';"
     check_contains 'Create_options: partitioned'
+
+    run_sql 'SELECT count(1), sum(a) FROM partitioned.range;'
+    check_contains 'count(1): 8'
+    check_contains 'sum(a): 277151781'
+
+    run_sql "SHOW TABLE STATUS FROM partitioned WHERE name = 'range';"
+    check_contains 'Create_options: partitioned'
+
+    run_sql 'SELECT count(1), sum(a) FROM partitioned.list;'
+    check_contains 'count(1): 8'
+    check_contains 'sum(a): 277151781'
+
+    run_sql "SHOW TABLE STATUS FROM partitioned WHERE name = 'list';"
+    check_contains 'Create_options: partitioned'
+
+    run_sql 'SELECT count(1), sum(a) FROM partitioned.defaultlist;'
+    check_contains 'count(1): 8'
+    check_contains 'sum(a): 277151781'
+
+    run_sql "SHOW TABLE STATUS FROM partitioned WHERE name = 'defaultlist';"
+    check_contains 'Create_options: partitioned'
 done

ddl/ddl_api.go

@@ -4119,6 +4119,13 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec *
 	if err != nil {
 		return errors.Trace(err)
 	}
+	if pi.Type == model.PartitionTypeList {
+		// TODO: make sure that checks in ddl_api and ddl_worker is the same.
+		err = checkAddListPartitions(meta)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
 	if err := d.assignPartitionIDs(partInfo.Definitions); err != nil {
 		return errors.Trace(err)
 	}
@@ -7479,6 +7486,11 @@ func BuildAddedPartitionInfo(ctx sessionctx.Context, meta *model.TableInfo, spec
 		if len(spec.PartDefinitions) == 0 {
 			return nil, ast.ErrPartitionsMustBeDefined.GenWithStackByArgs(meta.Partition.Type)
 		}
+		err := checkListPartitions(ctx, spec.PartDefinitions)
+		if err != nil {
+			return nil, err
+		}
+
 	case model.PartitionTypeRange:
 		if spec.Tp == ast.AlterTableAddLastPartition {
 			err := buildAddedPartitionDefs(ctx, meta, spec)
@@ -7555,6 +7567,12 @@ func checkAndGetColumnsTypeAndValuesMatch(ctx sessionctx.Context, colTypes []typ
 			valStrings = append(valStrings, partitionMaxValue)
 			continue
 		}
+		if d, ok := colExpr.(*ast.DefaultExpr); ok {
+			if d.Name != nil {
+				return nil, dbterror.ErrWrongTypeColumnValue.GenWithStackByArgs()
+			}
+			continue
+		}
 		colType := colTypes[i]
 		val, err := expression.EvalAstExpr(ctx, colExpr)
 		if err != nil {

ddl/partition.go

@@ -338,36 +338,63 @@ func rollbackAddingPartitionInfo(tblInfo *model.TableInfo) ([]int64, []string, [
 	return physicalTableIDs, partNames, rollbackBundles
 }
 
-// checkAddPartitionValue values less than value must be strictly increasing for each partition.
-func checkAddPartitionValue(meta *model.TableInfo, part *model.PartitionInfo) error {
-	if meta.Partition.Type == model.PartitionTypeRange && len(meta.Partition.Columns) == 0 {
-		newDefs, oldDefs := part.Definitions, meta.Partition.Definitions
-		rangeValue := oldDefs[len(oldDefs)-1].LessThan[0]
-		if strings.EqualFold(rangeValue, "MAXVALUE") {
-			return errors.Trace(dbterror.ErrPartitionMaxvalue)
-		}
-
-		currentRangeValue, err := strconv.Atoi(rangeValue)
-		if err != nil {
-			return errors.Trace(err)
+// Check if current table already contains DEFAULT list partition
+func checkAddListPartitions(tblInfo *model.TableInfo) error {
+	for i := range tblInfo.Partition.Definitions {
+		for j := range tblInfo.Partition.Definitions[i].InValues {
+			for _, val := range tblInfo.Partition.Definitions[i].InValues[j] {
+				if val == "DEFAULT" { // should already be normalized
+					return dbterror.ErrGeneralUnsupportedDDL.GenWithStackByArgs("ADD List partition, already contains DEFAULT partition. Please use REORGANIZE PARTITION instead")
+				}
+			}
 		}
+	}
+	return nil
+}
 
-		for i := 0; i < len(newDefs); i++ {
-			ifMaxvalue := strings.EqualFold(newDefs[i].LessThan[0], "MAXVALUE")
-			if ifMaxvalue && i == len(newDefs)-1 {
-				return nil
-			} else if ifMaxvalue && i != len(newDefs)-1 {
+// checkAddPartitionValue check add Partition Values,
+// For Range: values less than value must be strictly increasing for each partition.
+// For List: if a Default partition exists,
+//
+//	no ADD partition can be allowed
+//	(needs reorganize partition instead).
+func checkAddPartitionValue(meta *model.TableInfo, part *model.PartitionInfo) error {
+	switch meta.Partition.Type {
+	case model.PartitionTypeRange:
+		if len(meta.Partition.Columns) == 0 {
+			newDefs, oldDefs := part.Definitions, meta.Partition.Definitions
+			rangeValue := oldDefs[len(oldDefs)-1].LessThan[0]
+			if strings.EqualFold(rangeValue, "MAXVALUE") {
 				return errors.Trace(dbterror.ErrPartitionMaxvalue)
 			}
 
-			nextRangeValue, err := strconv.Atoi(newDefs[i].LessThan[0])
+			currentRangeValue, err := strconv.Atoi(rangeValue)
 			if err != nil {
 				return errors.Trace(err)
 			}
-			if nextRangeValue <= currentRangeValue {
-				return errors.Trace(dbterror.ErrRangeNotIncreasing)
+
+			for i := 0; i < len(newDefs); i++ {
+				ifMaxvalue := strings.EqualFold(newDefs[i].LessThan[0], "MAXVALUE")
+				if ifMaxvalue && i == len(newDefs)-1 {
+					return nil
+				} else if ifMaxvalue && i != len(newDefs)-1 {
+					return errors.Trace(dbterror.ErrPartitionMaxvalue)
+				}
+
+				nextRangeValue, err := strconv.Atoi(newDefs[i].LessThan[0])
+				if err != nil {
+					return errors.Trace(err)
+				}
+				if nextRangeValue <= currentRangeValue {
+					return errors.Trace(dbterror.ErrRangeNotIncreasing)
+				}
+				currentRangeValue = nextRangeValue
 			}
-			currentRangeValue = nextRangeValue
+		}
+	case model.PartitionTypeList:
+		err := checkAddListPartitions(meta)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -452,6 +479,30 @@ func storeHasEngineTiFlashLabel(store *metapb.Store) bool {
 	return false
 }
 
+func checkListPartitions(ctx sessionctx.Context, defs []*ast.PartitionDefinition) error {
+	for _, def := range defs {
+		valIn, ok := def.Clause.(*ast.PartitionDefinitionClauseIn)
+		if !ok {
+			switch def.Clause.(type) {
+			case *ast.PartitionDefinitionClauseLessThan:
+				return ast.ErrPartitionWrongValues.GenWithStackByArgs("RANGE", "LESS THAN")
+			case *ast.PartitionDefinitionClauseNone:
+				return ast.ErrPartitionRequiresValues.GenWithStackByArgs("LIST", "IN")
+			default:
+				return dbterror.ErrUnsupportedCreatePartition.GenWithStack("Only VALUES IN () is supported for LIST partitioning")
+			}
+		}
+		if !ctx.GetSessionVars().EnableDefaultListPartition {
+			for _, val := range valIn.Values {
+				if _, ok := val[0].(*ast.DefaultExpr); ok {
+					return dbterror.ErrUnsupportedCreatePartition.GenWithStack("VALUES IN (DEFAULT) is not supported, please use 'tidb_enable_default_list_partition'")
+				}
+			}
+		}
+	}
+	return nil
+}
+
 // buildTablePartitionInfo builds partition info and checks for some errors.
 func buildTablePartitionInfo(ctx sessionctx.Context, s *ast.PartitionOptions, tbInfo *model.TableInfo) error {
 	if s == nil {
@@ -470,6 +521,12 @@ func buildTablePartitionInfo(ctx sessionctx.Context, s *ast.PartitionOptions, tb
 	case model.PartitionTypeList:
 		// Partition by list is enabled only when tidb_enable_list_partition is 'ON'.
 		enable = ctx.GetSessionVars().EnableListTablePartition
+		if enable {
+			err := checkListPartitions(ctx, s.Definitions)
+			if err != nil {
+				return err
+			}
+		}
 	case model.PartitionTypeHash, model.PartitionTypeKey:
 		// Partition by hash and key is enabled by default.
 		if s.Sub != nil {
@@ -1344,6 +1401,12 @@ func checkPartitionValuesIsInt(ctx sessionctx.Context, defName interface{}, expr
 		if _, ok := exp.(*ast.MaxValueExpr); ok {
 			continue
 		}
+		if d, ok := exp.(*ast.DefaultExpr); ok {
+			if d.Name != nil {
+				return dbterror.ErrPartitionConstDomain.GenWithStackByArgs()
+			}
+			continue
+		}
 		val, err := expression.EvalAstExpr(ctx, exp)
 		if err != nil {
 			return err
@@ -1593,12 +1656,26 @@ func formatListPartitionValue(ctx sessionctx.Context, tblInfo *model.TableInfo)
 		}
 	}
 
+	haveDefault := false
 	exprStrs := make([]string, 0)
 	inValueStrs := make([]string, 0, mathutil.Max(len(pi.Columns), 1))
 	for i := range defs {
+	inValuesLoop:
 		for j, vs := range defs[i].InValues {
 			inValueStrs = inValueStrs[:0]
 			for k, v := range vs {
+				// if DEFAULT would be given as string, like "DEFAULT",
+				// it would be stored as "'DEFAULT'",
+				if strings.EqualFold(v, "DEFAULT") && k == 0 && len(vs) == 1 {
+					if haveDefault {
+						return nil, dbterror.ErrMultipleDefConstInListPart
+					}
+					haveDefault = true
+					continue inValuesLoop
+				}
+				if strings.EqualFold(v, "MAXVALUE") {
+					return nil, errors.Trace(dbterror.ErrMaxvalueInValuesIn)
+				}
 				expr, err := expression.ParseSimpleExprCastWithTableInfo(ctx, v, &model.TableInfo{}, colTps[k])
 				if err != nil {
 					return nil, errors.Trace(err)
@@ -3679,7 +3756,7 @@ func checkPartitionExprAllowed(_ sessionctx.Context, tb *model.TableInfo, e ast.
 			return errors.Trace(checkNoTimestampArgs(tb, v.V))
 		}
 	case *ast.ColumnNameExpr, *ast.ParenthesesExpr, *driver.ValueExpr, *ast.MaxValueExpr,
-		*ast.TimeUnitExpr:
+		*ast.DefaultExpr, *ast.TimeUnitExpr:
 		return nil
 	}
 	return errors.Trace(dbterror.ErrPartitionFunctionIsNotAllowed)
@@ -3913,24 +3990,32 @@ func AppendPartitionDefs(partitionInfo *model.PartitionInfo, buf *bytes.Buffer,
 			}
 			fmt.Fprintf(buf, " VALUES LESS THAN (%s)", strings.Join(lessThans, ","))
 		} else if partitionInfo.Type == model.PartitionTypeList {
-			values := bytes.NewBuffer(nil)
-			for j, inValues := range def.InValues {
-				if j > 0 {
-					values.WriteString(",")
-				}
-				if len(inValues) > 1 {
-					values.WriteString("(")
-					tmpVals := make([]string, len(inValues))
-					for idx, v := range inValues {
-						tmpVals[idx] = hexIfNonPrint(v)
+			if len(def.InValues) == 0 {
+				fmt.Fprintf(buf, " DEFAULT")
+			} else if len(def.InValues) == 1 &&
+				len(def.InValues[0]) == 1 &&
+				strings.EqualFold(def.InValues[0][0], "DEFAULT") {
+				fmt.Fprintf(buf, " DEFAULT")
+			} else {
+				values := bytes.NewBuffer(nil)
+				for j, inValues := range def.InValues {
+					if j > 0 {
+						values.WriteString(",")
+					}
+					if len(inValues) > 1 {
+						values.WriteString("(")
+						tmpVals := make([]string, len(inValues))
+						for idx, v := range inValues {
+							tmpVals[idx] = hexIfNonPrint(v)
+						}
+						values.WriteString(strings.Join(tmpVals, ","))
+						values.WriteString(")")
+					} else if len(inValues) == 1 {
+						values.WriteString(hexIfNonPrint(inValues[0]))
 					}
-					values.WriteString(strings.Join(tmpVals, ","))
-					values.WriteString(")")
-				} else if len(inValues) == 1 {
-					values.WriteString(hexIfNonPrint(inValues[0]))
 				}
+				fmt.Fprintf(buf, " VALUES IN (%s)", values.String())
 			}
-			fmt.Fprintf(buf, " VALUES IN (%s)", values.String())
 		}
 		if len(def.Comment) > 0 {
 			fmt.Fprintf(buf, " COMMENT '%s'", format.OutputFormat(def.Comment))

errors.toml

@@ -991,6 +991,11 @@ error = '''
 Partition column values of incorrect type
 '''
 
+["ddl:1656"]
+error = '''
+Cannot use MAXVALUE as value in VALUES IN
+'''
+
 ["ddl:1659"]
 error = '''
 Field '%-.192s' is of a not allowed type for this type of partitioning

executor/infoschema_reader.go

@@ -937,20 +937,15 @@ func (e *memtableRetriever) setDataFromPartitions(ctx context.Context, sctx sess
 					} else if table.Partition.Type == model.PartitionTypeList {
 						if len(pi.InValues) > 0 {
 							buf := bytes.NewBuffer(nil)
-							if len(pi.InValues[0]) == 1 {
-								for i, vs := range pi.InValues {
-									if i > 0 {
-										buf.WriteString(",")
-									}
-									buf.WriteString(vs[0])
+							for i, vs := range pi.InValues {
+								if i > 0 {
+									buf.WriteString(",")
 								}
-							} else if len(pi.InValues[0]) > 1 {
-								for i, vs := range pi.InValues {
-									if i > 0 {
-										buf.WriteString(",")
-									}
+								if len(vs) != 1 {
 									buf.WriteString("(")
-									buf.WriteString(strings.Join(vs, ","))
+								}
+								buf.WriteString(strings.Join(vs, ","))
+								if len(vs) != 1 {
 									buf.WriteString(")")
 								}
 							}