placement: supports survival preferences (pingcap#40613)

close pingcap#38605

ddl/ddl_api.go

@@ -3022,6 +3022,8 @@ func SetDirectPlacementOpt(placementSettings *model.PlacementSettings, placement
  placementSettings.FollowerConstraints = stringVal
  case ast.PlacementOptionVoterConstraints:
  placementSettings.VoterConstraints = stringVal
+ case ast.PlacementOptionSurvivalPreferences:
+ placementSettings.SurvivalPreferences = stringVal
  default:
  return errors.Trace(errors.New("unknown placement policy option"))
  }

ddl/placement/bundle.go

@@ -29,6 +29,7 @@ import (
  "github.com/pingcap/tidb/tablecodec"
  "github.com/pingcap/tidb/util/codec"
  "golang.org/x/exp/slices"
+ "gopkg.in/yaml.v2"
 )
 
 // Refer to https://github.com/tikv/pd/issues/2701 .
@@ -123,7 +124,13 @@ func NewBundleFromConstraintsOptions(options *model.PlacementSettings) (*Bundle,
  rules = append(rules, rule)
  }
  }
-
+ labels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
+ if err != nil {
+ return nil, err
+ }
+ for _, rule := range rules {
+ rule.LocationLabels = labels
+ }
  return &Bundle{Rules: rules}, nil
 }
 
@@ -155,9 +162,17 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
 
  var rules []*Rule
 
+ locationLabels, err := newLocationLabelsFromSurvivalPreferences(options.SurvivalPreferences)
+ if err != nil {
+ return nil, err
+ }
+
  // in case empty primaryRegion and regions, just return an empty bundle
  if primaryRegion == "" && len(regions) == 0 {
  rules = append(rules, NewRule(Voter, followers+1, NewConstraintsDirect()))
+ for _, rule := range rules {
+ rule.LocationLabels = locationLabels
+ }
  return &Bundle{Rules: rules}, nil
  }
 
@@ -195,6 +210,11 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
  }
  }
 
+ // set location labels
+ for _, rule := range rules {
+ rule.LocationLabels = locationLabels
+ }
+
  return &Bundle{Rules: rules}, nil
 }
 
@@ -223,6 +243,19 @@ func newBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err
  return bundle, err
 }
 
+// newLocationLabelsFromSurvivalPreferences will parse the survival preferences into location labels.
+func newLocationLabelsFromSurvivalPreferences(survivalPreferenceStr string) ([]string, error) {
+ if len(survivalPreferenceStr) > 0 {
+ labels := []string{}
+ err := yaml.UnmarshalStrict([]byte(survivalPreferenceStr), &labels)
+ if err != nil {
+ return nil, ErrInvalidSurvivalPreferenceFormat
+ }
+ return labels, nil
+ }
+ return nil, nil
+}
+
 // NewBundleFromOptions will transform options into the bundle.
 func NewBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err error) {
  bundle, err = newBundleFromOptions(options)
@@ -257,6 +290,15 @@ func (b *Bundle) String() string {
 func (b *Bundle) Tidy() error {
  extraCnt := map[PeerRoleType]int{}
  newRules := b.Rules[:0]
+
+ // One Bundle is from one PlacementSettings, rule share same location labels, so we can use the first rule's location labels.
+ var locationLabels []string
+ for _, rule := range b.Rules {
+ if len(rule.LocationLabels) > 0 {
+ locationLabels = rule.LocationLabels
+ break
+ }
+ }
  for i, rule := range b.Rules {
  // useless Rule
  if rule.Count <= 0 {
@@ -300,6 +342,8 @@ func (b *Bundle) Tidy() error {
  Key: EngineLabelKey,
  Values: []string{EngineLabelTiFlash},
  }},
+ // the merged rule should have the same location labels with the original rules.
+ LocationLabels: locationLabels,
  })
  }
  b.Rules = newRules

ddl/placement/bundle_test.go

@@ -887,6 +887,9 @@ func TestTidy(t *testing.T) {
  bundle.Rules = append(bundle.Rules, rules3...)
  bundle.Rules = append(bundle.Rules, rules4...)
 
+ for _, r := range bundle.Rules {
+ r.LocationLabels = []string{"zone", "host"}
+ }
  chkfunc := func() {
  require.NoError(t, err)
  require.Len(t, bundle.Rules, 3)
@@ -901,6 +904,7 @@ func TestTidy(t *testing.T) {
  Values: []string{EngineLabelTiFlash},
  },
  }, bundle.Rules[2].Constraints)
+ require.Equal(t, []string{"zone", "host"}, bundle.Rules[2].LocationLabels)
  }
  err = bundle.Tidy()
  chkfunc()

ddl/placement/errors.go

@@ -29,6 +29,8 @@ var (
  ErrInvalidConstraintsMapcnt = errors.New("label constraints in map syntax have invalid replicas")
  // ErrInvalidConstraintsFormat is from rule.go.
  ErrInvalidConstraintsFormat = errors.New("invalid label constraints format")
+ // ErrInvalidSurvivalPreferenceFormat is from rule.go.
+ ErrInvalidSurvivalPreferenceFormat = errors.New("survival preference format should be in format [xxx=yyy, ...]")
  // ErrInvalidConstraintsRelicas is from rule.go.
  ErrInvalidConstraintsRelicas = errors.New("label constraints with invalid REPLICAS")
  // ErrInvalidBundleID is from bundle.go.

ddl/placement/rule.go

@@ -45,15 +45,16 @@ type RuleGroupConfig struct {
 
 // Rule is the core placement rule struct. Check https://github.com/tikv/pd/blob/master/server/schedule/placement/rule.go.
 type Rule struct {
- GroupID string `json:"group_id"`
- ID string `json:"id"`
- Index int `json:"index,omitempty"`
- Override bool `json:"override,omitempty"`
- StartKeyHex string `json:"start_key"`
- EndKeyHex string `json:"end_key"`
- Role PeerRoleType `json:"role"`
- Count int `json:"count"`
- Constraints Constraints `json:"label_constraints,omitempty"`
+ GroupID string `json:"group_id"`
+ ID string `json:"id"`
+ Index int `json:"index,omitempty"`
+ Override bool `json:"override,omitempty"`
+ StartKeyHex string `json:"start_key"`
+ EndKeyHex string `json:"end_key"`
+ Role PeerRoleType `json:"role"`
+ Count int `json:"count"`
+ Constraints Constraints `json:"label_constraints,omitempty"`
+ LocationLabels []string `json:"location_labels,omitempty"`
 }
 
 // TiFlashRule extends Rule with other necessary fields.

ddl/placement_policy_test.go

@@ -193,7 +193,8 @@ func TestPlacementPolicy(t *testing.T) {
  "LEARNERS=1 " +
  "LEARNER_CONSTRAINTS=\"[+region=cn-west-1]\" " +
  "FOLLOWERS=3 " +
- "FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"")
+ "FOLLOWER_CONSTRAINTS=\"[+disk=ssd]\"" +
+ "SURVIVAL_PREFERENCES=\"[region, zone]\"")
 
  checkFunc := func(policyInfo *model.PolicyInfo) {
  require.Equal(t, true, policyInfo.ID != 0)
@@ -206,6 +207,7 @@ func TestPlacementPolicy(t *testing.T) {
  require.Equal(t, "[+region=cn-west-1]", policyInfo.LearnerConstraints)
  require.Equal(t, model.StatePublic, policyInfo.State)
  require.Equal(t, "", policyInfo.Schedule)
+ require.Equal(t, "[region, zone]", policyInfo.SurvivalPreferences)
  }
 
  // Check the policy is correctly reloaded in the information schema.
@@ -628,11 +630,16 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
  tk.MustExec("create placement policy x " +
  "FOLLOWERS=2 " +
  "CONSTRAINTS=\"[+disk=ssd]\" ")
+ tk.MustExec("create placement policy z " +
+ "FOLLOWERS=1 " +
+ "SURVIVAL_PREFERENCES=\"[region, zone]\"")
  tk.MustExec("create placement policy y " +
  "FOLLOWERS=3 " +
  "CONSTRAINTS=\"[+region=bj]\" ")
  tk.MustExec("create table t(a int)" +
  "PLACEMENT POLICY=\"x\"")
+ tk.MustExec("create table tt(a int)" +
+ "PLACEMENT POLICY=\"z\"")
  tk.MustQuery("SELECT TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TIDB_PLACEMENT_POLICY_NAME FROM information_schema.Tables WHERE TABLE_SCHEMA='test' AND TABLE_NAME = 't'").Check(testkit.Rows(`def test t x`))
  tk.MustExec("create table t_range_p(id int) placement policy x partition by range(id) (" +
  "PARTITION p0 VALUES LESS THAN (100)," +
@@ -655,7 +662,18 @@ func TestCreateTableWithPlacementPolicy(t *testing.T) {
  require.Equal(t, "y", policyY.Name.L)
  require.Equal(t, true, policyY.ID != 0)
 
- tbl := external.GetTableByName(t, tk, "test", "t")
+ policyZ := testGetPolicyByName(t, tk.Session(), "z", true)
+ require.Equal(t, "z", policyZ.Name.L)
+ require.Equal(t, true, policyZ.ID != 0)
+ require.Equal(t, "[region, zone]", policyZ.SurvivalPreferences)
+
+ tbl := external.GetTableByName(t, tk, "test", "tt")
+ require.NotNil(t, tbl)
+ require.NotNil(t, tbl.Meta().PlacementPolicyRef)
+ require.Equal(t, "z", tbl.Meta().PlacementPolicyRef.Name.L)
+ require.Equal(t, policyZ.ID, tbl.Meta().PlacementPolicyRef.ID)
+
+ tbl = external.GetTableByName(t, tk, "test", "t")
  require.NotNil(t, tbl)
  require.NotNil(t, tbl.Meta().PlacementPolicyRef)
  require.Equal(t, "x", tbl.Meta().PlacementPolicyRef.Name.L)

docs/design/2020-06-24-placement-rules-in-sql.md

@@ -413,6 +413,26 @@ If a table is imported when `tidb_placement_mode='IGNORE'`, and the placement po
 
 The default value for `tidb_placement_mode` is `STRICT`. The option is an enum, and in future we may add support for a `WARN` mode.
 
+
+#### Survival preference
+
+Some important data may need to store multiple copies across availability zones, so as to have high disaster recovery survivability, such as region-level survivability, `SURVIVAL_PREFERENCES` can provide survivability preference settings.
+
+The following example sets a constraint that the data try to satisfy the Survival Preferences setting:
+
+``` sql
+CREATE PLACEMENT POLICY multiregion
+ follower=4
+ PRIMARY_REGION="region1"
+ SURVIVAL_PREFERENCES="[region, zone]";
+```
+
+For tables with this policy set, the data will first meet the survival goal of cross-region data isolation, and then ensure the survival goal of cross-zone data isolation.
+
+> **Note:**
+>
+> `SURVIVAL_PREFERENCES` is equivalent to `LOCATION_LABELS` in PD. For more information, please refer to [Replica scheduling by topology label](https://docs.pingcap.com/tidb/dev/schedule-replicas-by-topology-labels#schedule-replicas-by-topology-labels).
+
 #### Ambiguous and edge cases
 
 The following two policies are not identical:
@@ -491,6 +511,7 @@ In this case the default rules will apply to placement, and the output from `SHO
 - `FOLLOWER_CONSTRAINTS`
 - `LEARNER_CONSTRAINTS`
 - `PLACEMENT POLICY`
+- `SURVIVAL_PREFERENCE`
 
 For a more complex rule using partitions, consider the following example:
 

domain/infosync/BUILD.bazel

@@ -41,6 +41,7 @@ go_library(
  "@com_github_pingcap_errors//:errors",
  "@com_github_pingcap_failpoint//:failpoint",
  "@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_log//:log",
  "@com_github_tikv_client_go_v2//oracle",
  "@com_github_tikv_pd_client//:client",
  "@io_etcd_go_etcd_client_v3//:client",

domain/infosync/placement_manager.go

@@ -21,9 +21,11 @@ import (
  "path"
  "sync"
 
+ "github.com/pingcap/log"
  "github.com/pingcap/tidb/ddl/placement"
  "github.com/pingcap/tidb/util/pdapi"
  clientv3 "go.etcd.io/etcd/client/v3"
+ "go.uber.org/zap"
 )
 
 // PlacementManager manages placement settings
@@ -72,6 +74,7 @@ func (m *PDPlacementManager) PutRuleBundles(ctx context.Context, bundles []*plac
  return err
  }
 
+ log.Debug("Put placement rule bundles", zap.String("rules", string(b)))
  _, err = doRequest(ctx, "PutPlacementRules", m.etcdCli.Endpoints(), path.Join(pdapi.Config, "placement-rule")+"?partial=true", "POST", bytes.NewReader(b))
  return err
 }

parser/ast/ddl.go

@@ -1958,6 +1958,7 @@ const (
  PlacementOptionLearnerConstraints
  PlacementOptionFollowerConstraints
  PlacementOptionVoterConstraints
+ PlacementOptionSurvivalPreferences
  PlacementOptionPolicy
 )
 
@@ -2022,6 +2023,10 @@ func (n *PlacementOption) Restore(ctx *format.RestoreCtx) error {
  ctx.WriteKeyWord("PLACEMENT POLICY ")
  ctx.WritePlain("= ")
  ctx.WriteName(n.StrValue)
+ case PlacementOptionSurvivalPreferences:
+ ctx.WriteKeyWord("SURVIVAL_PREFERENCES ")
+ ctx.WritePlain("= ")
+ ctx.WriteString(n.StrValue)
  default:
  return errors.Errorf("invalid PlacementOption: %d", n.Tp)
  }

parser/misc.go

@@ -707,6 +707,7 @@ var tokenMap = map[string]int{
  "SUBSTRING": substring,
  "SUM": sum,
  "SUPER": super,
+ "SURVIVAL_PREFERENCES": survivalPreferences,
  "SWAPS": swaps,
  "SWITCHES": switchesSym,
  "SYSTEM": system,

parser/model/model.go

@@ -1735,6 +1735,7 @@ type PlacementSettings struct {
  LearnerConstraints string `json:"learner_constraints"`
  FollowerConstraints string `json:"follower_constraints"`
  VoterConstraints string `json:"voter_constraints"`
+ SurvivalPreferences string `json:"survival_preferences"`
 }
 
 // PolicyInfo is the struct to store the placement policy.