master: add etcd compaction and quota config (#1521) (#1534)

dm/master/config.go

@@ -37,11 +37,15 @@ import (
 )
 
 const (
-	defaultRPCTimeout          = "30s"
-	defaultNamePrefix          = "dm-master"
-	defaultDataDirPrefix       = "default"
-	defaultPeerUrls            = "http://127.0.0.1:8291"
-	defaultInitialClusterState = embed.ClusterStateFlagNew
+	defaultRPCTimeout              = "30s"
+	defaultNamePrefix              = "dm-master"
+	defaultDataDirPrefix           = "default"
+	defaultPeerUrls                = "http://127.0.0.1:8291"
+	defaultInitialClusterState     = embed.ClusterStateFlagNew
+	defaultAutoCompactionMode      = "periodic"
+	defaultAutoCompactionRetention = "1h"
+	defaultQuotaBackendBytes       = 2 * 1024 * 1024 * 1024 // 2GB
+	quotaBackendBytesLowerBound    = 500 * 1024 * 1024      // 500MB
 )
 
 var (
@@ -75,6 +79,9 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.PeerUrls, "peer-urls", defaultPeerUrls, "URLs for peer traffic")
 	fs.StringVar(&cfg.AdvertisePeerUrls, "advertise-peer-urls", "", `advertise URLs for peer traffic (default "${peer-urls}")`)
 	fs.StringVar(&cfg.Join, "join", "", `join to an existing cluster (usage: cluster's "${master-addr}" list, e.g. "127.0.0.1:8261,127.0.0.1:18261"`)
+	fs.StringVar(&cfg.AutoCompactionMode, "auto-compaction-mode", defaultAutoCompactionMode, `etcd's auto-compaction-mode, either 'periodic' or 'revision'`)
+	fs.StringVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", defaultAutoCompactionRetention, `etcd's auto-compaction-retention, accept values like '5h' or '5' (5 hours in 'periodic' mode or 5 revisions in 'revision')`)
+	fs.Int64Var(&cfg.QuotaBackendBytes, "quota-backend-bytes", defaultQuotaBackendBytes, `etcd's storage quota in bytes`)
 
 	fs.StringVar(&cfg.SSLCA, "ssl-ca", "", "path of file that contains list of trusted SSL CAs for connection")
 	fs.StringVar(&cfg.SSLCert, "ssl-cert", "", "path of file that contains X509 certificate in PEM format for connection")
@@ -108,13 +115,16 @@ type Config struct {
 	// etcd relative config items
 	// NOTE: we use `MasterAddr` to generate `ClientUrls` and `AdvertiseClientUrls`
 	// NOTE: more items will be add when adding leader election
-	Name                string `toml:"name" json:"name"`
-	DataDir             string `toml:"data-dir" json:"data-dir"`
-	PeerUrls            string `toml:"peer-urls" json:"peer-urls"`
-	AdvertisePeerUrls   string `toml:"advertise-peer-urls" json:"advertise-peer-urls"`
-	InitialCluster      string `toml:"initial-cluster" json:"initial-cluster"`
-	InitialClusterState string `toml:"initial-cluster-state" json:"initial-cluster-state"`
-	Join                string `toml:"join" json:"join"` // cluster's client address (endpoints), not peer address
+	Name                    string `toml:"name" json:"name"`
+	DataDir                 string `toml:"data-dir" json:"data-dir"`
+	PeerUrls                string `toml:"peer-urls" json:"peer-urls"`
+	AdvertisePeerUrls       string `toml:"advertise-peer-urls" json:"advertise-peer-urls"`
+	InitialCluster          string `toml:"initial-cluster" json:"initial-cluster"`
+	InitialClusterState     string `toml:"initial-cluster-state" json:"initial-cluster-state"`
+	Join                    string `toml:"join" json:"join"` // cluster's client address (endpoints), not peer address
+	AutoCompactionMode      string `toml:"auto-compaction-mode" json:"auto-compaction-mode"`
+	AutoCompactionRetention string `toml:"auto-compaction-retention" json:"auto-compaction-retention"`
+	QuotaBackendBytes       int64  `toml:"quota-backend-bytes" json:"quota-backend-bytes"`
 
 	// directory path used to store source config files when upgrading from v1.0.x.
 	// if this path set, DM-master leader will try to upgrade from v1.0.x to the current version.
@@ -301,6 +311,13 @@ func (c *Config) adjust() error {
 		c.Join = utils.WrapSchemes(c.Join, c.SSLCA != "")
 	}
 
+	if c.QuotaBackendBytes < quotaBackendBytesLowerBound {
+		log.L().Warn("quota-backend-bytes is too low, will adjust it",
+			zap.Int64("from", c.QuotaBackendBytes),
+			zap.Int64("to", quotaBackendBytesLowerBound))
+		c.QuotaBackendBytes = quotaBackendBytesLowerBound
+	}
+
 	return err
 }
 
@@ -345,6 +362,9 @@ func (c *Config) genEmbedEtcdConfig(cfg *embed.Config) (*embed.Config, error) {
 
 	cfg.InitialCluster = c.InitialCluster
 	cfg.ClusterState = c.InitialClusterState
+	cfg.AutoCompactionMode = c.AutoCompactionMode
+	cfg.AutoCompactionRetention = c.AutoCompactionRetention
+	cfg.QuotaBackendBytes = c.QuotaBackendBytes
 
 	err = cfg.Validate() // verify & trigger the builder
 	if err != nil {

dm/master/config_test.go

@@ -191,6 +191,9 @@ func (t *testConfigSuite) TestGenEmbedEtcdConfig(c *check.C) {
 	c.Assert(etcdCfg.APUrls, check.DeepEquals, []url.URL{{Scheme: "http", Host: "127.0.0.1:8291"}})
 	c.Assert(etcdCfg.InitialCluster, check.DeepEquals, fmt.Sprintf("dm-master-%s=http://127.0.0.1:8291", hostname))
 	c.Assert(etcdCfg.ClusterState, check.Equals, embed.ClusterStateFlagExisting)
+	c.Assert(etcdCfg.AutoCompactionMode, check.Equals, "periodic")
+	c.Assert(etcdCfg.AutoCompactionRetention, check.Equals, "1h")
+	c.Assert(etcdCfg.QuotaBackendBytes, check.Equals, int64(2*1024*1024*1024))
 
 	cfg2 := *cfg1
 	cfg2.MasterAddr = "127.0.0.1\n:8261"

dm/master/server_test.go

@@ -1771,3 +1771,43 @@ func createTableInfo(c *check.C, p *parser.Parser, se sessionctx.Context, tableI
 	}
 	return info
 }
+
+type testEtcd struct {
+}
+
+var _ = check.Suite(&testEtcd{})
+
+func (t *testEtcd) TestEtcdAutoCompaction(c *check.C) {
+	cfg := NewConfig()
+	c.Assert(cfg.Parse([]string{"-config=./dm-master.toml"}), check.IsNil)
+
+	cfg.DataDir = c.MkDir()
+	cfg.MasterAddr = tempurl.Alloc()[len("http://"):]
+	cfg.AutoCompactionRetention = "1s"
+
+	ctx, cancel := context.WithCancel(context.Background())
+	s := NewServer(cfg)
+	c.Assert(s.Start(ctx), check.IsNil)
+
+	etcdCli, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{cfg.MasterAddr},
+	})
+	c.Assert(err, check.IsNil)
+
+	for i := 0; i < 100; i++ {
+		_, err = etcdCli.Put(ctx, "key", fmt.Sprintf("%03d", i))
+		c.Assert(err, check.IsNil)
+	}
+	time.Sleep(3 * time.Second)
+	resp, err := etcdCli.Get(ctx, "key")
+	c.Assert(err, check.IsNil)
+
+	utils.WaitSomething(10, time.Second, func() bool {
+		_, err = etcdCli.Get(ctx, "key", clientv3.WithRev(resp.Header.Revision-1))
+		return err != nil
+	})
+	c.Assert(err, check.ErrorMatches, ".*required revision has been compacted.*")
+
+	cancel()
+	s.Close()
+}

tests/all_mode/conf/dm-master.toml

@@ -3,3 +3,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 
 rpc-timeout = "30s"
+auto-compaction-retention = "3s"

tests/dm_syncer/conf/dm-master.toml

@@ -1,4 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
-
+auto-compaction-retention = "3s"

tests/dmctl_advance/conf/dm-master.toml

@@ -2,3 +2,4 @@
 
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/dmctl_basic/conf/get_master1.toml

@@ -15,6 +15,9 @@ advertise-peer-urls = "http://127.0.0.1:8291"
 initial-cluster = "master1=http://127.0.0.1:8291"
 initial-cluster-state = "new"
 join = ""
+auto-compaction-mode = "periodic"
+auto-compaction-retention = "1h"
+quota-backend-bytes = 2147483648
 v1-sources-path = ""
 ssl-ca = ""
 ssl-cert = ""

tests/dmctl_command/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/drop_column_with_index/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/full_mode/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/gtid/conf/dm-master.toml

@@ -3,3 +3,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 
 rpc-timeout = "30s"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master1.toml

@@ -4,3 +4,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 peer-urls = "127.0.0.1:8291"
 initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master2.toml

@@ -3,4 +3,5 @@ name = "master2"
 master-addr = ":8361"
 advertise-addr = "127.0.0.1:8361"
 peer-urls = "127.0.0.1:8292"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292"
+initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master3.toml

@@ -3,4 +3,5 @@ name = "master3"
 master-addr = ":8461"
 advertise-addr = "127.0.0.1:8461"
 peer-urls = "http://127.0.0.1:8293"
-join = "127.0.0.1:8261,127.0.0.1:8361"
+join = "127.0.0.1:8261,127.0.0.1:8361"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master4.toml

@@ -4,3 +4,4 @@ master-addr = ":8561"
 advertise-addr = "127.0.0.1:8561"
 peer-urls = "http://127.0.0.1:8294"
 join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master5.toml

@@ -4,3 +4,4 @@ master-addr = ":8661"
 advertise-addr = "127.0.0.1:8661"
 peer-urls = "http://127.0.0.1:8295"
 join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561"
+auto-compaction-retention = "3s"

tests/ha/conf/dm-master6.toml

@@ -4,3 +4,4 @@ master-addr = ":8761"
 advertise-addr = "127.0.0.1:8761"
 peer-urls = "http://127.0.0.1:8296"
 join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561,127.0.0.1:8661"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master-join1.toml

@@ -4,3 +4,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 peer-urls = "127.0.0.1:8291"
 initial-cluster = "master1=http://127.0.0.1:8291"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master-join2.toml

@@ -3,4 +3,5 @@ name = "master2"
 master-addr = ":8361"
 advertise-addr = "127.0.0.1:8361"
 peer-urls = "http://127.0.0.1:8292"
-join = "127.0.0.1:8261"
+join = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master-join3.toml

@@ -3,4 +3,5 @@ name = "master3"
 master-addr = ":8461"
 advertise-addr = "127.0.0.1:8461"
 peer-urls = "http://127.0.0.1:8293"
-join = "127.0.0.1:8261,127.0.0.1:8361"
+join = "127.0.0.1:8261,127.0.0.1:8361"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master-join4.toml

@@ -3,4 +3,5 @@ name = "master4"
 master-addr = ":8561"
 advertise-addr = "127.0.0.1:8561"
 peer-urls = "http://127.0.0.1:8294"
-join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461"
+join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master-join5.toml

@@ -3,4 +3,5 @@ name = "master5"
 master-addr = ":8661"
 advertise-addr = "127.0.0.1:8661"
 peer-urls = "http://127.0.0.1:8295"
-join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561"
+join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master1.toml

@@ -4,3 +4,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 peer-urls = "127.0.0.1:8291"
 initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master2.toml

@@ -3,4 +3,5 @@ name = "master2"
 master-addr = ":8361"
 advertise-addr = "127.0.0.1:8361"
 peer-urls = "127.0.0.1:8292"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293"
+initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293"
+auto-compaction-retention = "3s"

tests/ha_cases/conf/dm-master3.toml

@@ -3,4 +3,5 @@ name = "master3"
 master-addr = ":8461"
 advertise-addr = "127.0.0.1:8461"
 peer-urls = "127.0.0.1:8293"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293"
+initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master-join1.toml

@@ -4,3 +4,4 @@ master-addr = ":8261"
 advertise-addr = "localhost:8261"
 peer-urls = "http://localhost:8291"
 join = "localhost:8461,localhost:8561,localhost:8661"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master1.toml

@@ -3,4 +3,5 @@ name = "master1"
 master-addr = ":8261"
 advertise-addr = "localhost:8261"
 peer-urls = "localhost:8291"
-initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master2.toml

@@ -4,3 +4,4 @@ master-addr = ":8361"
 advertise-addr = "localhost:8361"
 peer-urls = "localhost:8292"
 initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master3.toml

@@ -4,3 +4,4 @@ master-addr = ":8461"
 advertise-addr = "localhost:8461"
 peer-urls = "localhost:8293"
 initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master4.toml

@@ -4,3 +4,4 @@ master-addr = ":8561"
 advertise-addr = "localhost:8561"
 peer-urls = "localhost:8294"
 initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+auto-compaction-retention = "3s"

tests/ha_master/conf/dm-master5.toml

@@ -4,3 +4,4 @@ master-addr = ":8661"
 advertise-addr = "localhost:8661"
 peer-urls = "localhost:8295"
 initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"
+auto-compaction-retention = "3s"

tests/handle_error/conf/dm-master.toml

@@ -4,3 +4,4 @@ master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 peer-urls = "127.0.0.1:8291"
 initial-cluster = "master1=http://127.0.0.1:8291"
+auto-compaction-retention = "3s"

tests/http_apis/conf/dm-master.toml

@@ -2,3 +2,4 @@
 name = "master1"
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/import_goroutine_leak/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/import_v10x/conf/dm-master.toml

@@ -2,3 +2,4 @@
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
 v1-sources-path = "./v1-sources"
+auto-compaction-retention = "3s"

tests/incremental_mode/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/initial_unit/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/load_interrupt/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/many_tables/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/online_ddl/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/only_dml/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/print_status/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/relay_interrupt/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/retry_cancel/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/safe_mode/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sequence_safe_mode/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sequence_sharding/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sequence_sharding_optimistic/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sequence_sharding_removemeta/conf/dm-master.toml

@@ -1,3 +1,4 @@
 # Master Configuration.
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"