Verification of persisted data

CHANGELOG-3.5.md

@@ -9,7 +9,7 @@ The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.
 <hr>
 
 
-## v3.5.0 (2021 TBD)
+## v3.5.0 (2021-06)
 
 See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.5.0) and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/) for any breaking changes.
 
@@ -160,6 +160,7 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 - Add [`--socket-reuse-address`](https://github.com/etcd-io/etcd/pull/12702) flag
   - Setting this flag enables `SO_REUSEADDR` which allows binding to an address in `TIME_WAIT` state, improving etcd restart time.
 - Reduce [around 30% memory allocation by logging range response size without marshal](https://github.com/etcd-io/etcd/pull/12871).
+- `ETCD_VERIFY="all"` enviroment triggers [additional verification of consistency](https://github.com/etcd-io/etcd/pull/) of etcd data-dir files. 
 ### Package `runtime`
 
 - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214).

etcdctl/ctlv2/command/backup_command.go

@@ -18,7 +18,6 @@ import (
 	"log"
 	"os"
 	"path"
-	"path/filepath"
 	"regexp"
 	"time"
 
@@ -28,6 +27,7 @@ import (
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
 	"go.etcd.io/etcd/raft/v3/raftpb"
+	"go.etcd.io/etcd/server/v3/datadir"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
@@ -93,19 +93,22 @@ func handleBackup(c *cli.Context) error {
 	lg := zap.NewExample()
 
 	withV3 := c.Bool("with-v3")
-	srcSnap := filepath.Join(c.String("data-dir"), "member", "snap")
-	destSnap := filepath.Join(c.String("backup-dir"), "member", "snap")
+	srcDir := c.String("data-dir")
+	destDir := c.String("backup-dir")
+
+	srcSnap := datadir.ToSnapDir(srcDir)
+	destSnap := datadir.ToSnapDir(destDir)
 
 	if c.String("wal-dir") != "" {
 		srcWAL = c.String("wal-dir")
 	} else {
-		srcWAL = filepath.Join(c.String("data-dir"), "member", "wal")
+		srcWAL = datadir.ToWalDir(srcDir)
 	}
 
 	if c.String("backup-wal-dir") != "" {
 		destWAL = c.String("backup-wal-dir")
 	} else {
-		destWAL = filepath.Join(c.String("backup-dir"), "member", "wal")
+		destWAL = datadir.ToWalDir(destDir)
 	}
 
 	if err := fileutil.CreateDirAll(destSnap); err != nil {
@@ -116,8 +119,8 @@ func handleBackup(c *cli.Context) error {
 
 	walsnap := saveSnap(lg, destSnap, srcSnap, &desired)
 	metadata, state, ents := loadWAL(srcWAL, walsnap, withV3)
-	destDbPath := filepath.Join(destSnap, "db")
-	saveDB(lg, destDbPath, filepath.Join(srcSnap, "db"), state.Commit, &desired, withV3)
+	destDbPath := datadir.ToBackendFileName(destDir)
+	saveDB(lg, destDbPath, datadir.ToBackendFileName(srcDir), state.Commit, &desired, withV3)
 
 	neww, err := wal.Create(zap.NewExample(), destWAL, pbutil.MustMarshal(&metadata))
 	if err != nil {

etcdctl/snapshot/v3_snapshot.go

@@ -41,6 +41,7 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
 	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
 	"go.etcd.io/etcd/server/v3/mvcc/backend"
+	"go.etcd.io/etcd/server/v3/verify"
 	"go.etcd.io/etcd/server/v3/wal"
 	"go.etcd.io/etcd/server/v3/wal/walpb"
 	"go.uber.org/zap"
@@ -276,7 +277,11 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error {
 		zap.String("snap-dir", s.snapDir),
 	)
 
-	return nil
+	return verify.VerifyIfEnabled(verify.Config{
+		ExactIndex: true,
+		Logger:     s.lg,
+		DataDir:    dataDir,
+	})
 }
 
 func (s *v3Manager) outDbPath() string {

server/config/config.go

@@ -25,6 +25,7 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/transport"
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/netutil"
+	"go.etcd.io/etcd/server/v3/datadir"
 
 	bolt "go.etcd.io/bbolt"
 	"go.uber.org/zap"
@@ -274,13 +275,13 @@ func (c *ServerConfig) advertiseMatchesCluster() error {
 	return fmt.Errorf("failed to resolve %s to match --initial-cluster=%s (%v)", apStr, umap.String(), err)
 }
 
-func (c *ServerConfig) MemberDir() string { return filepath.Join(c.DataDir, "member") }
+func (c *ServerConfig) MemberDir() string { return datadir.ToMemberDir(c.DataDir) }
 
 func (c *ServerConfig) WALDir() string {
 	if c.DedicatedWALDir != "" {
 		return c.DedicatedWALDir
 	}
-	return filepath.Join(c.MemberDir(), "wal")
+	return datadir.ToWalDir(c.DataDir)
 }
 
 func (c *ServerConfig) SnapDir() string { return filepath.Join(c.MemberDir(), "snap") }
@@ -324,4 +325,4 @@ func (c *ServerConfig) BootstrapTimeoutEffective() time.Duration {
 	return time.Second
 }
 
-func (c *ServerConfig) BackendPath() string { return filepath.Join(c.SnapDir(), "db") }
+func (c *ServerConfig) BackendPath() string { return datadir.ToBackendFileName(c.DataDir) }

server/datadir/datadir.go

@@ -0,0 +1,40 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datadir
+
+import "path/filepath"
+
+const (
+	memberDirSegment   = "member"
+	snapDirSegment     = "snap"
+	walDirSegment      = "wal"
+	backendFileSegment = "db"
+)
+
+func ToBackendFileName(dataDir string) string {
+	return filepath.Join(ToSnapDir(dataDir), backendFileSegment)
+}
+
+func ToSnapDir(dataDir string) string {
+	return filepath.Join(ToMemberDir(dataDir), snapDirSegment)
+}
+
+func ToWalDir(dataDir string) string {
+	return filepath.Join(ToMemberDir(dataDir), walDirSegment)
+}
+
+func ToMemberDir(dataDir string) string {
+	return filepath.Join(dataDir, memberDirSegment)
+}

server/datadir/datadir_test.go

@@ -0,0 +1,33 @@
+package datadir_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/etcd/server/v3/datadir"
+)
+
+func TestToBackendFileName(t *testing.T) {
+	result := datadir.ToBackendFileName("/dir/data-dir")
+	assert.Equal(t, "/dir/data-dir/member/snap/db", result)
+}
+
+func TestToMemberDir(t *testing.T) {
+	result := datadir.ToMemberDir("/dir/data-dir")
+	assert.Equal(t, "/dir/data-dir/member", result)
+}
+
+func TestToSnapDir(t *testing.T) {
+	result := datadir.ToSnapDir("/dir/data-dir")
+	assert.Equal(t, "/dir/data-dir/member/snap", result)
+}
+
+func TestToWalDir(t *testing.T) {
+	result := datadir.ToWalDir("/dir/data-dir")
+	assert.Equal(t, "/dir/data-dir/member/wal", result)
+}
+
+func TestToWalDirSlash(t *testing.T) {
+	result := datadir.ToWalDir("/dir/data-dir/")
+	assert.Equal(t, "/dir/data-dir/member/wal", result)
+}

server/datadir/doc.go

@@ -0,0 +1,17 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datadir
+
+// datadir contains functions to navigate file-layout of etcd data-directory.

server/embed/etcd.go

@@ -42,6 +42,7 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v2v3"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc"
+	"go.etcd.io/etcd/server/v3/verify"
 
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/soheilhy/cmux"
@@ -338,6 +339,11 @@ func (e *Etcd) Close() {
 	lg.Info("closing etcd server", fields...)
 	defer func() {
 		lg.Info("closed etcd server", fields...)
+		verify.MustVerifyIfEnabled(verify.Config{
+			Logger:     lg,
+			DataDir:    e.cfg.Dir,
+			ExactIndex: false,
+		})
 		lg.Sync()
 	}()
 
@@ -513,7 +519,6 @@ func (e *Etcd) servePeers() (err error) {
 			e.cfg.logger.Info(
 				"cmux::serve",
 				zap.String("address", u),
-				zap.String("cmuxp", fmt.Sprintf("%p", m)),
 			)
 			return m.Serve()
 		}
@@ -524,16 +529,13 @@ func (e *Etcd) servePeers() (err error) {
 			e.cfg.logger.Info(
 				"stopping serving peer traffic",
 				zap.String("address", u),
-				zap.String("cmuxp", fmt.Sprintf("%p", m)),
 			)
 			stopServers(ctx, &servers{secure: peerTLScfg != nil, grpc: gs, http: srv})
 			e.cfg.logger.Info(
 				"stopped serving peer traffic",
 				zap.String("address", u),
-				zap.String("cmuxp", fmt.Sprintf("%p", m)),
 			)
 			m.Close()
-			e.cfg.logger.Info("Closed", zap.String("cmuxp", fmt.Sprintf("%p", m)))
 			return nil
 		}
 	}

server/verify/doc.go

@@ -0,0 +1,20 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package verify
+
+// verify package is analyzing persistent state of etcd to find potential
+// inconsistencies.
+// In particular it covers cross-checking between different aspacts of etcd
+// storage like WAL & Backend.