Use unixs for "restore source" endpoint

When restoring a backup for both etcd and etcd-events, I observed a port-conflict on port 8002/8003. Switching to unixs:// endpoints (unix domain sockets) is a little more secure, and avoids the port conflict. Due to etcd-io/etcd#12450 the syntax is slightly awkward - we have to make it look like a host and port.
kubernetes-sigs · Nov 2, 2021 · 3da7e75 · 3da7e75
1 parent 12025fd
commit 3da7e75
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 45 deletions.
diff --git a/pkg/etcd/etcdprocess.go b/pkg/etcd/etcdprocess.go
@@ -63,6 +63,9 @@ type etcdProcess struct {
 	BinDir  string
 	DataDir string
 
+	// CurrentDir is the directory in which we launch the binary (cwd)
+	CurrentDir string
+
 	PKIPeersDir   string
 	PKIClientsDir string
 
@@ -197,6 +200,8 @@ func (p *etcdProcess) findMyNode() *protoetcd.EtcdNode {
 
 func (p *etcdProcess) Start() error {
 	c := exec.Command(path.Join(p.BinDir, "etcd"))
+	c.Dir = p.CurrentDir
+
 	if p.ForceNewCluster {
 		c.Args = append(c.Args, "--force-new-cluster")
 	}
@@ -291,6 +296,7 @@ func (p *etcdProcess) Start() error {
 	if err != nil {
 		return fmt.Errorf("error starting etcd: %v", err)
 	}
+	klog.Infof("started etcd with datadir %s; pid=%d", p.DataDir, c.Process.Pid)
 	p.cmd = c
 
 	go func() {
@@ -302,6 +308,11 @@ func (p *etcdProcess) Start() error {
 		p.exitState = processState
 		p.exitError = err
 		p.mutex.Unlock()
+		exitCode := -2
+		if processState != nil {
+			exitCode = processState.ExitCode()
+		}
+		klog.Infof("etcd process exited (datadir %s; pid=%d); exitCode=%d, exitErr=%v", p.DataDir, p.cmd.Process.Pid, exitCode, err)
 	}()
 
 	return nil
@@ -361,12 +372,46 @@ func (p *etcdProcess) NewClient() (*etcdclient.EtcdClient, error) {
 		return nil, fmt.Errorf("unable to find self node %q in %v", p.MyNodeName, p.Cluster.Nodes)
 	}
 
-	clientUrls := me.ClientUrls
+	clientURLs := me.ClientUrls
 	if p.Quarantined {
-		clientUrls = me.QuarantinedClientUrls
+		clientURLs = me.QuarantinedClientUrls
+	}
+
+	// If there are any relative paths, make them absolute for the client.
+	// This is a workaround for https://github.com/etcd-io/etcd/issues/12450
+	for i, clientURL := range clientURLs {
+		scheme := ""
+		if strings.HasPrefix(clientURL, "unix://") {
+			scheme = "unix"
+		} else if strings.HasPrefix(clientURL, "unixs://") {
+			scheme = "unixs"
+		}
+		if scheme == "" {
+			// Not unix domain socket
+			continue
+		}
+
+		if strings.HasPrefix(clientURL, scheme+":///") {
+			// Already absolute
+			continue
+		}
+
+		path := strings.TrimPrefix(clientURL, scheme+"://")
+
+		if p.CurrentDir == "" {
+			return nil, fmt.Errorf("clientURL was set to relative path %q, but process directory was not set", clientURL)
+		}
+
+		if !filepath.IsAbs(p.CurrentDir) {
+			return nil, fmt.Errorf("process directory %q was not absolute", p.CurrentDir)
+		}
+
+		rewrote := scheme + "://" + filepath.Join(p.CurrentDir, path)
+		klog.Infof("rewrote client url %q to %q", clientURLs[i], rewrote)
+		clientURLs[i] = rewrote
 	}
 
-	return etcdclient.NewClient(clientUrls, p.etcdClientTLSConfig)
+	return etcdclient.NewClient(clientURLs, p.etcdClientTLSConfig)
 }
 
 // DoBackup performs a backup/snapshot of the data

diff --git a/pkg/etcd/pki.go b/pkg/etcd/pki.go
@@ -146,6 +146,10 @@ func addAltNames(certConfig *certutil.Config, urls []string) error {
 				certConfig.AltNames.IPs = append(certConfig.AltNames.IPs, ip)
 			}
 
+		case "unix", "unixs":
+			h := u.Hostname() // Hostname does not include port
+			certConfig.AltNames.DNSNames = append(certConfig.AltNames.DNSNames, h)
+
 		default:
 			return fmt.Errorf("unknown URL %q", urlString)
 		}

diff --git a/pkg/etcd/restore.go b/pkg/etcd/restore.go
@@ -110,18 +110,12 @@ func RunEtcdFromBackup(backupStore backup.Store, backupName string, basedir stri
 		return nil, err
 	}
 
-	isV2 := false
 	if strings.HasPrefix(backupInfo.EtcdVersion, "2.") {
-		isV2 = true
+		return nil, fmt.Errorf("this version of etcd-manager does not support etcd v2")
 	}
 
-	var downloadFile string
-	if isV2 {
-		downloadFile = filepath.Join(basedir, "download", "backup.tar.gz")
-	} else {
-		// V3 requires that data dir not exist
-		downloadFile = filepath.Join(basedir, "download", "snapshot.db.gz")
-	}
+	// V3 requires that data dir not exist
+	downloadFile := filepath.Join(basedir, "download", "snapshot.db.gz")
 
 	klog.Infof("Downloading backup %q to %s", backupName, downloadFile)
 	if err := backupStore.DownloadBackup(backupName, downloadFile); err != nil {
@@ -143,17 +137,22 @@ func RunEtcdFromBackup(backupStore backup.Store, backupName string, basedir stri
 		return nil, err
 	}
 
-	// TODO: randomize port
-	port := 8002
-	peerPort := 8003 // Needed because otherwise etcd won't start (sadly)
-	clientUrl := "https://127.0.0.1:" + strconv.Itoa(port)
-	peerUrl := "https://127.0.0.1:" + strconv.Itoa(peerPort)
 	myNodeName := "restore"
 	myNode := &protoetcd.EtcdNode{
-		Name:       myNodeName,
-		ClientUrls: []string{clientUrl},
-		PeerUrls:   []string{peerUrl},
+		Name: myNodeName,
 	}
+
+	// Using unix domain sockets is more secure and avoids port conflicts.
+	// We have to construct the path to look like a host:port until https://github.com/etcd-io/etcd/pull/12469 lands.
+
+	// We have to set the etcd working directory so that we can pass a relative path as the socket
+	currentDir := basedir
+
+	clientPath := "127.0.0.1:8002"
+	peerPath := "127.0.0.1:8003"
+	myNode.ClientUrls = []string{"unixs://" + clientPath}
+	myNode.PeerUrls = []string{"unixs://" + peerPath}
+
 	p := &etcdProcess{
 		CreateNewCluster: true,
 		ForceNewCluster:  true,
@@ -168,6 +167,7 @@ func RunEtcdFromBackup(backupStore backup.Store, backupName string, basedir stri
 		MyNodeName:              myNodeName,
 		ListenAddress:           "127.0.0.1",
 		DisableTLS:              false,
+		CurrentDir:              currentDir,
 	}
 
 	etcdClientsCA, err := pki.NewCA(pki.NewInMemoryStore())
@@ -186,34 +186,16 @@ func RunEtcdFromBackup(backupStore backup.Store, backupName string, basedir stri
 		return nil, err
 	}
 
-	if isV2 {
-		if err := os.MkdirAll(dataDir, 0700); err != nil {
-			return nil, fmt.Errorf("error creating datadir %q: %v", dataDir, err)
-		}
-
-		archive := tgzArchive{File: downloadFile}
-		if err := archive.Extract(dataDir); err != nil {
-			return nil, fmt.Errorf("error expanding backup: %v", err)
-		}
+	klog.Infof("restoring snapshot")
 
-		// Not all backup stores store directories, but etcd2 requires a particular directory structure
-		// Create the directories even if there are no files
-		if err := os.MkdirAll(filepath.Join(p.DataDir, "member", "snap"), 0755); err != nil {
-			return nil, fmt.Errorf("error creating member/snap directory: %v", err)
-		}
+	snapshotFile := filepath.Join(basedir, "download", "snapshot.db")
+	archive := &gzFile{File: downloadFile}
+	if err := archive.expand(snapshotFile); err != nil {
+		return nil, fmt.Errorf("error expanding snapshot: %v", err)
 	}
-	if !isV2 {
-		klog.Infof("restoring snapshot")
 
-		snapshotFile := filepath.Join(basedir, "download", "snapshot.db")
-		archive := &gzFile{File: downloadFile}
-		if err := archive.expand(snapshotFile); err != nil {
-			return nil, fmt.Errorf("error expanding snapshot: %v", err)
-		}
-
-		if err := p.RestoreV3Snapshot(snapshotFile); err != nil {
-			return nil, err
-		}
+	if err := p.RestoreV3Snapshot(snapshotFile); err != nil {
+		return nil, err
 	}
 
 	klog.Infof("starting etcd to read backup")