feat: provide an option to recover etcd from data directory copy

Sometimes `talosctl etcd snapshot` might not be available, for example
when etcd is not healthy. In that case it's possible to copy raw etcd
data directory with `talosctl cp /var/lib/etcd .` and use
`member/snap/db` to recover the cluster. But such copy won't pass
integrity checks, so they should be disabled explicitly.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

api/machine/machine.proto

@@ -106,9 +106,16 @@ message ApplyConfigurationResponse { repeated ApplyConfiguration messages = 1; }
 message Reboot { common.Metadata metadata = 1; }
 message RebootResponse { repeated Reboot messages = 1; }
 
-// rpc bootstrap
+// rpc Bootstrap
 message BootstrapRequest {
+  // Enable etcd recovery from the snapshot.
+  //
+  // Snapshot should be uploaded before this call via EtcdRecover RPC.
   bool recover_etcd = 1;
+  // Skip hash check on the snapshot (etcd).
+  //
+  // Enable this when recovering from data directory copy to skip integrity check.
+  bool recover_skip_hash_check = 2;
 }
 
 // The bootstrap message containing the bootstrap status.

cmd/talosctl/cmd/talos/bootstrap.go

@@ -17,7 +17,8 @@ import (
 )
 
 var bootstrapCmdFlags struct {
-	recoverFrom string
+	recoverFrom          string
+	recoverSkipHashCheck bool
 }
 
 // bootstrapCmd represents the bootstrap command.
@@ -60,7 +61,8 @@ Talos etcd cluster can be recovered from a known snapshot with '--recover-from='
 			}
 
 			if err := c.Bootstrap(ctx, &machineapi.BootstrapRequest{
-				RecoverEtcd: bootstrapCmdFlags.recoverFrom != "",
+				RecoverEtcd:          bootstrapCmdFlags.recoverFrom != "",
+				RecoverSkipHashCheck: bootstrapCmdFlags.recoverSkipHashCheck,
 			}); err != nil {
 				return fmt.Errorf("error executing bootstrap: %s", err)
 			}
@@ -72,5 +74,6 @@ Talos etcd cluster can be recovered from a known snapshot with '--recover-from='
 
 func init() {
 	bootstrapCmd.Flags().StringVar(&bootstrapCmdFlags.recoverFrom, "recover-from", "", "recover etcd cluster from the snapshot")
+	bootstrapCmd.Flags().BoolVar(&bootstrapCmdFlags.recoverSkipHashCheck, "recover-skip-hash-check", false, "skip integrity check when recovering etcd (use when recovering from data directory copy)")
 	addCommand(bootstrapCmd)
 }

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go

@@ -1731,8 +1731,9 @@ func BootstrapEtcd(seq runtime.Sequence, data interface{}) (runtime.TaskExecutio
 		}
 
 		svc := &services.Etcd{
-			Bootstrap:           true,
-			RecoverFromSnapshot: req.RecoverEtcd,
+			Bootstrap:            true,
+			RecoverFromSnapshot:  req.RecoverEtcd,
+			RecoverSkipHashCheck: req.RecoverSkipHashCheck,
 		}
 
 		if err = system.Services(r).Unload(ctx, svc.ID(r)); err != nil {

internal/app/machined/pkg/system/services/etcd.go

@@ -46,8 +46,9 @@ import (
 // Etcd implements the Service interface. It serves as the concrete type with
 // the required methods.
 type Etcd struct {
-	Bootstrap           bool
-	RecoverFromSnapshot bool
+	Bootstrap            bool
+	RecoverFromSnapshot  bool
+	RecoverSkipHashCheck bool
 
 	args []string
 }
@@ -503,6 +504,8 @@ func (e *Etcd) recoverFromSnapshot(hostname, primaryAddr string) error {
 		PeerURLs: []string{"https://" + net.FormatAddress(primaryAddr) + ":2380"},
 
 		InitialCluster: fmt.Sprintf("%s=https://%s:2380", hostname, net.FormatAddress(primaryAddr)),
+
+		SkipHashCheck: e.RecoverSkipHashCheck,
 	}); err != nil {
 		return fmt.Errorf("error recovering from the snapshot: %w", err)
 	}

website/content/docs/v0.10/Reference/api.md

@@ -669,12 +669,17 @@ The bootstrap message containing the bootstrap status.
 <a name="machine.BootstrapRequest"></a>
 
 ### BootstrapRequest
-rpc bootstrap
+rpc Bootstrap
 
 
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
-| recover_etcd | [bool](#bool) |  |  |
+| recover_etcd | [bool](#bool) |  | Enable etcd recovery from the snapshot.
+
+Snapshot should be uploaded before this call via EtcdRecover RPC. |
+| recover_skip_hash_check | [bool](#bool) |  | Skip hash check on the snapshot (etcd).
+
+Enable this when recovering from data directory copy to skip integrity check. |
 
 
 

website/content/docs/v0.10/Reference/cli.md

@@ -60,8 +60,9 @@ talosctl bootstrap [flags]
 ### Options
 
 ```
-  -h, --help                  help for bootstrap
-      --recover-from string   recover etcd cluster from the snapshot
+  -h, --help                      help for bootstrap
+      --recover-from string       recover etcd cluster from the snapshot
+      --recover-skip-hash-check   skip integrity check when recovering etcd (use when recovering from data directory copy)
 ```
 
 ### Options inherited from parent commands