diff --git a/cmd/tarmak/cmd/cluster.go b/cmd/tarmak/cmd/cluster.go index e237b3b788..efc4e2f55a 100644 --- a/cmd/tarmak/cmd/cluster.go +++ b/cmd/tarmak/cmd/cluster.go @@ -115,6 +115,31 @@ func clusterKubeconfigFlags(fs *flag.FlagSet) { ) } +func clusterSnapshotEtcdRestoreFlags(fs *flag.FlagSet) { + store := &globalFlags.Cluster.Snapshot.Etcd.Restore + + fs.StringVar( + &store.K8sMain, + consts.RestoreK8sMainFlagName, + "", + "location of k8s-main snapshot backup", + ) + + fs.StringVar( + &store.K8sEvents, + consts.RestoreK8sEventsFlagName, + "", + "location of k8s-events snapshot backup", + ) + + fs.StringVar( + &store.Overlay, + consts.RestoreOverlayFlagName, + "", + "location of overlay snapshot backup", + ) +} + func init() { RootCmd.AddCommand(clusterCmd) } diff --git a/cmd/tarmak/cmd/cluster_snapshot.go b/cmd/tarmak/cmd/cluster_snapshot.go new file mode 100644 index 0000000000..d1faa1aefe --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "Manage snapshots of remote consul and etcd clusters", +} + +func init() { + clusterCmd.AddCommand(clusterSnapshotCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul.go b/cmd/tarmak/cmd/cluster_snapshot_consul.go new file mode 100644 index 0000000000..13b067c926 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotConsulCmd = &cobra.Command{ + Use: "consul", + Short: "Manage snapshots on remote consul clusters", +} + +func init() { + clusterSnapshotCmd.AddCommand(clusterSnapshotConsulCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go new file mode 100644 index 0000000000..756d73c951 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_restore.go @@ -0,0 +1,32 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/consul" +) + +var clusterSnapshotConsulRestoreCmd = &cobra.Command{ + Use: "restore [source path]", + Short: "restore consul cluster with source snapshot", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single source path, got=%d", len(args)) + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + s := consul.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Restore) + }, +} + +func init() { + clusterSnapshotConsulCmd.AddCommand(clusterSnapshotConsulRestoreCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_consul_save.go b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go new file mode 100644 index 0000000000..84c99aea75 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_consul_save.go @@ -0,0 +1,32 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/consul" +) + +var clusterSnapshotConsulSaveCmd = &cobra.Command{ + Use: "save [target path]", + Short: "save consul cluster snapshot to target path", + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("expecting single target path, got=%d", len(args)) + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + s := consul.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Save) + }, +} + +func init() { + clusterSnapshotConsulCmd.AddCommand(clusterSnapshotConsulSaveCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd.go b/cmd/tarmak/cmd/cluster_snapshot_etcd.go new file mode 100644 index 0000000000..f14853cb62 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd.go @@ -0,0 +1,15 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" +) + +var clusterSnapshotEtcdCmd = &cobra.Command{ + Use: "etcd", + Short: "Manage snapshots on remote etcd clusters", +} + +func init() { + clusterSnapshotCmd.AddCommand(clusterSnapshotEtcdCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go new file mode 100644 index 0000000000..6454fca411 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_restore.go @@ -0,0 +1,42 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/etcd" + "github.com/jetstack/tarmak/pkg/tarmak/utils/consts" +) + +var clusterSnapshotEtcdRestoreCmd = &cobra.Command{ + Use: "restore", + Short: "restore etcd cluster with source snapshots", + PreRunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed(consts.RestoreK8sMainFlagName) && + !cmd.Flags().Changed(consts.RestoreK8sEventsFlagName) && + !cmd.Flags().Changed(consts.RestoreOverlayFlagName) { + + return fmt.Errorf("expecting at least one set flag of [%s %s %s]", + consts.RestoreK8sMainFlagName, + consts.RestoreK8sEventsFlagName, + consts.RestoreOverlayFlagName, + ) + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + t := tarmak.New(globalFlags) + s := etcd.New(t, "") + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Restore) + }, +} + +func init() { + clusterSnapshotEtcdRestoreFlags( + clusterSnapshotEtcdRestoreCmd.PersistentFlags(), + ) + clusterSnapshotEtcdCmd.AddCommand(clusterSnapshotEtcdRestoreCmd) +} diff --git a/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go new file mode 100644 index 0000000000..62f1ffdc79 --- /dev/null +++ b/cmd/tarmak/cmd/cluster_snapshot_etcd_save.go @@ -0,0 +1,26 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/jetstack/tarmak/pkg/tarmak" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot/etcd" +) + +var clusterSnapshotEtcdSaveCmd = &cobra.Command{ + Use: "save [target path prefix]", + Short: "save etcd snapshot to target path prefix, i.e 'backup-'", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + args = []string{""} + } + t := tarmak.New(globalFlags) + s := etcd.New(t, args[0]) + t.CancellationContext().WaitOrCancel(t.NewCmdSnapshot(cmd.Flags(), args, s).Save) + }, +} + +func init() { + clusterSnapshotEtcdCmd.AddCommand(clusterSnapshotEtcdSaveCmd) +} diff --git a/docs/cmd-docs.rst b/docs/cmd-docs.rst index 0e3d4c8f88..426047ff3d 100644 --- a/docs/cmd-docs.rst +++ b/docs/cmd-docs.rst @@ -94,6 +94,41 @@ Command line documentation for tarmak, wing and tagging_control commands generated/cmd/tarmak/tarmak_clusters_set-current +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore + +.. toctree:: + :maxdepth: 1 + + generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save + .. toctree:: :maxdepth: 1 diff --git a/docs/generated/cmd/tarmak/tarmak_clusters.rst b/docs/generated/cmd/tarmak/tarmak_clusters.rst index f675847c2d..a2654f094c 100644 --- a/docs/generated/cmd/tarmak/tarmak_clusters.rst +++ b/docs/generated/cmd/tarmak/tarmak_clusters.rst @@ -47,5 +47,6 @@ SEE ALSO * `tarmak clusters list `_ - Print a list of clusters * `tarmak clusters plan `_ - Plan changes on the currently configured cluster * `tarmak clusters set-current `_ - Set current cluster in config +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters * `tarmak clusters ssh `_ - Log into an instance with SSH diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst new file mode 100644 index 0000000000..27530b0dfd --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot: + +tarmak clusters snapshot +------------------------ + +Manage snapshots of remote consul and etcd clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots of remote consul and etcd clusters + +Options +~~~~~~~ + +:: + + -h, --help help for snapshot + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters `_ - Operations on clusters +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst new file mode 100644 index 0000000000..cc93112f30 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot_consul: + +tarmak clusters snapshot consul +------------------------------- + +Manage snapshots on remote consul clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots on remote consul clusters + +Options +~~~~~~~ + +:: + + -h, --help help for consul + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters +* `tarmak clusters snapshot consul restore `_ - restore consul cluster with source snapshot +* `tarmak clusters snapshot consul save `_ - save consul cluster snapshot to target path + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst new file mode 100644 index 0000000000..a0a06500fe --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_restore.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_consul_restore: + +tarmak clusters snapshot consul restore +--------------------------------------- + +restore consul cluster with source snapshot + +Synopsis +~~~~~~~~ + + +restore consul cluster with source snapshot + +:: + + tarmak clusters snapshot consul restore [source path] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for restore + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst new file mode 100644 index 0000000000..865d7f3c67 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_consul_save.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_consul_save: + +tarmak clusters snapshot consul save +------------------------------------ + +save consul cluster snapshot to target path + +Synopsis +~~~~~~~~ + + +save consul cluster snapshot to target path + +:: + + tarmak clusters snapshot consul save [target path] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for save + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot consul `_ - Manage snapshots on remote consul clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst new file mode 100644 index 0000000000..edfbdc74a0 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd.rst @@ -0,0 +1,40 @@ +.. _tarmak_clusters_snapshot_etcd: + +tarmak clusters snapshot etcd +----------------------------- + +Manage snapshots on remote etcd clusters + +Synopsis +~~~~~~~~ + + +Manage snapshots on remote etcd clusters + +Options +~~~~~~~ + +:: + + -h, --help help for etcd + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot `_ - Manage snapshots of remote consul and etcd clusters +* `tarmak clusters snapshot etcd restore `_ - restore etcd cluster with source snapshots +* `tarmak clusters snapshot etcd save `_ - save etcd snapshot to target path prefix, i.e 'backup-' + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst new file mode 100644 index 0000000000..b552a1025d --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_restore.rst @@ -0,0 +1,45 @@ +.. _tarmak_clusters_snapshot_etcd_restore: + +tarmak clusters snapshot etcd restore +------------------------------------- + +restore etcd cluster with source snapshots + +Synopsis +~~~~~~~~ + + +restore etcd cluster with source snapshots + +:: + + tarmak clusters snapshot etcd restore [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for restore + --k8s-events string location of k8s-events snapshot backup + --k8s-main string location of k8s-main snapshot backup + --overlay string location of overlay snapshot backup + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst new file mode 100644 index 0000000000..f59490c0b9 --- /dev/null +++ b/docs/generated/cmd/tarmak/tarmak_clusters_snapshot_etcd_save.rst @@ -0,0 +1,42 @@ +.. _tarmak_clusters_snapshot_etcd_save: + +tarmak clusters snapshot etcd save +---------------------------------- + +save etcd snapshot to target path prefix, i.e 'backup-' + +Synopsis +~~~~~~~~ + + +save etcd snapshot to target path prefix, i.e 'backup-' + +:: + + tarmak clusters snapshot etcd save [target path prefix] [flags] + +Options +~~~~~~~ + +:: + + -h, --help help for save + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + -c, --config-directory string config directory for tarmak's configuration (default "~/.tarmak") + --current-cluster string override the current cluster set in the config + --ignore-missing-public-key-tags ssh_known_hosts ignore missing public key tags on instances, by falling back to populating ssh_known_hosts with the first connection (default true) + --keep-containers do not clean-up terraform/packer containers after running them + --public-api-endpoint Override kubeconfig to point to cluster's public API endpoint + -v, --verbose enable verbose logging + --wing-dev-mode use a bundled wing version rather than a tagged release from GitHub + +SEE ALSO +~~~~~~~~ + +* `tarmak clusters snapshot etcd `_ - Manage snapshots on remote etcd clusters + diff --git a/docs/generated/reference/output/api-docs.html b/docs/generated/reference/output/api-docs.html index d5e813f552..8c3b85269f 100644 --- a/docs/generated/reference/output/api-docs.html +++ b/docs/generated/reference/output/api-docs.html @@ -11,7 +11,7 @@ - +

    Tarmak

    @@ -648,6 +648,10 @@

    ClusterFlags v1alpha1

    plan
    ClusterPlanFlags flags for handling images + +snapshot
    ClusterSnapshotFlags +flags for kubeconfig of clusters +

    ClusterImagesBuildFlags v1alpha1

    @@ -1561,6 +1565,131 @@

    ClusterPodSecurityPolicy v1alpha1

    +

    ClusterSnapshotEtcdFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotEtcdFlags
    +

    Contains the cluster snapshot etcd flags

    + + + + + + + + + + + + + + + +
    FieldDescription
    restore
    ClusterSnapshotEtcdRestoreFlags
    +

    ClusterSnapshotEtcdRestoreFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotEtcdRestoreFlags
    +

    Contains the cluster snapshot etcd restore flags

    + + + + + + + + + + + + + + + + + + + + + + + +
    FieldDescription
    k8sEvents
    string    		Path to k8s-main snapshot backup
    k8sMain
    string
    overlay
    string    		Path to k8s-events snapshot backup
    +

    ClusterSnapshotFlags v1alpha1

    + + + + + + + + + + + + + + + +
    GroupVersionKind
    tarmakv1alpha1ClusterSnapshotFlags
    +

    Contains the cluster snapshot flags

    + + + + + + + + + + + + + + + +
    FieldDescription
    etcd
    ClusterSnapshotEtcdFlags

    ClusterVaultHelper v1alpha1

    diff --git a/docs/generated/reference/output/navData.js b/docs/generated/reference/output/navData.js index 4a6a007661..0875d80bc0 100644 --- a/docs/generated/reference/output/navData.js +++ b/docs/generated/reference/output/navData.js @@ -1 +1 @@ -(function(){navData = {"toc":[{"section":"-strong-field-definitions-strong-","subsections":[{"section":"volume-v1alpha1"},{"section":"values-v1alpha1"},{"section":"taint-v1alpha1"},{"section":"subnet-v1alpha1"},{"section":"ssh-v1alpha1"},{"section":"providergcp-v1alpha1"},{"section":"providerazure-v1alpha1"},{"section":"provideramazon-v1alpha1"},{"section":"provider-v1alpha1"},{"section":"network-v1alpha1"},{"section":"loggingsinkelasticsearch-v1alpha1"},{"section":"loggingsink-v1alpha1"},{"section":"label-v1alpha1"},{"section":"kubernetesapi-v1alpha1"},{"section":"internetgw-v1alpha1"},{"section":"instancestatusmanifest-v1alpha1"},{"section":"instancespecmanifest-v1alpha1"},{"section":"instancepoolkubernetes-v1alpha1"},{"section":"instancepoolamazon-v1alpha1"},{"section":"instancepool-v1alpha1"},{"section":"ingressrule-v1alpha1"},{"section":"httpbasicauth-v1alpha1"},{"section":"firewall-v1alpha1"},{"section":"environment-v1alpha1"},{"section":"egressrule-v1alpha1"},{"section":"clustervaulthelper-v1alpha1"},{"section":"clusterpodsecuritypolicy-v1alpha1"},{"section":"clusterplanflags-v1alpha1"},{"section":"clusterkubernetestiller-v1alpha1"},{"section":"clusterkubernetesscheduler-v1alpha1"},{"section":"clusterkubernetesproxy-v1alpha1"},{"section":"clusterkubernetesprometheus-v1alpha1"},{"section":"clusterkuberneteskubelet-v1alpha1"},{"section":"clusterkubernetesdashboard-v1alpha1"},{"section":"clusterkubernetescontrollermanager-v1alpha1"},{"section":"clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1"},{"section":"clusterkubernetesclusterautoscaler-v1alpha1"},{"section":"clusterkubernetesapiserveroidc-v1alpha1"},{"section":"clusterkubernetesapiserveramazonaccesslogs-v1alpha1"},{"section":"clusterkubernetesapiserveramazon-v1alpha1"},{"section":"clusterkubernetesapiserver-v1alpha1"},{"section":"clusterkubernetes-v1alpha1"},{"section":"clusterkubeconfigflags-v1alpha1"},{"section":"clusterimagesflags-v1alpha1"},{"section":"clusterimagesbuildflags-v1alpha1"},{"section":"clusterflags-v1alpha1"},{"section":"clusterdestroyflags-v1alpha1"},{"section":"clusterapplyflags-v1alpha1"},{"section":"clusteramazon-v1alpha1"},{"section":"amazonesproxy-v1alpha1"}]},{"section":"-strong-old-api-versions-strong-","subsections":[]},{"section":"instance-v1alpha1","subsections":[]},{"section":"cluster-v1alpha1","subsections":[]},{"section":"flags-v1alpha1","subsections":[]},{"section":"image-v1alpha1","subsections":[]},{"section":"config-v1alpha1","subsections":[]},{"section":"-strong-tarmak-strong-","subsections":[]}],"flatToc":["volume-v1alpha1","values-v1alpha1","taint-v1alpha1","subnet-v1alpha1","ssh-v1alpha1","providergcp-v1alpha1","providerazure-v1alpha1","provideramazon-v1alpha1","provider-v1alpha1","network-v1alpha1","loggingsinkelasticsearch-v1alpha1","loggingsink-v1alpha1","label-v1alpha1","kubernetesapi-v1alpha1","internetgw-v1alpha1","instancestatusmanifest-v1alpha1","instancespecmanifest-v1alpha1","instancepoolkubernetes-v1alpha1","instancepoolamazon-v1alpha1","instancepool-v1alpha1","ingressrule-v1alpha1","httpbasicauth-v1alpha1","firewall-v1alpha1","environment-v1alpha1","egressrule-v1alpha1","clustervaulthelper-v1alpha1","clusterpodsecuritypolicy-v1alpha1","clusterplanflags-v1alpha1","clusterkubernetestiller-v1alpha1","clusterkubernetesscheduler-v1alpha1","clusterkubernetesproxy-v1alpha1","clusterkubernetesprometheus-v1alpha1","clusterkuberneteskubelet-v1alpha1","clusterkubernetesdashboard-v1alpha1","clusterkubernetescontrollermanager-v1alpha1","clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1","clusterkubernetesclusterautoscaler-v1alpha1","clusterkubernetesapiserveroidc-v1alpha1","clusterkubernetesapiserveramazonaccesslogs-v1alpha1","clusterkubernetesapiserveramazon-v1alpha1","clusterkubernetesapiserver-v1alpha1","clusterkubernetes-v1alpha1","clusterkubeconfigflags-v1alpha1","clusterimagesflags-v1alpha1","clusterimagesbuildflags-v1alpha1","clusterflags-v1alpha1","clusterdestroyflags-v1alpha1","clusterapplyflags-v1alpha1","clusteramazon-v1alpha1","amazonesproxy-v1alpha1","-strong-field-definitions-strong-","-strong-old-api-versions-strong-","instance-v1alpha1","cluster-v1alpha1","flags-v1alpha1","image-v1alpha1","config-v1alpha1","-strong-tarmak-strong-"]};})(); \ No newline at end of file +(function(){navData = {"toc":[{"section":"-strong-field-definitions-strong-","subsections":[{"section":"volume-v1alpha1"},{"section":"values-v1alpha1"},{"section":"taint-v1alpha1"},{"section":"subnet-v1alpha1"},{"section":"ssh-v1alpha1"},{"section":"providergcp-v1alpha1"},{"section":"providerazure-v1alpha1"},{"section":"provideramazon-v1alpha1"},{"section":"provider-v1alpha1"},{"section":"network-v1alpha1"},{"section":"loggingsinkelasticsearch-v1alpha1"},{"section":"loggingsink-v1alpha1"},{"section":"label-v1alpha1"},{"section":"kubernetesapi-v1alpha1"},{"section":"internetgw-v1alpha1"},{"section":"instancestatusmanifest-v1alpha1"},{"section":"instancespecmanifest-v1alpha1"},{"section":"instancepoolkubernetes-v1alpha1"},{"section":"instancepoolamazon-v1alpha1"},{"section":"instancepool-v1alpha1"},{"section":"ingressrule-v1alpha1"},{"section":"httpbasicauth-v1alpha1"},{"section":"firewall-v1alpha1"},{"section":"environment-v1alpha1"},{"section":"egressrule-v1alpha1"},{"section":"clustervaulthelper-v1alpha1"},{"section":"clustersnapshotflags-v1alpha1"},{"section":"clustersnapshotetcdrestoreflags-v1alpha1"},{"section":"clustersnapshotetcdflags-v1alpha1"},{"section":"clusterpodsecuritypolicy-v1alpha1"},{"section":"clusterplanflags-v1alpha1"},{"section":"clusterkubernetestiller-v1alpha1"},{"section":"clusterkubernetesscheduler-v1alpha1"},{"section":"clusterkubernetesproxy-v1alpha1"},{"section":"clusterkubernetesprometheus-v1alpha1"},{"section":"clusterkuberneteskubelet-v1alpha1"},{"section":"clusterkubernetesdashboard-v1alpha1"},{"section":"clusterkubernetescontrollermanager-v1alpha1"},{"section":"clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1"},{"section":"clusterkubernetesclusterautoscaler-v1alpha1"},{"section":"clusterkubernetesapiserveroidc-v1alpha1"},{"section":"clusterkubernetesapiserveramazonaccesslogs-v1alpha1"},{"section":"clusterkubernetesapiserveramazon-v1alpha1"},{"section":"clusterkubernetesapiserver-v1alpha1"},{"section":"clusterkubernetes-v1alpha1"},{"section":"clusterkubeconfigflags-v1alpha1"},{"section":"clusterimagesflags-v1alpha1"},{"section":"clusterimagesbuildflags-v1alpha1"},{"section":"clusterflags-v1alpha1"},{"section":"clusterdestroyflags-v1alpha1"},{"section":"clusterapplyflags-v1alpha1"},{"section":"clusteramazon-v1alpha1"},{"section":"amazonesproxy-v1alpha1"}]},{"section":"-strong-old-api-versions-strong-","subsections":[]},{"section":"instance-v1alpha1","subsections":[]},{"section":"cluster-v1alpha1","subsections":[]},{"section":"flags-v1alpha1","subsections":[]},{"section":"image-v1alpha1","subsections":[]},{"section":"config-v1alpha1","subsections":[]},{"section":"-strong-tarmak-strong-","subsections":[]}],"flatToc":["volume-v1alpha1","values-v1alpha1","taint-v1alpha1","subnet-v1alpha1","ssh-v1alpha1","providergcp-v1alpha1","providerazure-v1alpha1","provideramazon-v1alpha1","provider-v1alpha1","network-v1alpha1","loggingsinkelasticsearch-v1alpha1","loggingsink-v1alpha1","label-v1alpha1","kubernetesapi-v1alpha1","internetgw-v1alpha1","instancestatusmanifest-v1alpha1","instancespecmanifest-v1alpha1","instancepoolkubernetes-v1alpha1","instancepoolamazon-v1alpha1","instancepool-v1alpha1","ingressrule-v1alpha1","httpbasicauth-v1alpha1","firewall-v1alpha1","environment-v1alpha1","egressrule-v1alpha1","clustervaulthelper-v1alpha1","clustersnapshotflags-v1alpha1","clustersnapshotetcdrestoreflags-v1alpha1","clustersnapshotetcdflags-v1alpha1","clusterpodsecuritypolicy-v1alpha1","clusterplanflags-v1alpha1","clusterkubernetestiller-v1alpha1","clusterkubernetesscheduler-v1alpha1","clusterkubernetesproxy-v1alpha1","clusterkubernetesprometheus-v1alpha1","clusterkuberneteskubelet-v1alpha1","clusterkubernetesdashboard-v1alpha1","clusterkubernetescontrollermanager-v1alpha1","clusterkubernetesclusterautoscaleroverprovisioning-v1alpha1","clusterkubernetesclusterautoscaler-v1alpha1","clusterkubernetesapiserveroidc-v1alpha1","clusterkubernetesapiserveramazonaccesslogs-v1alpha1","clusterkubernetesapiserveramazon-v1alpha1","clusterkubernetesapiserver-v1alpha1","clusterkubernetes-v1alpha1","clusterkubeconfigflags-v1alpha1","clusterimagesflags-v1alpha1","clusterimagesbuildflags-v1alpha1","clusterflags-v1alpha1","clusterdestroyflags-v1alpha1","clusterapplyflags-v1alpha1","clusteramazon-v1alpha1","amazonesproxy-v1alpha1","-strong-field-definitions-strong-","-strong-old-api-versions-strong-","instance-v1alpha1","cluster-v1alpha1","flags-v1alpha1","image-v1alpha1","config-v1alpha1","-strong-tarmak-strong-"]};})(); \ No newline at end of file diff --git a/pkg/apis/tarmak/v1alpha1/types.go b/pkg/apis/tarmak/v1alpha1/types.go index 7130de8088..d5cc7c57b7 100644 --- a/pkg/apis/tarmak/v1alpha1/types.go +++ b/pkg/apis/tarmak/v1alpha1/types.go @@ -149,6 +149,7 @@ type ClusterFlags struct { Images ClusterImagesFlags `json:"images,omitempty"` // flags for handling images Plan ClusterPlanFlags `json:"plan,omitempty"` // flags for planning clusters Kubeconfig ClusterKubeconfigFlags `json:"kubeconfig,omitempty"` // flags for kubeconfig of clusters + Snapshot ClusterSnapshotFlags `json:"snapshot,omitempty"` // flags for snapshots of clusters } // Contains the cluster plan flags @@ -190,3 +191,20 @@ type ClusterImagesBuildFlags struct { type ClusterKubeconfigFlags struct { Path string `json:"path,omitempty"` // Path to save kubeconfig to } + +// Contains the cluster snapshot flags +type ClusterSnapshotFlags struct { + Etcd ClusterSnapshotEtcdFlags `json:"etcd,omitempty"` // flags for handling etcd snapshots +} + +// Contains the cluster snapshot etcd flags +type ClusterSnapshotEtcdFlags struct { + Restore ClusterSnapshotEtcdRestoreFlags `json:"restore,omitempty"` // flags for handling etcd snapshot restore +} + +// Contains the cluster snapshot etcd restore flags +type ClusterSnapshotEtcdRestoreFlags struct { + K8sMain string `json:"k8sMain,omitempty"` // Path to k8s-main snapshot backup + K8sEvents string `json:"k8sEvents,omitempty"` // Path to k8s-events snapshot backup + Overlay string `json:"overlay,omitempty"` // Path to overlay snapshot backup +} diff --git a/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go index e1e40e0e6c..3d23af269c 100644 --- a/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/tarmak/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,7 @@ func (in *ClusterFlags) DeepCopyInto(out *ClusterFlags) { out.Images = in.Images out.Plan = in.Plan out.Kubeconfig = in.Kubeconfig + out.Snapshot = in.Snapshot return } @@ -129,6 +130,56 @@ func (in *ClusterPlanFlags) DeepCopy() *ClusterPlanFlags { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotEtcdFlags) DeepCopyInto(out *ClusterSnapshotEtcdFlags) { + *out = *in + out.Restore = in.Restore + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotEtcdFlags. +func (in *ClusterSnapshotEtcdFlags) DeepCopy() *ClusterSnapshotEtcdFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotEtcdFlags) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotEtcdRestoreFlags) DeepCopyInto(out *ClusterSnapshotEtcdRestoreFlags) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotEtcdRestoreFlags. +func (in *ClusterSnapshotEtcdRestoreFlags) DeepCopy() *ClusterSnapshotEtcdRestoreFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotEtcdRestoreFlags) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSnapshotFlags) DeepCopyInto(out *ClusterSnapshotFlags) { + *out = *in + out.Etcd = in.Etcd + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSnapshotFlags. +func (in *ClusterSnapshotFlags) DeepCopy() *ClusterSnapshotFlags { + if in == nil { + return nil + } + out := new(ClusterSnapshotFlags) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Config) DeepCopyInto(out *Config) { *out = *in diff --git a/pkg/tarmak/cmd.go b/pkg/tarmak/cmd.go index c53e789d77..ddae349bbb 100644 --- a/pkg/tarmak/cmd.go +++ b/pkg/tarmak/cmd.go @@ -30,6 +30,11 @@ type CmdTarmak struct { ctx interfaces.CancellationContext } +type CmdSnapshot struct { + *CmdTarmak + snapshot interfaces.Snapshot +} + func (t *Tarmak) NewCmdTarmak(pflags *pflag.FlagSet, args []string) *CmdTarmak { return &CmdTarmak{ Tarmak: t, @@ -40,6 +45,13 @@ func (t *Tarmak) NewCmdTarmak(pflags *pflag.FlagSet, args []string) *CmdTarmak { } } +func (t *Tarmak) NewCmdSnapshot(pflags *pflag.FlagSet, args []string, sh interfaces.Snapshot) *CmdSnapshot { + return &CmdSnapshot{ + CmdTarmak: t.NewCmdTarmak(pflags, args), + snapshot: sh, + } +} + func (c *CmdTarmak) Plan() (returnCode int, err error) { if err := c.setup(); err != nil { return 1, err @@ -290,6 +302,14 @@ func (c *CmdTarmak) kubePublicAPIEndpoint() bool { return publicEndpoint } +func (c *CmdSnapshot) Save() error { + return c.snapshot.Save() +} + +func (c *CmdSnapshot) Restore() error { + return c.snapshot.Restore() +} + func (c *CmdTarmak) verifyTerraformBinaryVersion() error { cmd := exec.Command("terraform", "version") cmd.Env = os.Environ() diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index 18d9b3cc3b..b6be24a07d 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -294,3 +294,10 @@ type CancellationContext interface { WaitOrCancel(f func() error) WaitOrCancelReturnCode(f func() (int, error)) } + +type Snapshot interface { + Save() error + Restore() error + Log() *logrus.Entry + SSH() SSH +} diff --git a/pkg/tarmak/snapshot/consul/consul.go b/pkg/tarmak/snapshot/consul/consul.go new file mode 100644 index 0000000000..646b6f9319 --- /dev/null +++ b/pkg/tarmak/snapshot/consul/consul.go @@ -0,0 +1,133 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package consul + +import ( + "fmt" + "io" + "time" + + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot" +) + +var _ interfaces.Snapshot = &Consul{} + +type Consul struct { + tarmak interfaces.Tarmak + ssh interfaces.SSH + log *logrus.Entry + + path string +} + +func New(tarmak interfaces.Tarmak, path string) *Consul { + return &Consul{ + tarmak: tarmak, + ssh: tarmak.SSH(), + log: tarmak.Log(), + path: path, + } +} + +func (c *Consul) Save() error { + aliases, err := snapshot.Prepare(c.tarmak, clusterv1alpha1.InstancePoolTypeVault) + if err != nil { + return err + } + + c.log.Infof("saving snapshot from instance %s", aliases[0]) + + hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", + time.Now().Format(snapshot.TimeLayout)) + cmdArgs := fmt.Sprintf(`set -e; +export CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token'); +export DATACENTER=$(sudo cat /etc/consul/consul.json | jq -r '.datacenter'); +/opt/bin/consul snapshot save -datacenter $DATACENTER %s; +/opt/bin/consul snapshot inspect %s`, hostPath, hostPath) + + err = snapshot.SSHCmd(c, aliases[0], cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + reader, writer := io.Pipe() + sshCmd := func() error { + err := snapshot.SSHCmd(c, aliases[0], + fmt.Sprintf(snapshot.GZipCCmd, hostPath), + nil, writer, nil) + writer.Close() + + return err + } + + err = snapshot.TarFromStream(sshCmd, reader, c.path) + if err != nil { + return err + } + + err = snapshot.SSHCmd(c, aliases[0], fmt.Sprintf("sudo rm %s", hostPath), nil, nil, nil) + if err != nil { + return err + } + + c.log.Infof("consul snapshot saved to %s", c.path) + + return nil +} + +func (c *Consul) Restore() error { + aliases, err := snapshot.Prepare(c.tarmak, clusterv1alpha1.InstancePoolTypeVault) + if err != nil { + return err + } + + for _, a := range aliases { + c.log.Infof("restoring snapshot to instance %s", a) + + reader, writer := io.Pipe() + hostPath := fmt.Sprintf("/tmp/consul-snapshot-%s.snap", + time.Now().Format(snapshot.TimeLayout)) + + sshCmd := func() error { + err := snapshot.SSHCmd(c, a, + fmt.Sprintf(snapshot.GZipDCmd, hostPath), + reader, nil, nil) + + return err + } + + err = snapshot.TarToStream(sshCmd, writer, c.path) + if err != nil { + return err + } + + cmdArgs := fmt.Sprintf(`set -e; +export CONSUL_HTTP_TOKEN=$(sudo cat /etc/consul/consul.json | jq -r '.acl_master_token'); +export DATACENTER=$(sudo cat /etc/consul/consul.json | jq -r '.datacenter'); +/opt/bin/consul snapshot inspect %s; +/opt/bin/consul snapshot restore -datacenter $DATACENTER %s; +echo number of keys: $(curl --header "X-Consul-Token: $CONSUL_HTTP_TOKEN" -s 'http://127.0.0.1:8500/v1/kv/?keys' | jq '. | length'); +sudo rm %s; +`, hostPath, hostPath, hostPath) + + err = snapshot.SSHCmd(c, a, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + } + + c.log.Infof("consul snapshot restored from %s", c.path) + + return nil +} + +func (c *Consul) Log() *logrus.Entry { + return c.log +} + +func (c *Consul) SSH() interfaces.SSH { + return c.ssh +} diff --git a/pkg/tarmak/snapshot/etcd/etcd.go b/pkg/tarmak/snapshot/etcd/etcd.go new file mode 100644 index 0000000000..4a7ca9de35 --- /dev/null +++ b/pkg/tarmak/snapshot/etcd/etcd.go @@ -0,0 +1,394 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package etcd + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "strings" + "sync" + "time" + + multierror "github.com/hashicorp/go-multierror" + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/jetstack/tarmak/pkg/apis/cluster/v1alpha1" + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/snapshot" + "github.com/jetstack/tarmak/pkg/tarmak/utils/consts" +) + +var _ interfaces.Snapshot = &Etcd{} + +const ( + etcdctlCmd = `/opt/bin/etcdctl snapshot %s %s;` +) + +var ( + stores = []map[string]string{ + {"cluster": consts.RestoreK8sMainFlagName, "file": "k8s", "client_port": "2379", "peer_port": "2380"}, + {"cluster": consts.RestoreK8sEventsFlagName, "file": "k8s", "client_port": "2369", "peer_port": "2370"}, + {"cluster": consts.RestoreOverlayFlagName, "file": "overlay", "client_port": "2359", "peer_port": "2360"}, + } + + envCmd = ` +set -e; +export ETCDCTL_CERT=/etc/etcd/ssl/etcd-{{file}}.pem; +export ETCDCTL_KEY=/etc/etcd/ssl/etcd-{{file}}-key.pem; +export ETCDCTL_CACERT=/etc/etcd/ssl/etcd-{{file}}-ca.pem; +export ETCDCTL_API=3; +export ETCDCTL_ENDPOINTS=https://127.0.0.1:{{client_port}}; +` +) + +type Etcd struct { + tarmak interfaces.Tarmak + ssh interfaces.SSH + log *logrus.Entry + ctx interfaces.CancellationContext + + path string +} + +func New(tarmak interfaces.Tarmak, path string) *Etcd { + return &Etcd{ + tarmak: tarmak, + ssh: tarmak.SSH(), + ctx: tarmak.CancellationContext(), + log: tarmak.Log(), + path: path, + } +} + +func (e *Etcd) Save() error { + aliases, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeEtcd) + if err != nil { + return err + } + + e.log.Infof("saving snapshots from instance %s", aliases[0]) + + var wg sync.WaitGroup + var result *multierror.Error + var errLock sync.Mutex + + saveFunc := func(store map[string]string) { + defer wg.Done() + + hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", + store["cluster"], time.Now().Format(snapshot.TimeLayout)) + + cmdArgs := fmt.Sprintf( + `sudo /bin/bash -c "%s %s %s"`, + e.template(envCmd, store), + fmt.Sprintf(etcdctlCmd, "save", hostPath), + fmt.Sprintf(etcdctlCmd, "status", hostPath), + ) + + err = snapshot.SSHCmd(e, aliases[0], cmdArgs, nil, nil, nil) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + targetPath := fmt.Sprintf("%s%s.db", e.path, store["cluster"]) + reader, writer := io.Pipe() + + err = snapshot.TarFromStream(func() error { + err := snapshot.SSHCmd(e, aliases[0], fmt.Sprintf(snapshot.GZipCCmd, hostPath), + nil, writer, nil) + writer.Close() + return err + + }, reader, targetPath) + + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + err = snapshot.SSHCmd(e, aliases[0], fmt.Sprintf("sudo rm %s", hostPath), nil, nil, nil) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + return + } + + e.log.Infof("etcd %s snapshot saved to %s", store["cluster"], targetPath) + + select { + case <-e.ctx.Done(): + return + default: + } + } + + wg.Add(len(stores)) + for _, store := range stores { + go saveFunc(store) + } + + wg.Wait() + + select { + case <-e.ctx.Done(): + return e.ctx.Err() + default: + } + + return result.ErrorOrNil() +} + +func (e *Etcd) Restore() error { + aliases, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeEtcd) + if err != nil { + return err + } + + stopEtcdFunc := func(host string, store map[string]string) error { + cmdArgs := fmt.Sprintf(`set -e; sudo systemctl stop etcd-%s `, store["cluster"]) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + + return err + } + + //restoreFunc := func(host, path, token string, store map[string]string) error { + // reader, writer := io.Pipe() + // hostPath := fmt.Sprintf("/tmp/etcd-snapshot-%s-%s.db", + // store["cluster"], time.Now().Format(snapshot.TimeLayout)) + + // err = snapshot.TarToStream(func() error { + // err := snapshot.SSHCmd(e, host, fmt.Sprintf(snapshot.GZipDCmd, hostPath), reader, nil, nil) + // return err + // }, writer, path) + // if err != nil { + // return err + // } + + // + // cmdArgs = e.template(`set -e; + //sudo mkdir -p /var/lib/etcd_backup; + //sudo rsync -a --delete --ignore-missing-args /var/lib/etcd/{{cluster}} /var/lib/etcd_backup/; + //sudo rm -rf /var/lib/etcd/{{cluster}}; + //`, store) + // err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + // if err != nil { + // return err + // } + // + // initialCluster := e.initialClusterString(host, store) + // for _, a := range aliases[1:] { + // initialCluster = strings.Join( + // []string{initialCluster, e.initialClusterString(a, store)}, ",", + // ) + // } + // + // cmdArgs = e.template(fmt.Sprintf(`set -e; + //sudo ETCDCTL_API=3 /opt/bin/etcdctl snapshot restore %s \ + //--name=%s.%s.%s \ + //--data-dir=/var/lib/etcd/{{cluster}} \ + //--initial-advertise-peer-urls=https://%s.%s.%s:{{peer_port}} \ + //--initial-cluster=%s \ + //--initial-cluster-token=etcd-{{cluster}}-%s + //`, + // hostPath, + // host, e.clusterName(), e.privateZone(), + // host, e.clusterName(), e.privateZone(), + // initialCluster, + // token, + // ), store) + // err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + // if err != nil { + // return err + // } + // + // cmdArgs = e.template(`set -e; + // sudo chown -R etcd:etcd /var/lib/etcd/{{cluster}} + // `, store) + // err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + // if err != nil { + // return err + // } + // + // return nil + // } + // + startEtcdFunc := func(host string, store map[string]string) error { + cmdArgs := fmt.Sprintf(`set -e; sudo systemctl start etcd-%s`, store["cluster"]) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + return nil + } + + healthCheckFunc := func(host string, store map[string]string) error { + endpoints := e.endpointsString(host, store) + for _, a := range aliases[1:] { + endpoints = strings.Join( + []string{endpoints, e.endpointsString(a, store)}, ",", + ) + } + + cmdArgs := e.template(fmt.Sprint(`%s; sudo /opt/bin/etcdctl endpoint health --endpoints=%s `, + envCmd, endpoints), store) + err = snapshot.SSHCmd(e, host, cmdArgs, nil, nil, nil) + if err != nil { + return err + } + + return nil + } + + for _, store := range stores { + value := e.restoreFlagValue(store["cluster"]) + if value == "" { + continue + } + + b := make([]byte, 32) + _, err := rand.Read(b) + if err != nil { + return fmt.Errorf("failed to create random etcd initial token: %s", err) + } + token := base64.URLEncoding.EncodeToString(b) + + fmt.Printf(">%s\n", token) + + for _, a := range aliases { + e.log.Infof("stopping etcd unit %s on host %s", store["cluster"], a) + err := stopEtcdFunc(a, store) + if err != nil { + return err + } + } + + time.Sleep(5 * time.Second) + + //for _, a := range aliases { + // e.log.Infof("restoring etcd %s on host %s", store["cluster"], a) + // err := restoreFunc(a, value, token, store) + // if err != nil { + // return err + // } + //} + + var wg sync.WaitGroup + var result *multierror.Error + var errLock sync.Mutex + + wg.Add(len(aliases)) + + for _, a := range aliases { + e.log.Infof("starting etcd %s on host %s", store["cluster"], a) + + go func() { + defer wg.Done() + + err := startEtcdFunc(a, store) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + + }() + } + + wg.Wait() + + if result != nil { + return result + } + + for _, a := range aliases { + e.log.Infof("checking health of etcd %s on host %s", store["cluster"], a) + err := healthCheckFunc(a, store) + if err != nil { + return err + } + } + + e.log.Infof("successfully restored etcd cluster %s with snapshot %s", store["cluster"], value) + + select { + case <-e.tarmak.CancellationContext().Done(): + return e.tarmak.CancellationContext().Err() + default: + } + } + + e.log.Info("restarting API servers on master hosts") + //masters, err := snapshot.Prepare(e.tarmak, clusterv1alpha1.InstancePoolTypeMaster) + //for _, master := range masters { + // cmdArgs := " sudo systemctl restart kube-apiserver" + // err = snapshot.SSHCmd(e, master, cmdArgs, nil, nil, nil) + // if err != nil { + // return err + // } + //} + + return nil +} + +func (e *Etcd) template(args string, vars map[string]string) string { + for k, v := range vars { + args = strings.Replace(args, fmt.Sprintf("{{%s}}", k), v, -1) + } + + return args +} + +func (e *Etcd) restoreFlagValue(flag string) string { + rf := e.tarmak.ClusterFlags().Snapshot.Etcd.Restore + for _, db := range []struct { + name, value string + }{ + {consts.RestoreK8sMainFlagName, rf.K8sMain}, + {consts.RestoreK8sEventsFlagName, rf.K8sEvents}, + {consts.RestoreOverlayFlagName, rf.Overlay}, + } { + if db.name == flag { + return db.value + } + } + + return "" +} + +func (e *Etcd) endpointsString(host string, store map[string]string) string { + return fmt.Sprintf("%s.%s.%s=https://%s.%s.%s:%s", + host, e.clusterName(), e.privateZone(), + host, e.clusterName(), e.privateZone(), store["client_port"]) +} + +func (e *Etcd) Log() *logrus.Entry { + return e.log +} + +func (e *Etcd) SSH() interfaces.SSH { + return e.ssh +} + +func (e *Etcd) clusterName() string { + return e.tarmak.Cluster().ClusterName() +} + +func (e *Etcd) privateZone() string { + return e.tarmak.Environment().Config().PrivateZone +} diff --git a/pkg/tarmak/snapshot/snapshot.go b/pkg/tarmak/snapshot/snapshot.go new file mode 100644 index 0000000000..810947c550 --- /dev/null +++ b/pkg/tarmak/snapshot/snapshot.go @@ -0,0 +1,182 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package snapshot + +import ( + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/hashicorp/go-multierror" + + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" + "github.com/jetstack/tarmak/pkg/tarmak/utils" +) + +const ( + TimeLayout = "2006-01-02_15-04-05" + GZipCCmd = "gzip -c %s" + GZipDCmd = "gzip -d > %s" +) + +func Prepare(tarmak interfaces.Tarmak, role string) (aliases []string, err error) { + if err := tarmak.SSH().WriteConfig(tarmak.Cluster()); err != nil { + return nil, err + } + + hosts, err := tarmak.Cluster().ListHosts() + if err != nil { + return nil, err + } + + var result *multierror.Error + for _, host := range hosts { + if utils.SliceContainsPrefix(host.Roles(), role) { + if len(host.Aliases()) == 0 { + err := fmt.Errorf( + "host with correct role '%v' found without alias: %v", + host.Roles(), + host.ID(), + ) + result = multierror.Append(result, err) + continue + } + + aliases = append(aliases, host.Aliases()[0]) + } + } + + if result != nil { + return nil, result + } + + if len(aliases) == 0 { + return nil, fmt.Errorf("no host aliases were found with role %s", role) + } + + return aliases, result.ErrorOrNil() +} + +func TarFromStream(sshCmd func() error, stream io.Reader, path string) error { + var result *multierror.Error + var errLock sync.Mutex + var wg sync.WaitGroup + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + wg.Add(1) + go func() { + defer wg.Done() + + if err := sshCmd(); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + return + }() + + gzr, err := gzip.NewReader(stream) + if err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + defer gzr.Close() + + if _, err := io.Copy(f, gzr); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + + wg.Wait() + + return result.ErrorOrNil() +} + +func TarToStream(sshCmd func() error, stream io.WriteCloser, src string) error { + var result *multierror.Error + var errLock sync.Mutex + var wg sync.WaitGroup + + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + wg.Add(1) + go func() { + defer wg.Done() + + if err := sshCmd(); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + return + }() + + gzw := gzip.NewWriter(stream) + defer gzw.Close() + if _, err := io.Copy(gzw, f); err != nil { + + errLock.Lock() + result = multierror.Append(result, err) + errLock.Unlock() + + } + + f.Close() + gzw.Close() + stream.Close() + wg.Wait() + + return result.ErrorOrNil() +} + +func SSHCmd(s interfaces.Snapshot, host, cmd string, stdin io.Reader, stdout, stderr io.Writer) error { + for _, w := range []struct { + writer io.Writer + std string + }{ + {stdout, "out"}, + {stderr, "err"}, + } { + + if w.writer == nil { + var reader *io.PipeReader + reader, w.writer = io.Pipe() + scanner := bufio.NewScanner(reader) + + go func() { + for scanner.Scan() { + s.Log().WithField("std", w.std).Warn(scanner.Text()) + } + }() + } + } + + ret, err := s.SSH().Execute(host, strings.Split(cmd, " "), stdin, stdout, stderr) + if ret != 0 { + return fmt.Errorf("command [%s] returned non-zero (%d): %s", cmd, ret, err) + } + + return err +} diff --git a/pkg/tarmak/utils/consts/consts.go b/pkg/tarmak/utils/consts/consts.go index 98c07253f4..4ca16fb8d1 100644 --- a/pkg/tarmak/utils/consts/consts.go +++ b/pkg/tarmak/utils/consts/consts.go @@ -9,4 +9,8 @@ const ( DefaultKubeconfigPath = "${TARMAK_CONFIG}/${CURRENT_CLUSTER}/kubeconfig" KubeconfigFlagName = "public-api-endpoint" + + RestoreK8sMainFlagName = "k8s-main" + RestoreK8sEventsFlagName = "k8s-events" + RestoreOverlayFlagName = "overlay" ) diff --git a/pkg/tarmak/utils/slices.go b/pkg/tarmak/utils/slices.go index 726e64d6c9..9dcc46e6dd 100644 --- a/pkg/tarmak/utils/slices.go +++ b/pkg/tarmak/utils/slices.go @@ -1,6 +1,8 @@ // Copyright Jetstack Ltd. See LICENSE for details. package utils +import "strings" + func RemoveDuplicateStrings(slice []string) (result []string) { seen := make(map[string]bool) @@ -46,3 +48,12 @@ func IndexOfString(slice []string, str string) int { return -1 } + +func SliceContainsPrefix(slice []string, prefix string) bool { + for _, s := range slice { + if strings.HasPrefix(s, prefix) { + return true + } + } + return false +} diff --git a/puppet/modules/etcd/manifests/init.pp b/puppet/modules/etcd/manifests/init.pp index 1dc08d0d7f..fd4b83f35a 100644 --- a/puppet/modules/etcd/manifests/init.pp +++ b/puppet/modules/etcd/manifests/init.pp @@ -13,6 +13,7 @@ $gid = $::etcd::params::gid, $user = $::etcd::params::user, $group = $::etcd::params::group, + $bin_dir = $::etcd::params::bin_dir, Boolean $backup_enabled = false, Enum['aws:kms',''] $backup_sse = '', String $backup_bucket_prefix = '', diff --git a/puppet/modules/etcd/manifests/install.pp b/puppet/modules/etcd/manifests/install.pp index ccd25240e7..871e70fca1 100644 --- a/puppet/modules/etcd/manifests/install.pp +++ b/puppet/modules/etcd/manifests/install.pp @@ -33,5 +33,8 @@ creates => "${dest_dir}/etcd", path => ['/usr/bin/', '/bin'], } - + -> file { "${::etcd::bin_dir}/etcdctl": + ensure => 'link', + target => "${dest_dir}/etcdctl", + } }