diff --git a/entrypoint/clusterresetrestore/clusterresetrestore_test.go b/entrypoint/clusterresetrestore/clusterresetrestore_test.go index 7ded6a12..20b7e7f1 100644 --- a/entrypoint/clusterresetrestore/clusterresetrestore_test.go +++ b/entrypoint/clusterresetrestore/clusterresetrestore_test.go @@ -39,7 +39,7 @@ var _ = Describe("Test:", func() { // deploy more workloads before and after snapshot -- do not delete the workloads It("Verifies Cluster Reset Restore", func() { - testcase.TestClusterResetRestoreS3Snapshot(cluster, true, false) + testcase.TestClusterRestoreS3(cluster, true, false) }) // It("Verifies Ingress After Snapshot", func() { diff --git a/entrypoint/rebootinstances/rebootinstances_suite_test.go b/entrypoint/rebootinstances/rebootinstances_suite_test.go index a5a398b0..f58a7bb3 100644 --- a/entrypoint/rebootinstances/rebootinstances_suite_test.go +++ b/entrypoint/rebootinstances/rebootinstances_suite_test.go @@ -69,7 +69,7 @@ func cleanEIPs() { if release != "" && release == "false" { shared.LogLevel("info", "EIPs not released, being used to run test with kubeconfig") } else { - awsDependencies, err := aws.AddClient(cluster) + awsDependencies, err := aws.AddEc2Client(cluster) Expect(err).NotTo(HaveOccurred()) eips := append(cluster.ServerIPs, cluster.AgentIPs...) diff --git a/pkg/aws/client.go b/pkg/aws/client.go index 24107f4b..a90c1606 100644 --- a/pkg/aws/client.go +++ b/pkg/aws/client.go @@ -39,7 +39,3 @@ func AddEc2Client(c *shared.Cluster) (*Client, error) { ec2: ec2.New(sess), }, nil } - -func AddS3Client(c *shared.Cluster) (*Client, error) { - -} diff --git a/pkg/aws/ec2.go b/pkg/aws/ec2.go index c8d20634..0e8afd52 100644 --- a/pkg/aws/ec2.go +++ b/pkg/aws/ec2.go @@ -18,7 +18,7 @@ func (c Client) CreateInstances(names ...string) (externalIPs, privateIPs, ids [ } errChan := make(chan error, len(names)) - resChan := make(chan response, len(names)) + resChan := make(chan ec2response, len(names)) var wg sync.WaitGroup for _, n := range names { @@ -44,7 +44,7 @@ func (c Client) CreateInstances(names ...string) (externalIPs, privateIPs, ids [ return } - resChan <- response{nodeId: nodeID, externalIp: externalIp, privateIp: privateIp} + resChan <- ec2response{nodeId: nodeID, externalIp: externalIp, privateIp: privateIp} }(n) } go func() { diff --git a/pkg/testcase/clusterresetrestore.go b/pkg/testcase/clusterresetrestore.go index f8801fbe..1d53ff7e 100644 --- a/pkg/testcase/clusterresetrestore.go +++ b/pkg/testcase/clusterresetrestore.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/gomega" ) -func TestClusterResetRestoreS3Snapshot( +func TestClusterRestoreS3( cluster *shared.Cluster, applyWorkload, deleteWorkload bool, @@ -40,22 +40,45 @@ func TestClusterResetRestoreS3Snapshot( false, ) - onDemandPathCmd := fmt.Sprintf("sudo ls /var/lib/rancher/%s/server/db/snapshots", cluster.Config.Product) - onDemandPath, _ := shared.RunCommandOnNode(onDemandPathCmd, cluster.ServerIPs[0]) + onDemandPath, onDemandPathErr := shared.FetchSnapshotOnDemandPath(cluster.Config.Product, cluster.ServerIPs[0]) + Expect(onDemandPathErr).NotTo(HaveOccurred()) fmt.Println("\non-demand-path: ", onDemandPath) - clusterTokenCmd := fmt.Sprintf("sudo cat /var/lib/rancher/%s/server/token", cluster.Config.Product) - clusterToken, _ := shared.RunCommandOnNode(clusterTokenCmd, cluster.ServerIPs[0]) + clusterToken, clusterTokenErr := shared.FetchToken(cluster.Config.Product, cluster.ServerIPs[0]) + Expect(clusterTokenErr).NotTo(HaveOccurred()) fmt.Println("\ntoken: ", clusterToken) - // stopInstances(cluster) - // create fresh new VM and install K3s/RKE2 using RunCommandOnNode - // createNewServer(cluster) + // for i := 0; i < len(cluster.ServerIPs); i++ { + // shared.LogLevel("info", "stopping server instances: %s", cluster.ServerIPs[i]) + // } + stopServerInstances(cluster) + + // shared.LogLevel("info", "stopping agent instance: %s", cluster.AgentIPs[0]) + + stopAgentInstance(cluster) + + resourceName := os.Getenv("resource_name") + awsDependencies, err := aws.AddEc2Client(cluster) + Expect(err).NotTo(HaveOccurred(), "error adding aws nodes: %s", err) + + // create server names. + var serverName []string + + serverName = append(serverName, fmt.Sprintf("%s-server-fresh", resourceName)) + + externalServerIP, _, _, createErr := + awsDependencies.CreateInstances(serverName...) + Expect(createErr).NotTo(HaveOccurred(), createErr) + + shared.LogLevel("info", "Created server public ip: %s", + externalServerIP) + + // createNewServer(externalServerIP) // how do I delete the instances, bring up a new instance and install K3s/RKE2 using what we currently have? - shared.LogLevel("info", "running cluster reset on server %s\n", cluster.ServerIPs[0]) + // shared.LogLevel("info", "running cluster reset on server %s\n", cluster.ServerIPs[0]) // restoreS3Snapshot( // cluster, // s3Bucket, @@ -66,7 +89,6 @@ func TestClusterResetRestoreS3Snapshot( // secretAccessKey, // clusterToken, // ) - } // perform snapshot and list snapshot commands -- deploy workloads after snapshot [apply workload] @@ -103,16 +125,35 @@ func takeS3Snapshot( } -func stopInstances( - cluster *shared.Cluster, -) { +func stopServerInstances(cluster *shared.Cluster) { + + awsDependencies, err := aws.AddEc2Client(cluster) + Expect(err).NotTo(HaveOccurred()) + // stop server Instances + for i := 0; i < len(cluster.ServerIPs); i++ { + serverInstanceIDs, serverInstanceIDsErr := awsDependencies.GetInstanceIDByIP(cluster.ServerIPs[i]) + Expect(serverInstanceIDsErr).NotTo(HaveOccurred()) + fmt.Println(serverInstanceIDs) + awsDependencies.StopInstance(serverInstanceIDs) + } + +} +func stopAgentInstance(cluster *shared.Cluster) { + // stop agent Instances + awsDependencies, err := aws.AddEc2Client(cluster) + Expect(err).NotTo(HaveOccurred()) + + agentInstanceIDs, agentInstanceIDsErr := awsDependencies.GetInstanceIDByIP(cluster.AgentIPs[0]) + Expect(agentInstanceIDsErr).NotTo(HaveOccurred()) + fmt.Println(agentInstanceIDs) + awsDependencies.StopInstance(agentInstanceIDs) } func createNewServer(cluster *shared.Cluster) (externalServerIP []string) { resourceName := os.Getenv("resource_name") - awsDependencies, err := aws.AddClient(cluster) + awsDependencies, err := aws.AddEc2Client(cluster) Expect(err).NotTo(HaveOccurred(), "error adding aws nodes: %s", err) // create server names. @@ -120,24 +161,36 @@ func createNewServer(cluster *shared.Cluster) (externalServerIP []string) { serverName = append(serverName, fmt.Sprintf("%s-server-fresh", resourceName)) - externalServerIp, _, _, createErr := + externalServerIP, _, _, createErr := awsDependencies.CreateInstances(serverName...) Expect(createErr).NotTo(HaveOccurred(), createErr) - return externalServerIp + shared.LogLevel("info", "Created server public ip: %s", + externalServerIP) + + return externalServerIP } -// func installProduct(cluster *share.cluster) { +// func installProduct( +// cluster *share.cluster, +// token string, +// externalServerIP string, +// ) { // version := cluster.Config.Version + // if cluster.Config.Product == "k3s" { // installCmd := fmt.Sprintf("curl -sfL https://get.k3s.io/ | sudo INSTALL_K3S_VERSION=%s INSTALL_K3S_SKIP_ENABLE=true sh -", version) +// _, installCmdErr := shared.RunCommandOnNode(installCmd, externalServerIP) +// Expect(installCmdErr).NotTo(HaveOccurred()) // } else { // installCmd := fmt.Sprintf("curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_VERSION=%s sh -", version) // } - -// installRes, installCmdErr := shared.RunCommandOnNode(installCmd, cluster.ServerIPs[0]) // } +func installProduct(cluster *shared.Cluster, token string, externalServerIP string) { + +} + func restoreS3Snapshot( cluster *shared.Cluster, s3Bucket, diff --git a/pkg/testcase/rebootinstances.go b/pkg/testcase/rebootinstances.go index 27f599e6..bf305bbf 100644 --- a/pkg/testcase/rebootinstances.go +++ b/pkg/testcase/rebootinstances.go @@ -11,7 +11,7 @@ import ( ) func TestRebootInstances(cluster *shared.Cluster) { - awsDependencies, err := aws.AddClient(cluster) + awsDependencies, err := aws.AddEc2Client(cluster) Expect(err).NotTo(HaveOccurred()) // reboot server instances. diff --git a/pkg/testcase/upgradenodereplacement.go b/pkg/testcase/upgradenodereplacement.go index ba64bad5..f120806a 100644 --- a/pkg/testcase/upgradenodereplacement.go +++ b/pkg/testcase/upgradenodereplacement.go @@ -27,7 +27,7 @@ func TestUpgradeReplaceNode(cluster *shared.Cluster, flags *customflag.FlagConfi } resourceName := os.Getenv("resource_name") - awsDependencies, err := aws.AddClient(cluster) + awsDependencies, err := aws.AddEc2Client(cluster) Expect(err).NotTo(HaveOccurred(), "error adding aws nodes: %s", err) // create server names. diff --git a/shared/cluster.go b/shared/cluster.go index a0d2ca9b..a982f3e8 100644 --- a/shared/cluster.go +++ b/shared/cluster.go @@ -602,6 +602,15 @@ func FetchToken(product, ip string) (string, error) { return token, nil } +func FetchSnapshotOnDemandPath(product, ip string) (string, error) { + token, err := RunCommandOnNode(fmt.Sprintf("sudo ls /var/lib/rancher/%s/server/db/snapshots", product), ip) + if err != nil { + return "", ReturnLogError("failed to fetch snapshot on-demand path: %w\n", err) + } + + return token, nil +} + // PrintGetAll prints the output of kubectl get all -A -o wide and kubectl get nodes -o wide. func PrintGetAll() { kubeconfigFile := " --kubeconfig=" + KubeConfigFile diff --git a/shared/clusterconfig.go b/shared/clusterconfig.go index 36eb88ea..4de9d3cc 100644 --- a/shared/clusterconfig.go +++ b/shared/clusterconfig.go @@ -32,6 +32,7 @@ type Cluster struct { Config clusterConfig AwsEc2 awsEc2Config GeneralConfig generalConfig + AwsS3 awsS3Config } type awsEc2Config struct { @@ -47,6 +48,12 @@ type awsEc2Config struct { KeyName string } +type awsS3Config struct { + Bucket string + Region string + Folder string +} + type clusterConfig struct { RenderedTemplate string ExternalDb string