diff --git a/docs/aws-config.md b/docs/aws-config.md index add8b5a87d..a943738137 100644 --- a/docs/aws-config.md +++ b/docs/aws-config.md @@ -151,6 +151,31 @@ Specify the following values in the example files: * Replace `` with `gp2`. This is AWS's default `StorageClass` name. +* (Optional) If you have multiple clusters and you want to support migration of resources between them, in file `examples/aws/10-deployment.yaml`: + + * Uncomment the environment variable `AWS_CLUSTER_NAME` and replace `` with the current cluster's name. When restoring backup, it will make Ark (and cluster it's running on) claim ownership of AWS volumes created from snapshots taken on different cluster. + The best way to get the current cluster's name is to either check it with used deployment tool or to read it directly from the EC2 instances tags. + + The following listing shows how to get the cluster's nodes EC2 Tags. First, get the nodes external IDs (EC2 IDs): + + ```bash + kubectl get nodes -o jsonpath='{.items[*].spec.externalID}' + ``` + + Copy one of the returned IDs `` and use it with the `aws` CLI tool to search for one of the following: + + * The `kubernetes.io/cluster/` tag of the value `owned`. The `` is then your cluster's name: + + ```bash + aws ec2 describe-tags --filters "Name=resource-id,Values=" "Name=value,Values=owned" + ``` + + * If the first output returns nothing, then check for the legacy Tag `KubernetesCluster` of the value ``: + + ```bash + aws ec2 describe-tags --filters "Name=resource-id,Values=" "Name=key,Values=KubernetesCluster" + ``` + ## Start the server In the root of your Ark directory, run: @@ -278,4 +303,4 @@ It can be set up for Ark by creating a role that will have required permissions, [6]: config-definition.md#aws [14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html [20]: faq.md -[21]: backupstoragelocation-definition.md#aws \ No newline at end of file +[21]: backupstoragelocation-definition.md#aws diff --git a/examples/aws/10-deployment.yaml b/examples/aws/10-deployment.yaml index 1742c47135..5549fea67e 100644 --- a/examples/aws/10-deployment.yaml +++ b/examples/aws/10-deployment.yaml @@ -50,6 +50,8 @@ spec: value: /credentials/cloud - name: ARK_SCRATCH_DIR value: /scratch + #- name: AWS_CLUSTER_NAME + # value: volumes: - name: cloud-credentials secret: @@ -57,4 +59,4 @@ spec: - name: plugins emptyDir: {} - name: scratch - emptyDir: {} \ No newline at end of file + emptyDir: {} diff --git a/pkg/cloudprovider/aws/block_store.go b/pkg/cloudprovider/aws/block_store.go index 6e9e54e1b9..904735111d 100644 --- a/pkg/cloudprovider/aws/block_store.go +++ b/pkg/cloudprovider/aws/block_store.go @@ -17,7 +17,9 @@ limitations under the License. package aws import ( + "os" "regexp" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -95,6 +97,8 @@ func (b *blockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ s return "", errors.Errorf("expected 1 snapshot from DescribeSnapshots for %s, got %v", snapshotID, count) } + // filter tags through getTagsForCluster() function in order to apply + // proper ownership tags to restored volumes req := &ec2.CreateVolumeInput{ SnapshotId: &snapshotID, AvailabilityZone: &volumeAZ, @@ -102,7 +106,7 @@ func (b *blockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ s TagSpecifications: []*ec2.TagSpecification{ { ResourceType: aws.String(ec2.ResourceTypeVolume), - Tags: snapRes.Snapshots[0].Tags, + Tags: getTagsForCluster(snapRes.Snapshots[0].Tags), }, }, } @@ -180,6 +184,29 @@ func (b *blockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]s return *res.SnapshotId, nil } +func getTagsForCluster(snapshotTags []*ec2.Tag) []*ec2.Tag { + var result []*ec2.Tag + + clusterName, haveAWSClusterNameEnvVar := os.LookupEnv("AWS_CLUSTER_NAME") + + if haveAWSClusterNameEnvVar { + result = append(result, ec2Tag("kubernetes.io/cluster/"+clusterName, "owned")) + result = append(result, ec2Tag("KubernetesCluster", clusterName)) + } + + for _, tag := range snapshotTags { + if haveAWSClusterNameEnvVar && (strings.HasPrefix(*tag.Key, "kubernetes.io/cluster/") || *tag.Key == "KubernetesCluster") { + // if the AWS_CLUSTER_NAME variable is found we want current cluster + // to overwrite the old ownership on volumes + continue + } + + result = append(result, ec2Tag(*tag.Key, *tag.Value)) + } + + return result +} + func getTags(arkTags map[string]string, volumeTags []*ec2.Tag) []*ec2.Tag { var result []*ec2.Tag diff --git a/pkg/cloudprovider/aws/block_store_test.go b/pkg/cloudprovider/aws/block_store_test.go index b503ca8268..f92c5b39a6 100644 --- a/pkg/cloudprovider/aws/block_store_test.go +++ b/pkg/cloudprovider/aws/block_store_test.go @@ -17,6 +17,7 @@ limitations under the License. package aws import ( + "os" "sort" "testing" @@ -91,6 +92,91 @@ func TestSetVolumeID(t *testing.T) { assert.Equal(t, "vol-updated", actual) } +func TestGetTagsForCluster(t *testing.T) { + tests := []struct { + name string + isNameSet bool + snapshotTags []*ec2.Tag + expected []*ec2.Tag + }{ + { + name: "degenerate case (no tags)", + isNameSet: false, + snapshotTags: nil, + expected: nil, + }, + { + name: "cluster tags exist and remain set", + isNameSet: false, + snapshotTags: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "old-cluster"), + ec2Tag("kubernetes.io/cluster/old-cluster", "owned"), + ec2Tag("aws-key", "aws-val"), + }, + expected: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "old-cluster"), + ec2Tag("kubernetes.io/cluster/old-cluster", "owned"), + ec2Tag("aws-key", "aws-val"), + }, + }, + { + name: "cluster tags only get applied", + isNameSet: true, + snapshotTags: nil, + expected: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "current-cluster"), + ec2Tag("kubernetes.io/cluster/current-cluster", "owned"), + }, + }, + { + name: "non-overlaping cluster and snapshot tags both get applied", + isNameSet: true, + snapshotTags: []*ec2.Tag{ec2Tag("aws-key", "aws-val")}, + expected: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "current-cluster"), + ec2Tag("kubernetes.io/cluster/current-cluster", "owned"), + ec2Tag("aws-key", "aws-val"), + }, + }, + {name: "overlaping cluster tags, current cluster tags take precedence", + isNameSet: true, + snapshotTags: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "old-name"), + ec2Tag("kubernetes.io/cluster/old-name", "owned"), + ec2Tag("aws-key", "aws-val"), + }, + expected: []*ec2.Tag{ + ec2Tag("KubernetesCluster", "current-cluster"), + ec2Tag("kubernetes.io/cluster/current-cluster", "owned"), + ec2Tag("aws-key", "aws-val"), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.isNameSet { + os.Setenv("AWS_CLUSTER_NAME", "current-cluster") + } + res := getTagsForCluster(test.snapshotTags) + + sort.Slice(res, func(i, j int) bool { + return *res[i].Key < *res[j].Key + }) + + sort.Slice(test.expected, func(i, j int) bool { + return *test.expected[i].Key < *test.expected[j].Key + }) + + assert.Equal(t, test.expected, res) + + if test.isNameSet { + os.Unsetenv("AWS_CLUSTER_NAME") + } + }) + } +} + func TestGetTags(t *testing.T) { tests := []struct { name string