Skip to content

Commit

Permalink
functional tests + cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
jmdeal committed Sep 19, 2023
1 parent 9fa04d2 commit 21006dd
Show file tree
Hide file tree
Showing 4 changed files with 220 additions and 17 deletions.
27 changes: 13 additions & 14 deletions pkg/controllers/nodeclaim/tagging/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package nodeclaim
package tagging

import (
"context"
Expand All @@ -38,8 +38,8 @@ import (
)

const (
tagNodeClaim = corev1beta1.Group + "/nodeclaim"
tagName = "Name"
TagNodeClaim = corev1beta1.Group + "/nodeclaim"
TagName = "Name"
)

type Controller struct {
Expand All @@ -48,7 +48,7 @@ type Controller struct {
}

func NewController(kubeClient client.Client, instanceProvider *instance.Provider) corecontroller.Controller {
return corecontroller.Typed[*v1beta1.NodeClaim](kubeClient, &Controller{
return corecontroller.Typed[*corev1beta1.NodeClaim](kubeClient, &Controller{
kubeClient: kubeClient,
instanceProvider: instanceProvider,
})
Expand All @@ -58,8 +58,7 @@ func (c *Controller) Name() string {
return "nodeclaim.tagging"
}

func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (reconcile.Result, error) {
ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name))
func (c *Controller) Reconcile(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (reconcile.Result, error) {
storedNodeClaim := nodeClaim.DeepCopy()

if !isTaggable(nodeClaim) {
Expand All @@ -68,6 +67,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim

id, err := utils.ParseInstanceID(nodeClaim.Status.ProviderID)
if err != nil {
// We don't throw an error here since we don't want to retry until the ProviderID has been updated.
logging.FromContext(ctx).Errorf("failed to parse instance ID, %w", err)
return reconcile.Result{}, nil
}
Expand All @@ -80,10 +80,9 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim
return reconcile.Result{}, nil
}

nodeClaim.Annotations[v1beta1.AnnotationInstanceTagged] = "true"
nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationInstanceTagged: "true"})
if !equality.Semantic.DeepEqual(nodeClaim, storedNodeClaim) {
err = c.kubeClient.Patch(ctx, nodeClaim, client.MergeFrom(storedNodeClaim))
if err != nil {
if err := c.kubeClient.Patch(ctx, nodeClaim, client.MergeFrom(storedNodeClaim)); err != nil {
return reconcile.Result{}, client.IgnoreNotFound(err)
}
}
Expand All @@ -97,16 +96,16 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle
NewControllerManagedBy(m).
For(&corev1beta1.NodeClaim{}).
WithEventFilter(predicate.NewPredicateFuncs(func(o client.Object) bool {
nc := o.(*v1beta1.NodeClaim)
nc := o.(*corev1beta1.NodeClaim)
return isTaggable(nc)
})),
)
}

func (c *Controller) tagInstance(ctx context.Context, nc *v1beta1.NodeClaim, id string) error {
func (c *Controller) tagInstance(ctx context.Context, nc *corev1beta1.NodeClaim, id string) error {
tags := map[string]string{
tagName: nc.Status.NodeName,
tagNodeClaim: nc.Name,
TagName: nc.Status.NodeName,
TagNodeClaim: nc.Name,
}

// Remove tags which have been already populated
Expand All @@ -124,7 +123,7 @@ func (c *Controller) tagInstance(ctx context.Context, nc *v1beta1.NodeClaim, id
return nil
}

func isTaggable(nc *v1beta1.NodeClaim) bool {
func isTaggable(nc *corev1beta1.NodeClaim) bool {
// Instance has already been tagged
if val := nc.Annotations[v1beta1.AnnotationInstanceTagged]; val == "true" {
return false
Expand Down
204 changes: 204 additions & 0 deletions pkg/controllers/nodeclaim/tagging/suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tagging_test

import (
"context"
"fmt"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/samber/lo"
"sigs.k8s.io/controller-runtime/pkg/client"

. "github.com/aws/karpenter-core/pkg/test/expectations"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
. "knative.dev/pkg/logging/testing"

coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
coretest "github.com/aws/karpenter-core/pkg/test"
"github.com/aws/karpenter/pkg/apis"
"github.com/aws/karpenter/pkg/apis/settings"
"github.com/aws/karpenter/pkg/apis/v1beta1"
"github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging"
"github.com/aws/karpenter/pkg/fake"
"github.com/aws/karpenter/pkg/providers/instance"
"github.com/aws/karpenter/pkg/test"

"github.com/aws/karpenter-core/pkg/operator/controller"
"github.com/aws/karpenter-core/pkg/operator/injection"
"github.com/aws/karpenter-core/pkg/operator/options"
"github.com/aws/karpenter-core/pkg/operator/scheme"
)

var ctx context.Context
var awsEnv *test.Environment
var env *coretest.Environment
var taggingController controller.Controller
var opts options.Options

func TestAPIs(t *testing.T) {
ctx = TestContextWithLogger(t)
RegisterFailHandler(Fail)
RunSpecs(t, "Tagging")
}

var _ = BeforeSuite(func() {
s := scheme.Scheme
env = coretest.NewEnvironment(s, coretest.WithCRDs(apis.CRDs...))
ctx = coresettings.ToContext(ctx, coretest.Settings())
ctx = settings.ToContext(ctx, test.Settings())
awsEnv = test.NewEnvironment(ctx, env)
taggingController = tagging.NewController(env.Client, awsEnv.InstanceProvider)
})
var _ = AfterSuite(func() {
Expect(env.Stop()).To(Succeed(), "Failed to stop environment")
})

var _ = BeforeEach(func() {
ctx = injection.WithOptions(ctx, opts)
awsEnv.Reset()
})

var _ = Describe("Tagging", func() {
var ec2Instance *ec2.Instance

BeforeEach(func() {
ec2Instance = &ec2.Instance{
State: &ec2.InstanceState{
Name: aws.String(ec2.InstanceStateNameRunning),
},
Tags: []*ec2.Tag{
{
Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)),
Value: aws.String("owned"),
},
{
Key: aws.String(corev1beta1.NodePoolLabelKey),
Value: aws.String("default"),
},
{
Key: aws.String(corev1beta1.ManagedByAnnotationKey),
Value: aws.String(settings.FromContext(ctx).ClusterName),
},
},
PrivateDnsName: aws.String(fake.PrivateDNSName()),
Placement: &ec2.Placement{
AvailabilityZone: aws.String(fake.DefaultRegion),
},
InstanceId: aws.String(fake.InstanceID()),
InstanceType: aws.String("m5.large"),
}

awsEnv.EC2API.Instances.Store(*ec2Instance.InstanceId, ec2Instance)
})
AfterEach(func() {
ExpectCleanedUp(ctx, env.Client)
})

It("shouldn't tag instances without a Node", func() {
nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{
Status: corev1beta1.NodeClaimStatus{
ProviderID: fake.ProviderID(*ec2Instance.InstanceId),
},
})

ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim))
Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged)))
Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool {
return *tag.Key == tagging.TagName
})).To(BeFalse())
})

It("should't tag instance with malformed providerID", func() {
nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{
Status: corev1beta1.NodeClaimStatus{
ProviderID: "Bad providerID",
NodeName: "default",
},
})

ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim))
Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged)))
Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool {
return *tag.Key == tagging.TagName
})).To(BeFalse())
})

It("should tag instance with a Node", func() {
nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{
Status: corev1beta1.NodeClaimStatus{
ProviderID: fake.ProviderID(*ec2Instance.InstanceId),
NodeName: "default",
},
})

ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim))
nodeClaim = ExpectExists(ctx, env.Client, nodeClaim)
Expect(nodeClaim.Annotations).To(HaveKey(v1beta1.AnnotationInstanceTagged))

inst := instance.NewInstance(ec2Instance).Tags
Expect(inst).To(HaveKeyWithValue(tagging.TagName, "default"))
Expect(inst).To(HaveKeyWithValue(tagging.TagNodeClaim, nodeClaim.Name))
})

DescribeTable(
"should tag taggable instances",
func(customTags ...string) {
nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{
Status: corev1beta1.NodeClaimStatus{
ProviderID: fake.ProviderID(*ec2Instance.InstanceId),
NodeName: "default",
},
})

for _, tag := range customTags {
ec2Instance.Tags = append(ec2Instance.Tags, &ec2.Tag{
Key: aws.String(tag),
Value: aws.String("custom-tag"),
})
}
awsEnv.EC2API.Instances.Store(*ec2Instance.InstanceId, ec2Instance)

ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim))
nodeClaim = ExpectExists(ctx, env.Client, nodeClaim)
Expect(nodeClaim.Annotations).To(HaveKey(v1beta1.AnnotationInstanceTagged))

expectedTags := map[string]string{
tagging.TagName: nodeClaim.Status.NodeName,
tagging.TagNodeClaim: nodeClaim.Name,
}
instanceTags := instance.NewInstance(ec2Instance).Tags
for tag, value := range expectedTags {
if lo.Contains(customTags, tag) {
value = "custom-tag"
}
Expect(instanceTags).To(HaveKeyWithValue(tag, value))
}
},
Entry("with only NodeClaim", tagging.TagName),
Entry("with only Name", tagging.TagNodeClaim),
Entry("with both Name and NodeClaim"),
Entry("with nothing", tagging.TagName, tagging.TagNodeClaim),
)
})
2 changes: 1 addition & 1 deletion pkg/fake/ec2api.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func (e *EC2API) CreateTagsWithContext(_ context.Context, input *ec2.CreateTagsI

// Upsert any tags that have the same key
newTagKeys := sets.New(lo.Map(input.Tags, func(t *ec2.Tag, _ int) string { return aws.StringValue(t.Key) })...)
instance.Tags = lo.Filter(input.Tags, func(t *ec2.Tag, _ int) bool { return newTagKeys.Has(aws.StringValue(t.Key)) })
instance.Tags = lo.Filter(instance.Tags, func(t *ec2.Tag, _ int) bool { return !newTagKeys.Has(aws.StringValue(t.Key)) })
instance.Tags = append(instance.Tags, input.Tags...)
}
return nil, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/providers/instance/instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,12 @@ func (p *Provider) Delete(ctx context.Context, id string) error {
return nil
}

func (p *Provider) CreateTags(ctx context.Context, resourceID string, tags map[string]string) error {
func (p *Provider) CreateTags(ctx context.Context, id string, tags map[string]string) error {
ec2Tags := lo.MapToSlice(tags, func(key, value string) *ec2.Tag {
return &ec2.Tag{Key: aws.String(key), Value: aws.String(value)}
})
_, err := p.ec2api.CreateTagsWithContext(ctx, &ec2.CreateTagsInput{
Resources: aws.StringSlice([]string{resourceID}),
Resources: aws.StringSlice([]string{id}),
Tags: ec2Tags,
})
if err != nil {
Expand Down

0 comments on commit 21006dd

Please sign in to comment.