From 8be18d86fa2d9dcea732561643c090dd013a0397 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Wed, 29 May 2024 13:52:58 +0200 Subject: [PATCH] Add an inttest for stack application Make sure that stacks that are badly ordered are eventually applied in their entirety. Signed-off-by: Tom Wieczorek --- inttest/Makefile.variables | 1 + inttest/stackapplier/rings.yaml | 82 ++++++++++++++ inttest/stackapplier/stackapplier_test.go | 132 ++++++++++++++++++++++ pkg/kubernetes/watch/unstructured.go | 25 ++++ 4 files changed, 240 insertions(+) create mode 100644 inttest/stackapplier/rings.yaml create mode 100644 inttest/stackapplier/stackapplier_test.go create mode 100644 pkg/kubernetes/watch/unstructured.go diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index 0f2582b89210..2573c91b69b5 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -59,5 +59,6 @@ smoketests := \ check-psp \ check-reset \ check-singlenode \ + check-stackapplier \ check-statussocket \ check-upgrade \ diff --git a/inttest/stackapplier/rings.yaml b/inttest/stackapplier/rings.yaml new file mode 100644 index 000000000000..b7985019b575 --- /dev/null +++ b/inttest/stackapplier/rings.yaml @@ -0,0 +1,82 @@ +# Have the wrong order: First the custom resources, then their definitions. Let +# one of the the resource be a cluster resource, so that the Stack's resource +# reordering won't let the CRD go before the CR. + +--- +apiVersion: k0s.example.com/v1 +kind: Character +metadata: + name: frodo + namespace: shire +spec: + speciesRef: + name: hobbit + +--- +apiVersion: k0s.example.com/v1 +kind: Species +metadata: + name: hobbit +spec: + characteristics: hairy feet + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: shire + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: species.k0s.example.com +spec: + group: k0s.example.com + names: + kind: Species + singular: species + plural: species + scope: Cluster + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + characteristics: + type: string + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: characters.k0s.example.com +spec: + group: k0s.example.com + names: + kind: Character + singular: character + plural: characters + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + speciesRef: + type: object + properties: + name: + type: string diff --git a/inttest/stackapplier/stackapplier_test.go b/inttest/stackapplier/stackapplier_test.go new file mode 100644 index 000000000000..aa2b5251e153 --- /dev/null +++ b/inttest/stackapplier/stackapplier_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nllb + +import ( + "context" + _ "embed" + "fmt" + "testing" + "time" + + "github.com/k0sproject/k0s/inttest/common" + "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + + "github.com/stretchr/testify/assert" + testifysuite "github.com/stretchr/testify/suite" + "sigs.k8s.io/yaml" +) + +type suite struct { + common.BootlooseSuite +} + +//go:embed rings.yaml +var rings string + +func (s *suite) TestStackApplier() { + ctx, cancel := context.WithCancelCause(s.Context()) + s.T().Cleanup(func() { cancel(nil) }) + + k0sConfig, err := yaml.Marshal(&v1beta1.ClusterConfig{ + Spec: &v1beta1.ClusterSpec{ + Storage: &v1beta1.StorageSpec{Type: v1beta1.KineStorageType}, + }, + }) + s.Require().NoError(err) + + s.WriteFileContent(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfig) + s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=control-api,konnectivity-server,kube-controller-manager,kube-scheduler")) + s.MakeDir(s.ControllerNode(0), "/var/lib/k0s/manifests/rings") + s.PutFile(s.ControllerNode(0), "/var/lib/k0s/manifests/rings/rings.yaml", rings) + + kubeconfig, err := s.GetKubeConfig(s.ControllerNode(0)) + s.Require().NoError(err) + client, err := dynamic.NewForConfig(kubeconfig) + s.Require().NoError(err) + + sgv := schema.GroupVersion{Group: "k0s.example.com", Version: "v1"} + + s.T().Run("hobbit", func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + cancel(fmt.Errorf("%s failed", t.Name())) + } + }) + t.Parallel() + species := client.Resource(sgv.WithResource("species")) + assert.NoError(t, watch.Unstructured(species). + WithObjectName("hobbit"). + WithErrorCallback(retryWatchErrors(s.T().Logf)). + Until(ctx, func(item *unstructured.Unstructured) (bool, error) { + speciesName, found, err := unstructured.NestedString(item.Object, "spec", "characteristics") + if assert.NoError(t, err) && assert.True(t, found, "no characteristics found: %v", item.Object) { + assert.Equal(t, "hairy feet", speciesName) + } + return true, nil + })) + }) + + s.T().Run("frodo", func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + cancel(fmt.Errorf("%s failed", t.Name())) + } + }) + t.Parallel() + characters := client.Resource(sgv.WithResource("characters")) + assert.NoError(t, watch.Unstructured(characters.Namespace("shire")). + WithObjectName("frodo"). + WithErrorCallback(retryWatchErrors(s.T().Logf)). + Until(ctx, func(item *unstructured.Unstructured) (bool, error) { + speciesName, found, err := unstructured.NestedString(item.Object, "spec", "speciesRef", "name") + if assert.NoError(t, err) && assert.True(t, found, "no species found: %v", item.Object) { + assert.Equal(t, "hobbit", speciesName) + } + return true, nil + })) + }) +} + +func retryWatchErrors(logf common.LogfFn) watch.ErrorCallback { + commonRetry := common.RetryWatchErrors(logf) + return func(err error) (time.Duration, error) { + if retryDelay, err := commonRetry(err); err == nil { + return retryDelay, nil + } + if apierrors.IsNotFound(err) { + return 350 * time.Millisecond, nil + } + return 0, err + } +} + +func TestStackApplierSuite(t *testing.T) { + s := suite{ + common.BootlooseSuite{ + ControllerCount: 1, + WorkerCount: 0, + }, + } + testifysuite.Run(t, &s) +} diff --git a/pkg/kubernetes/watch/unstructured.go b/pkg/kubernetes/watch/unstructured.go new file mode 100644 index 000000000000..85db5460ea89 --- /dev/null +++ b/pkg/kubernetes/watch/unstructured.go @@ -0,0 +1,25 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func Unstructured(client Provider[*unstructured.UnstructuredList]) *Watcher[unstructured.Unstructured] { + return FromClient[*unstructured.UnstructuredList, unstructured.Unstructured](client) +}