Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for Kubernetes and OpenShift-type DevWorkspace components #961

Merged
merged 17 commits into from
Dec 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
923e74f
Add webhook for Kubernetes components to validate permissions
amisevsk Sep 21, 2022
ce4d787
Add ability to deserialize yaml/json to client.Object
amisevsk Oct 24, 2022
77987bc
Add tests for object deserialization
amisevsk Oct 24, 2022
3594b15
Update sync package to support creating objects it doesn't recognize
amisevsk Oct 25, 2022
370a4c0
Add support for syncing objects from kubernetes components to cluster
amisevsk Oct 25, 2022
f42300f
Plug syncing Kubernetes components to cluster into main reconcile loop
amisevsk Oct 25, 2022
2ad6419
Update sync diff options to avoid common loops with k8s components
amisevsk Oct 25, 2022
be44144
Make errors used in cluster sync API unwrappable
amisevsk Oct 25, 2022
1b38e5c
Add Pods to recognized objects in sync package
amisevsk Oct 25, 2022
8a1cc9f
Block creation of RBAC objects via DevWorkspace components
amisevsk Oct 25, 2022
e2cd1a7
Fail dw webhook if controller does not have permissions to manage type
amisevsk Oct 25, 2022
8651ae6
Add webhook handling for k8s components when v1alpha1 dws are used
amisevsk Oct 25, 2022
15dbdc1
Extend kube component webhooks to DevWorkspaceTemplates as well
amisevsk Oct 26, 2022
89b27f8
Ignore Kubernetes/OpenShift components that have deployByDefault=false
amisevsk Oct 26, 2022
3edb8e4
Add tests for provisioning Kubernetes components
amisevsk Oct 26, 2022
f839a58
Prevent adding DevWorkspace(Template)s through kube components
amisevsk Oct 27, 2022
d864400
Use pointer package to create int64 reference
amisevsk Nov 7, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions controllers/workspace/condition.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ var conditionOrder = []dw.DevWorkspaceConditionType{
dw.DevWorkspaceRoutingReady,
dw.DevWorkspaceServiceAccountReady,
conditions.PullSecretsReady,
conditions.KubeComponentsReady,
conditions.DeploymentReady,
dw.DevWorkspaceReady,
}
Expand Down
19 changes: 19 additions & 0 deletions controllers/workspace/devworkspace_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
containerlib "github.com/devfile/devworkspace-operator/pkg/library/container"
"github.com/devfile/devworkspace-operator/pkg/library/env"
"github.com/devfile/devworkspace-operator/pkg/library/flatten"
kubesync "github.com/devfile/devworkspace-operator/pkg/library/kubernetes"
"github.com/devfile/devworkspace-operator/pkg/library/projects"
"github.com/devfile/devworkspace-operator/pkg/provision/automount"
"github.com/devfile/devworkspace-operator/pkg/provision/metadata"
Expand Down Expand Up @@ -467,6 +468,24 @@ func (r *DevWorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request
allPodAdditions = append(allPodAdditions, pullSecretStatus.PodAdditions)
reconcileStatus.setConditionTrue(conditions.PullSecretsReady, "DevWorkspace secrets ready")

if kubesync.HasKubelikeComponent(workspace) {
if err := kubesync.HandleKubernetesComponents(workspace, clusterAPI); err != nil {
switch syncErr := err.(type) {
case *kubesync.RetryError:
reqLogger.Info(syncErr.Error())
reconcileStatus.setConditionFalse(conditions.KubeComponentsReady, "Waiting for DevWorkspace Kubernetes components to be created on cluster")
return reconcile.Result{Requeue: true}, nil
case *kubesync.FailError:
return r.failWorkspace(workspace, fmt.Sprintf("Error provisioning workspace Kubernetes components: %s", syncErr), metrics.ReasonBadRequest, reqLogger, &reconcileStatus)
case *kubesync.WarningError:
reconcileStatus.setConditionTrue(conditions.DevWorkspaceWarning, fmt.Sprintf("Warning in Kubernetes components: %s", syncErr))
default:
return reconcile.Result{}, err
}
}
reconcileStatus.setConditionTrue(conditions.KubeComponentsReady, "Kubernetes components ready")
amisevsk marked this conversation as resolved.
Show resolved Hide resolved
}

// Step six: Create deployment and wait for it to be ready
timing.SetTime(timingInfo, timing.DeploymentCreated)
deploymentStatus := wsprovision.SyncDeploymentToCluster(workspace, allPodAdditions, serviceAcctName, clusterAPI)
Expand Down
5 changes: 5 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/devfile/devworkspace-operator/pkg/cache"
"github.com/devfile/devworkspace-operator/pkg/config"
"github.com/devfile/devworkspace-operator/pkg/infrastructure"
kubesync "github.com/devfile/devworkspace-operator/pkg/library/kubernetes"
"github.com/devfile/devworkspace-operator/pkg/webhook"
"github.com/devfile/devworkspace-operator/version"

Expand Down Expand Up @@ -105,6 +106,10 @@ func main() {
setupLog.Info(fmt.Sprintf("Commit: %s", version.Commit))
setupLog.Info(fmt.Sprintf("BuildTime: %s", version.BuildTime))

if err := kubesync.InitializeDeserializer(scheme); err != nil {
setupLog.Error(err, "failed to initialized Kubernetes objects decoder")
}

cacheFunc, err := cache.GetCacheFunc()
if err != nil {
setupLog.Error(err, "failed to set up objects cache")
Expand Down
1 change: 1 addition & 0 deletions pkg/conditions/conditions.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const (
PullSecretsReady dw.DevWorkspaceConditionType = "PullSecretsReady"
DevWorkspaceResolved dw.DevWorkspaceConditionType = "DevWorkspaceResolved"
StorageReady dw.DevWorkspaceConditionType = "StorageReady"
KubeComponentsReady dw.DevWorkspaceConditionType = "KubernetesComponentsProvisioned"
DeploymentReady dw.DevWorkspaceConditionType = "DeploymentReady"
DevWorkspaceWarning dw.DevWorkspaceConditionType = "DevWorkspaceWarning"
)
Expand Down
36 changes: 36 additions & 0 deletions pkg/library/kubernetes/common_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright (c) 2019-2022 Red Hat, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
"github.com/devfile/devworkspace-operator/pkg/provision/sync"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
)

var (
testScheme = runtime.NewScheme()
testAPI = sync.ClusterAPI{
Scheme: testScheme,
}
)

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(testScheme))
utilruntime.Must(v1alpha1.AddToScheme(testScheme))
utilruntime.Must(dw.AddToScheme(testScheme))
}
58 changes: 58 additions & 0 deletions pkg/library/kubernetes/deserialize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright (c) 2019-2022 Red Hat, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
"fmt"
gosync "sync"

"github.com/devfile/devworkspace-operator/pkg/provision/sync"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var (
decoder runtime.Decoder
decoderMutex gosync.Mutex
)

func InitializeDeserializer(scheme *runtime.Scheme) error {
decoderMutex.Lock()
defer decoderMutex.Unlock()
if decoder != nil {
return fmt.Errorf("attempting to re-initialize kubernetes object decoder")
}
decoder = serializer.NewCodecFactory(scheme).UniversalDeserializer()
return nil
}

func deserializeToObject(jsonObj []byte, api sync.ClusterAPI) (client.Object, error) {
if decoder == nil {
return nil, fmt.Errorf("kubernetes object deserializer is not initialized")
}
obj, _, err := decoder.Decode(jsonObj, nil, nil)
if err != nil {
return nil, err
}
if obj.GetObjectKind().GroupVersionKind().Kind == "List" {
return nil, fmt.Errorf("objects of kind 'List' are unsupported")
}
clientObj, ok := obj.(client.Object)
if !ok {
// Should never occur but to avoid a panic
return nil, fmt.Errorf("object does not have standard metadata and cannot be processed")
}
return clientObj, nil
}
105 changes: 105 additions & 0 deletions pkg/library/kubernetes/deserialize_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Copyright (c) 2019-2022 Red Hat, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
"fmt"
"os"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)

func TestDeserializeObject(t *testing.T) {
if err := InitializeDeserializer(testScheme); err != nil {
t.Fatalf("Unexpected error: %s", err)
}
defer func() {
decoder = nil
}()
tests := []struct {
name string
filePath string
expectedObjStub client.Object
expectedErrRegexp string
}{
{
name: "Deserializes Pod",
filePath: "testdata/k8s_objects/pod.yaml",
expectedObjStub: &corev1.Pod{},
},
{
name: "Deserializes Service",
filePath: "testdata/k8s_objects/service.yaml",
expectedObjStub: &corev1.Service{},
},
{
name: "Deserializes ConfigMap",
filePath: "testdata/k8s_objects/configmap.yaml",
expectedObjStub: &corev1.ConfigMap{},
},
{
name: "Kubernetes list",
filePath: "testdata/k8s_objects/kubernetes-list.yaml",
expectedErrRegexp: "objects of kind 'List' are unsupported",
},
{
name: "Random yaml that is not a Kubernetes object",
filePath: "testdata/k8s_objects/non-k8s-object.yaml",
expectedErrRegexp: "Object 'Kind' is missing",
},
{
name: "Object with unrecognized kind",
filePath: "testdata/k8s_objects/unrecognized-kind.yaml",
expectedErrRegexp: "no kind .* is registered for version .* in scheme",
},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("%s (%s)", tt.name, tt.filePath), func(t *testing.T) {
jsonBytes := readBytesFromFile(t, tt.filePath)
actualObj, err := deserializeToObject(jsonBytes, testAPI)
if tt.expectedErrRegexp != "" {
if !assert.Error(t, err, "Expect error to be returned") {
return
}
assert.Regexp(t, tt.expectedErrRegexp, err.Error(), "Expect error to match pattern")
} else {
if !assert.NoError(t, err, "Expect no error to be returned") {
return
}
err := yaml.Unmarshal(jsonBytes, tt.expectedObjStub)
assert.NoError(t, err)
assert.True(t, cmp.Equal(tt.expectedObjStub, actualObj), cmp.Diff(tt.expectedObjStub, actualObj))
}
})
}
}

func TestErrorIfDeserializerNotInitialized(t *testing.T) {
_, err := deserializeToObject([]byte(""), testAPI)
assert.Error(t, err)
assert.Equal(t, "kubernetes object deserializer is not initialized", err.Error())
}

func readBytesFromFile(t *testing.T, filePath string) []byte {
bytes, err := os.ReadFile(filePath)
if err != nil {
t.Fatal(err)
}
return bytes
}
53 changes: 53 additions & 0 deletions pkg/library/kubernetes/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright (c) 2019-2022 Red Hat, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import "github.com/devfile/devworkspace-operator/pkg/provision/sync"

type RetryError struct {
Err error
}

func (e *RetryError) Error() string {
return e.Err.Error()
}

type FailError struct {
Err error
}

func (e *FailError) Error() string {
return e.Err.Error()
}

func wrapSyncError(err error) error {
switch syncErr := err.(type) {
case *sync.NotInSyncError:
return &RetryError{syncErr}
case *sync.UnrecoverableSyncError:
return &FailError{syncErr}
case *sync.WarningError:
return &WarningError{syncErr}
default:
return err
}
}

type WarningError struct {
Err error
}

func (e *WarningError) Error() string {
return e.Err.Error()
}
Loading