Skip to content

Commit

Permalink
Bug 2005816: make projectid and region anonymization consistent (#534)
Browse files Browse the repository at this point in the history
* make projectid and region anonymization consistent

* anonymized the rest

* cluster-config-v1

* test

* anonymize service accounts

* lint
  • Loading branch information
Serhii Zakharov authored Dec 15, 2021
1 parent f23f521 commit 0fefe84
Show file tree
Hide file tree
Showing 11 changed files with 195 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,6 @@
]
},
"data": {
"install-config": "apiVersion: v1\nbaseDomain: xxxxxxxxxxxxxxxxxxx\ncompute:\n- hyperthreading: Enabled\n name: worker\n platform: {}\n replicas: 3\ncontrolPlane:\n hyperthreading: Enabled\n name: master\n platform: {}\n replicas: 3\nmetadata:\n creationTimestamp: null\n name: tremes-testing.tremes.2021.07.12.ccxdev.devshift.net\nnetworking:\n clusterNetwork:\n - cidr: 10.128.0.0/14\n hostPrefix: 23\n machineNetwork:\n - cidr: 10.0.0.0/16\n networkType: OpenShiftSDN\n serviceNetwork:\n - 172.30.0.0/16\nplatform:\n aws:\n region: us-east-2\npublish: External\npullSecret: \"\"\n"
"install-config": "apiVersion: v1\nbaseDomain: xxxxxxxxxxxxxxxxxxx\ncompute:\n- hyperthreading: Enabled\n name: worker\n platform: {}\n replicas: 3\ncontrolPlane:\n hyperthreading: Enabled\n name: master\n platform: {}\n replicas: 3\nmetadata:\n creationTimestamp: null\n name: tremes-testing.tremes.2021.07.12.ccxdev.devshift.net\nnetworking:\n clusterNetwork:\n - cidr: 10.128.0.0/14\n hostPrefix: 23\n machineNetwork:\n - cidr: 10.0.0.0/16\n networkType: OpenShiftSDN\n serviceNetwork:\n - 172.30.0.0/16\nplatform:\n aws:\n region: xxxxxxxxx\npublish: External\npullSecret: \"\"\n"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
}
],
"projectID": "openshift-gce-devel-ci",
"region": "us-east1",
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
"region": "xxxxxxxx",
"serviceAccounts": [
{
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
}
],
"projectID": "openshift-gce-devel-ci",
"region": "us-east1",
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
"region": "xxxxxxxx",
"serviceAccounts": [
{
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@
"subnetwork": "ci-ln-fyvthbt-f76d1-nl2fh-worker-subnet"
}
],
"projectID": "openshift-gce-devel-ci",
"region": "us-east1",
"projectID": "xxxxxxxxxxxxxxxxxxxxxx",
"region": "xxxxxxxx",
"serviceAccounts": [
{
"email": "ci-ln-fyvthbt-f76d1-nl2fh-w@openshift-gce-devel-ci.iam.gserviceaccount.com",
Expand Down
20 changes: 20 additions & 0 deletions pkg/gatherers/clusterconfig/cluster_config_v1_config_map.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,25 @@ func anonymizeInstallConfig(installConfig *installertypes.InstallConfig) *instal
// we don't use it
installConfig.BaseDomain = anonymize.String(installConfig.BaseDomain)

if installConfig.Platform.AWS != nil {
installConfig.Platform.AWS.Region = anonymize.String(installConfig.Platform.AWS.Region)
}
if installConfig.Platform.Azure != nil {
installConfig.Platform.Azure.Region = anonymize.String(installConfig.Platform.Azure.Region)
}
if installConfig.Platform.GCP != nil {
installConfig.Platform.GCP.Region = anonymize.String(installConfig.Platform.GCP.Region)
installConfig.Platform.GCP.ProjectID = anonymize.String(installConfig.Platform.GCP.ProjectID)
}
if installConfig.Platform.VSphere != nil {
installConfig.Platform.VSphere.Datacenter = anonymize.String(installConfig.Platform.VSphere.Datacenter)
installConfig.Platform.VSphere.Username = anonymize.String(installConfig.Platform.VSphere.Username)
installConfig.Platform.VSphere.Password = anonymize.String(installConfig.Platform.VSphere.Password)
}
if installConfig.Platform.OpenStack != nil {
installConfig.Platform.OpenStack.Region = anonymize.String(installConfig.Platform.OpenStack.Region)
installConfig.Platform.OpenStack.Cloud = anonymize.String(installConfig.Platform.OpenStack.Cloud)
}

return installConfig
}
50 changes: 50 additions & 0 deletions pkg/gatherers/clusterconfig/cluster_config_v1_config_map_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package clusterconfig

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubefake "k8s.io/client-go/kubernetes/fake"
)

func Test_gatherClusterConfigV1(t *testing.T) {
coreClient := kubefake.NewSimpleClientset()

_, err := coreClient.CoreV1().ConfigMaps("kube-system").Create(context.Background(), &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-config-v1",
},
Immutable: nil,
Data: map[string]string{
"install-config": "{}",
},
BinaryData: nil,
}, metav1.CreateOptions{})
assert.NoError(t, err)

records, errs := gatherClusterConfigV1(context.Background(), coreClient.CoreV1())
assert.Empty(t, errs)

assert.Len(t, records, 1)
assert.Equal(t, "config/configmaps/kube-system/cluster-config-v1", records[0].Name)

data, err := records[0].Item.Marshal(context.Background())
assert.NoError(t, err)

installConfig := `baseDomain: \"\"\nmetadata:\n creationTimestamp: null\nplatform: {}\npullSecret: \"\"\n`

assert.JSONEq(t, `{
"metadata": {
"name": "cluster-config-v1",
"namespace": "kube-system",
"creationTimestamp": null
},
"data": {
"install-config": "`+installConfig+`"
}
}`, string(data))
}
8 changes: 5 additions & 3 deletions pkg/gatherers/clusterconfig/image_registries.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,21 +137,23 @@ func anonymizeImageRegistry(config *registryv1.Config) *registryv1.Config {
if config.Spec.Storage.IBMCOS != nil {
anonymizeIBMCOSStorage(config.Spec.Storage.IBMCOS)
}

if config.Status.Storage.S3 != nil {
anonymizeS3Storage(config.Status.Storage.S3)
}
if config.Status.Storage.GCS != nil {
anonymizeGCSStorage(config.Status.Storage.GCS)
}
if config.Status.Storage.Azure != nil {
anonymizeAzureStorage(config.Status.Storage.Azure)
}
if config.Status.Storage.GCS != nil {
anonymizeGCSStorage(config.Status.Storage.GCS)
}
if config.Status.Storage.Swift != nil {
anonymizeSwiftStorage(config.Status.Storage.Swift)
}
if config.Status.Storage.IBMCOS != nil {
anonymizeIBMCOSStorage(config.Status.Storage.IBMCOS)
}

// kubectl.kubernetes.io/last-applied-configuration annotation contains complete previous resource definition
// including the sensitive information as bucket, keyIDs, etc.
if lac, ok := config.Annotations[lacAnnotation]; ok {
Expand Down
20 changes: 20 additions & 0 deletions pkg/gatherers/clusterconfig/infrastructures.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,25 @@ func gatherClusterInfrastructure(ctx context.Context, configClient configv1clien

func anonymizeInfrastructure(config *configv1.Infrastructure) *configv1.Infrastructure {
config.Status.InfrastructureName = anonymize.URL(config.Status.InfrastructureName)

if config.Status.PlatformStatus.AWS != nil {
config.Status.PlatformStatus.AWS.Region = anonymize.String(config.Status.PlatformStatus.AWS.Region)
}
if config.Status.PlatformStatus.Azure != nil {
config.Status.PlatformStatus.Azure.CloudName = configv1.AzureCloudEnvironment(
anonymize.String(string(config.Status.PlatformStatus.Azure.CloudName)),
)
}
if config.Status.PlatformStatus.GCP != nil {
config.Status.PlatformStatus.GCP.Region = anonymize.String(config.Status.PlatformStatus.GCP.Region)
config.Status.PlatformStatus.GCP.ProjectID = anonymize.String(config.Status.PlatformStatus.GCP.ProjectID)
}
if config.Status.PlatformStatus.IBMCloud != nil {
config.Status.PlatformStatus.IBMCloud.Location = anonymize.String(config.Status.PlatformStatus.IBMCloud.Location)
}
if config.Status.PlatformStatus.OpenStack != nil {
config.Status.PlatformStatus.OpenStack.CloudName = anonymize.String(config.Status.PlatformStatus.OpenStack.CloudName)
}

return config
}
62 changes: 61 additions & 1 deletion pkg/gatherers/clusterconfig/machine_sets.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import (

"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/klog/v2"

"github.com/openshift/insights-operator/pkg/record"
"github.com/openshift/insights-operator/pkg/utils/anonymize"
)

// GatherMachineSet collects MachineSet information
Expand Down Expand Up @@ -52,9 +55,66 @@ func gatherMachineSet(ctx context.Context, dynamicClient dynamic.Interface) ([]r
}
records = append(records, record.Record{
Name: recordName,
Item: record.ResourceMarshaller{Resource: &machineSets.Items[i]},
Item: record.ResourceMarshaller{Resource: anonymizeMachineset(&machineSets.Items[i])},
})
}

return records, nil
}

func anonymizeMachineset(data *unstructured.Unstructured) *unstructured.Unstructured {
fieldsToAnonymize := [][]string{
{"spec", "template", "spec", "providerSpec", "value", "projectID"},
{"spec", "template", "spec", "providerSpec", "value", "region"},
{"spec", "template", "spec", "providerSpec", "value", "placement", "availabilityZone"},
{"spec", "template", "spec", "providerSpec", "value", "placement", "region"},
}

for _, fieldToAnonymize := range fieldsToAnonymize {
err := anonymize.UnstructuredNestedStringField(data.Object, fieldToAnonymize...)
if err != nil {
klog.Infof("error during anonymizing machineset: %v", err)
}
}

return anonymizeServiceAccounts(data)
}

func anonymizeServiceAccounts(data *unstructured.Unstructured) *unstructured.Unstructured {
serviceAccounts, found, err := unstructured.NestedSlice(
data.Object, "spec", "template", "spec", "providerSpec", "value", "serviceAccounts",
)
if !found || err != nil {
klog.Infof("error during anonymizing machineset: unable to find service accounts %v %v", found, err)
return data
}

for i := range serviceAccounts {
serviceAccount, ok := serviceAccounts[i].(map[string]interface{})
if !ok {
klog.Infof("error during anonymizing machineset: service account is not a map")
continue
}

emailI, found := serviceAccount["email"]
if !found {
klog.Infof("error during anonymizing machineset: email was not found in service account map")
continue
}

email, ok := emailI.(string)
if !ok {
klog.Infof("error during anonymizing machineset: email was not a string")
continue
}

serviceAccount["email"] = anonymize.String(email)
}

err = unstructured.SetNestedSlice(data.Object, serviceAccounts, "spec", "template", "spec", "providerSpec", "value", "serviceAccounts")
if err != nil {
klog.Infof("error during anonymizing machineset: unable to set anonymized service accounts: %v", err.Error())
}

return data
}
11 changes: 10 additions & 1 deletion pkg/gatherers/clusterconfig/nodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,30 @@ func anonymizeNode(node *corev1.Node) *corev1.Node {
if isProductNamespacedKey(k) {
continue
}

node.Annotations[k] = ""
}

for k, v := range node.Labels {
if isProductNamespacedKey(k) {
if isProductNamespacedKey(k) && !isRegionLabel(k) {
continue
}

node.Labels[k] = anonymize.String(v)
}

node.Status.NodeInfo.BootID = anonymize.String(node.Status.NodeInfo.BootID)
node.Status.NodeInfo.SystemUUID = anonymize.String(node.Status.NodeInfo.SystemUUID)
node.Status.NodeInfo.MachineID = anonymize.String(node.Status.NodeInfo.MachineID)
node.Status.Images = nil

return node
}

func isProductNamespacedKey(key string) bool {
return strings.Contains(key, "openshift.io/") || strings.Contains(key, "k8s.io/") || strings.Contains(key, "kubernetes.io/")
}

func isRegionLabel(key string) bool {
return key == "failure-domain.beta.kubernetes.io/region" || key == "topology.kubernetes.io/region"
}
22 changes: 22 additions & 0 deletions pkg/utils/anonymize/unstructured.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package anonymize

import (
"fmt"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

// UnstructuredNestedStringField anonymizes the nested field in the unstructured object
// or returns error if it is not possible
func UnstructuredNestedStringField(data map[string]interface{}, fields ...string) error {
value, found, err := unstructured.NestedFieldCopy(data, fields...)
if err != nil {
return err
}
if !found {
return fmt.Errorf("unable to find field '%v'", fields)
}

valueStr, _ := value.(string)
return unstructured.SetNestedField(data, String(valueStr), fields...)
}

0 comments on commit 0fefe84

Please sign in to comment.