Skip to content

Commit

Permalink
Modify API Resource Collector to Convert YAML to JSON Format
Browse files Browse the repository at this point in the history
Modify API resource collector to detect and convert fetched resources to json format when they are returned in yaml format. This is required because some API resources are not available in json format and need to be converted to json format for reading by OpenSCAP.
  • Loading branch information
Vincent056 committed Mar 21, 2023
1 parent 65f9879 commit 35b839c
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 3 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).

- Fix openscap image substitution in Makefile so that the correct image is used.

- Modify API resource collector to detect if fetched resource is yaml string and
convert it to json when found, this is necessary because some of the API resources
are not available in json format, and we need to convert it to json format so that
it can be read by OpenSCAP.

### Deprecations

-
Expand Down
26 changes: 23 additions & 3 deletions cmd/manager/scap.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
mcfgcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
"github.com/wI2L/jsondiff"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
runtimejson "k8s.io/apimachinery/pkg/runtime/serializer/json"
Expand Down Expand Up @@ -652,9 +653,28 @@ func filter(ctx context.Context, rawobj []byte, filter string) ([]byte, error) {
return nil, err
}

out, marshallErr := json.Marshal(&v)
if marshallErr != nil {
return nil, fmt.Errorf("Error marshalling json: %w", marshallErr)
var out []byte
var err error
switch val := v.(type) {
case string:
// If filter result is a string type, check if it is YAML
var yamlData map[string]interface{}
err = yaml.Unmarshal([]byte(val), &yamlData)
if err != nil {
// If it is not YAML, return the string as is
out = []byte(val)
} else {
// If it is YAML, convert it to JSON
out, err = json.Marshal(yamlData)
if err != nil {
return nil, fmt.Errorf("error marshalling JSON: %w", err)
}
}
default:
out, err = json.Marshal(&v)
if err != nil {
return nil, fmt.Errorf("error marshalling JSON: %w", err)
}
}
_, isNotEOF := iter.Next()
if isNotEOF {
Expand Down
36 changes: 36 additions & 0 deletions cmd/manager/scap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,42 @@ var _ = Describe("Testing filtering", func() {
})
})

Context("Filtering configmaps yaml data", func() {
var rawcm []byte
expectedYAML := `{"admission":{},"aggregatorConfig":{"allowedNames":null,"clientCA":"","extraHeaderPrefixes":null,"groupHeaders":null,"usernameHeaders":null},"apiServerArguments":{"audit-log-format":["json"],"audit-log-maxsize":["100"],"audit-log-path":["/var/log/openshift-apiserver/audit.log"],"audit-policy-file":["/etc/kubernetes/audit-config/policy.yaml"],"shutdown-delay-duration":["3s"]},"apiVersion":"openshiftcontrolplane.config.openshift.io/v1","auditConfig":{"auditFilePath":"","enabled":false,"logFormat":"","maximumFileRetentionDays":0,"maximumFileSizeMegabytes":0,"maximumRetainedFiles":0,"policyConfiguration":null,"policyFile":"","webHookKubeConfig":"","webHookMode":""},"cloudProviderFile":"","corsAllowedOrigins":null,"imagePolicyConfig":{"additionalTrustedCA":"","allowedRegistriesForImport":null,"externalRegistryHostnames":null,"internalRegistryHostname":"image-registry.openshift-image-registry.svc:5000","maxImagesBulkImportedPerRepository":0},"jenkinsPipelineConfig":{"autoProvisionEnabled":null,"parameters":null,"serviceName":"","templateName":"","templateNamespace":""},"kind":"OpenShiftAPIServerConfig","kubeClientConfig":{"connectionOverrides":{"acceptContentTypes":"","burst":0,"contentType":"","qps":0},"kubeConfig":"/etc/kubernetes/secrets/svc-kubeconfig/kubeconfig"},"projectConfig":{"defaultNodeSelector":"","projectRequestMessage":"","projectRequestTemplate":""},"routingConfig":{"subdomain":"apps.wenshen-hypershift.devcluster.openshift.com"},"serviceAccountOAuthGrantMethod":"","servingInfo":{"bindAddress":"","bindNetwork":"","certFile":"/etc/kubernetes/certs/serving/tls.crt","cipherSuites":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"],"clientCA":"/etc/kubernetes/certs/client-ca/ca.crt","keyFile":"/etc/kubernetes/certs/serving/tls.key","maxRequestsInFlight":0,"minTLSVersion":"VersionTLS12","requestTimeoutSeconds":0},"storageConfig":{"ca":"/etc/kubernetes/certs/etcd-client-ca/ca.crt","certFile":"/etc/kubernetes/certs/etcd-client/etcd-client.crt","keyFile":"/etc/kubernetes/certs/etcd-client/etcd-client.key","storagePrefix":"","urls":["https://etcd-client:2379"]}}`
BeforeEach(func() {
cmFile, err := os.Open("../../tests/data/configmap_yaml.json")
Expect(err).To(BeNil())
var readErr error
rawcm, readErr = io.ReadAll(cmFile)
Expect(readErr).To(BeNil())
})
It("filters configmaps YAML data appropriately", func() {
filteredOut, filterErr := filter(context.TODO(), rawcm,
`.data["config.yaml"]`)
Expect(filterErr).To(BeNil())
Expect(string(filteredOut)).To(Equal(expectedYAML))
})
})

Context("Filtering configmaps json data", func() {
var rawcm []byte
expectedJSON := `{"apiServerArguments":{"audit-log-format":["json"],"audit-log-maxbackup":["10"],"audit-log-maxsize":["100"],"audit-log-path":["/var/log/openshift-apiserver/audit.log"],"audit-policy-file":["/var/run/configmaps/audit/policy.yaml"],"shutdown-delay-duration":["15s"],"shutdown-send-retry-after":["true"]},"apiVersion":"openshiftcontrolplane.config.openshift.io/v1","imagePolicyConfig":{"internalRegistryHostname":"image-registry.openshift-image-registry.svc:5000"},"kind":"OpenShiftAPIServerConfig","projectConfig":{"projectRequestMessage":""},"routingConfig":{"subdomain":"apps.ci-ln-xllhdgb-76ef8.origin-ci-int-aws.dev.rhcloud.com"},"servingInfo":{"bindNetwork":"tcp","cipherSuites":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"],"minTLSVersion":"VersionTLS12"},"storageConfig":{"urls":["https://10.0.137.27:2379","https://10.0.158.132:2379","https://10.0.204.8:2379"]}}`
BeforeEach(func() {
cmFile, err := os.Open("../../tests/data/configmap_json.json")
Expect(err).To(BeNil())
var readErr error
rawcm, readErr = io.ReadAll(cmFile)
Expect(readErr).To(BeNil())
})
It("filters configmaps JSON data appropriately", func() {
filteredOut, filterErr := filter(context.TODO(), rawcm,
`.data["config.yaml"] | fromjson`)
Expect(filterErr).To(BeNil())
Expect(string(filteredOut)).To(Equal(expectedJSON))
})
})

Context("Testing errors", func() {
It("outputs error if it can't create filter", func() {
_, filterErr := filter(context.TODO(), []byte{},
Expand Down
14 changes: 14 additions & 0 deletions tests/data/configmap_json.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"apiVersion": "v1",
"data": {
"config.yaml": "{\"apiServerArguments\":{\"audit-log-format\":[\"json\"],\"audit-log-maxbackup\":[\"10\"],\"audit-log-maxsize\":[\"100\"],\"audit-log-path\":[\"/var/log/openshift-apiserver/audit.log\"],\"audit-policy-file\":[\"/var/run/configmaps/audit/policy.yaml\"],\"shutdown-delay-duration\":[\"15s\"],\"shutdown-send-retry-after\":[\"true\"]},\"apiVersion\":\"openshiftcontrolplane.config.openshift.io/v1\",\"imagePolicyConfig\":{\"internalRegistryHostname\":\"image-registry.openshift-image-registry.svc:5000\"},\"kind\":\"OpenShiftAPIServerConfig\",\"projectConfig\":{\"projectRequestMessage\":\"\"},\"routingConfig\":{\"subdomain\":\"apps.ci-ln-xllhdgb-76ef8.origin-ci-int-aws.dev.rhcloud.com\"},\"servingInfo\":{\"bindNetwork\":\"tcp\",\"cipherSuites\":[\"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\",\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\",\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\",\"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\"],\"minTLSVersion\":\"VersionTLS12\"},\"storageConfig\":{\"urls\":[\"https://10.0.137.27:2379\",\"https://10.0.158.132:2379\",\"https://10.0.204.8:2379\"]}}"
},
"kind": "ConfigMap",
"metadata": {
"creationTimestamp": "2023-02-28T05:44:46Z",
"name": "config",
"namespace": "openshift-apiserver",
"resourceVersion": "20949",
"uid": "50c1c8ce-5ae3-465d-b60c-47ef2208f665"
}
}
53 changes: 53 additions & 0 deletions tests/data/configmap_yaml.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"kind":"ConfigMap",
"apiVersion":"v1",
"metadata":{
"name":"openshift-apiserver",
"namespace":"clusters-wenshen-hypershift",
"uid":"5ec57109-8d0d-46a0-8c6e-b711afa03dec",
"resourceVersion":"158040",
"creationTimestamp":"2023-02-27T21:49:46Z",
"ownerReferences":[
{
"apiVersion":"hypershift.openshift.io/v1beta1",
"kind":"HostedControlPlane",
"name":"wenshen-hypershift",
"uid":"50a4550a-e450-4546-a7b4-254011fc5dfe",
"controller":true,
"blockOwnerDeletion":true
}
],
"managedFields":[
{
"manager":"hypershift-controlplane-manager",
"operation":"Update",
"apiVersion":"v1",
"time":"2023-02-27T21:49:46Z",
"fieldsType":"FieldsV1",
"fieldsV1":{
"f:data":{
".":{

},
"f:config.yaml":{

}
},
"f:metadata":{
"f:ownerReferences":{
".":{

},
"k:{\"uid\":\"50a4550a-e450-4546-a7b4-254011fc5dfe\"}":{

}
}
}
}
}
]
},
"data":{
"config.yaml":"admission: {}\naggregatorConfig:\n allowedNames: null\n clientCA: \"\"\n extraHeaderPrefixes: null\n groupHeaders: null\n usernameHeaders: null\napiServerArguments:\n audit-log-format:\n - json\n audit-log-maxsize:\n - \"100\"\n audit-log-path:\n - /var/log/openshift-apiserver/audit.log\n audit-policy-file:\n - /etc/kubernetes/audit-config/policy.yaml\n shutdown-delay-duration:\n - 3s\napiVersion: openshiftcontrolplane.config.openshift.io/v1\nauditConfig:\n auditFilePath: \"\"\n enabled: false\n logFormat: \"\"\n maximumFileRetentionDays: 0\n maximumFileSizeMegabytes: 0\n maximumRetainedFiles: 0\n policyConfiguration: null\n policyFile: \"\"\n webHookKubeConfig: \"\"\n webHookMode: \"\"\ncloudProviderFile: \"\"\ncorsAllowedOrigins: null\nimagePolicyConfig:\n additionalTrustedCA: \"\"\n allowedRegistriesForImport: null\n externalRegistryHostnames: null\n internalRegistryHostname: image-registry.openshift-image-registry.svc:5000\n maxImagesBulkImportedPerRepository: 0\njenkinsPipelineConfig:\n autoProvisionEnabled: null\n parameters: null\n serviceName: \"\"\n templateName: \"\"\n templateNamespace: \"\"\nkind: OpenShiftAPIServerConfig\nkubeClientConfig:\n connectionOverrides:\n acceptContentTypes: \"\"\n burst: 0\n contentType: \"\"\n qps: 0\n kubeConfig: /etc/kubernetes/secrets/svc-kubeconfig/kubeconfig\nprojectConfig:\n defaultNodeSelector: \"\"\n projectRequestMessage: \"\"\n projectRequestTemplate: \"\"\nroutingConfig:\n subdomain: apps.wenshen-hypershift.devcluster.openshift.com\nserviceAccountOAuthGrantMethod: \"\"\nservingInfo:\n bindAddress: \"\"\n bindNetwork: \"\"\n certFile: /etc/kubernetes/certs/serving/tls.crt\n cipherSuites:\n - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n clientCA: /etc/kubernetes/certs/client-ca/ca.crt\n keyFile: /etc/kubernetes/certs/serving/tls.key\n maxRequestsInFlight: 0\n minTLSVersion: VersionTLS12\n requestTimeoutSeconds: 0\nstorageConfig:\n ca: /etc/kubernetes/certs/etcd-client-ca/ca.crt\n certFile: /etc/kubernetes/certs/etcd-client/etcd-client.crt\n keyFile: /etc/kubernetes/certs/etcd-client/etcd-client.key\n storagePrefix: \"\"\n urls:\n - https://etcd-client:2379\n"
}
}

0 comments on commit 35b839c

Please sign in to comment.