Extend YAML load functionality to *LIST and multi-resources

examples/create_deployment_from_yaml.py

@@ -23,7 +23,8 @@ def main():
     # default location.
     config.load_kube_config()
     k8s_client = client.ApiClient()
-    k8s_api = utils.create_from_yaml(k8s_client, "nginx-deployment.yaml")
+    utils.create_from_yaml(k8s_client, "nginx-deployment.yaml")
+    k8s_api = client.ExtensionsV1beta1Api(k8s_client)
     deps = k8s_api.read_namespaced_deployment("nginx-deployment", "default")
     print("Deployment {0} created".format(deps.metadata.name))
 

kubernetes/e2e_test/test_yaml/implicit-svclist.json

@@ -0,0 +1,60 @@
+{
+   "kind":"ServiceList",
+   "apiVersion":"v1",
+   "items":[
+     {
+       "metadata":{
+         "name":"mock-3",
+         "labels":{
+           "app":"mock-3"
+         }
+       },
+       "spec":{
+         "ports": [{
+           "protocol": "TCP",
+           "port": 99,
+           "targetPort": 9949
+         }],
+         "selector":{
+           "app":"mock-3"
+         }
+       }
+     },
+     {
+        "metadata":{
+          "name":"mock-3",
+          "labels":{
+            "app":"mock-3"
+          }
+        },
+        "spec":{
+          "ports": [{
+            "protocol": "TCP",
+            "port": 99,
+            "targetPort": 9949
+          }],
+          "selector":{
+            "app":"mock-3"
+          }
+        }
+     },
+     {
+       "metadata":{
+         "name":"mock-4",
+         "labels":{
+           "app":"mock-4"
+         }
+       },
+       "spec":{
+         "ports": [{
+           "protocol": "TCP",
+           "port": 99,
+           "targetPort": 9949
+         }],
+         "selector":{
+           "app":"mock-4"
+         }
+       }
+     }
+   ]
+}

kubernetes/e2e_test/test_yaml/list.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: List
+items:
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: list-service-test
+  spec:
+    ports:
+    - protocol: TCP
+      port: 80
+    selector:
+      app: list-deployment-test
+- apiVersion: extensions/v1beta1
+  kind: Deployment
+  metadata:
+    name: list-deployment-test
+    labels:
+      app: list-deployment-test
+  spec:
+    replicas: 1
+    template:
+      metadata:
+        labels:
+          app: list-deployment-test
+      spec:
+        containers:
+          - name: nginx
+            image: nginx

kubernetes/e2e_test/test_yaml/multi-resource-with-list.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: PodList
+items:
+- apiVersion: v1
+  kind: Pod
+  metadata:
+    name: mock-pod-0
+    labels:
+      app: mock-pod-0
+  spec:
+    containers:
+    - name: mock-pod-0
+      image: busybox
+      command: ['sh', '-c', 'echo Hello Kubernetes! && sleep 3600']
+- apiVersion: v1
+  kind: Pod
+  metadata:
+    name: mock-pod-1
+    labels:
+      app: mock-pod-1
+  spec:
+    containers:
+    - name: mock-pod-1
+      image: busybox
+      command: ['sh', '-c', 'echo Hello Kubernetes! && sleep 3600']
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: mock
+  labels:
+    app: mock
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mock
+  template:
+    metadata:
+      labels:
+        app: mock
+    spec:
+      containers:
+      - name: mock
+        image: nginx:1.15.4
+        ports:
+        - containerPort: 80

kubernetes/e2e_test/test_yaml/multi-resource.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock
+  labels:
+    app: mock
+spec:
+  ports:
+  - port: 99
+    protocol: TCP
+    targetPort: 9949
+  selector:
+    app: mock
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: mock
+spec:
+  replicas: 1
+  selector:
+    app: mock
+  template:
+    metadata:
+      labels:
+        app: mock
+    spec:
+      containers:
+      - name: mock-container
+        image: k8s.gcr.io/pause:2.0
+        ports:
+        - containerPort: 9949
+          protocol: TCP

kubernetes/e2e_test/test_yaml/namespace-list.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: NamespaceList
+items:
+- apiVersion: v1
+  kind: Namespace
+  metadata:
+    name: mock-1
+    labels:
+        name: mock-1
+- apiVersion: v1
+  kind: Namespace
+  metadata:
+    name: mock-2
+    labels:
+        name: mock-2

kubernetes/e2e_test/test_yaml/triple-nginx.yaml

@@ -0,0 +1,50 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: triple-nginx
+spec:
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: triple-nginx
+spec:
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: triple-nginx
+spec:
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80

kubernetes/e2e_test/test_yaml/yaml-conflict-first.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-2
+  labels:
+    app: mock-2
+spec:
+  ports:
+  - port: 99
+    protocol: TCP
+    targetPort: 9949
+  selector:
+    app: mock-2

kubernetes/e2e_test/test_yaml/yaml-conflict-multi.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-2
+  labels:
+    app: mock-2
+spec:
+  ports:
+  - port: 99
+    protocol: TCP
+    targetPort: 9949
+  selector:
+    app: mock-2
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: mock-2
+spec:
+  replicas: 1
+  selector:
+    app: mock-2
+  template:
+    metadata:
+      labels:
+        app: mock-2
+    spec:
+      containers:
+      - name: mock-container
+        image: k8s.gcr.io/pause:2.0
+        ports:
+        - containerPort: 9949
+          protocol: TCP

kubernetes/utils/__init__.py

@@ -12,4 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .create_from_yaml import create_from_yaml
+from __future__ import absolute_import
+
+from .create_from_yaml import FailToCreateError, create_from_yaml

kubernetes/utils/create_from_yaml.py

@@ -14,60 +14,125 @@
 
 
 import re
-import sys
 from os import path
 
 import yaml
-from six import iteritems
 
 from kubernetes import client
 
 
-def create_from_yaml(k8s_client, yaml_file, verbose=False, **kwargs):
+def create_from_yaml(
+        k8s_client,
+        yaml_file,
+        verbose=False,
+        **kwargs):
     """
     Perform an action from a yaml file. Pass True for verbose to
     print confirmation information.
     Input:
     yaml_file: string. Contains the path to yaml file.
-    k8s_cline: an ApiClient object, initialized with the client args.
+    k8s_client: an ApiClient object, initialized with the client args.
+    verbose: If True, print confirmation from the create action.
+        Default is False.
 
-    Available parameters for performing the subsequent action:
+    Returns:
+    An k8s api object or list of apis objects created from YAML.
+    When a single object is generated, return type is dependent
+    on output_list.
+
+    Throws a FailToCreateError exception if creation of any object
+    fails with helpful messages from the server.
+
+    Available parameters for creating <kind>:
     :param async_req bool
-    :param bool include_uninitialized: If true, partially initialized resources are included in the response.
+    :param bool include_uninitialized: If true, partially initialized
+        resources are included in the response.
     :param str pretty: If 'true', then the output is pretty printed.
-    :param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
+    :param str dry_run: When present, indicates that modifications
+        should not be persisted. An invalid or unrecognized dryRun
+        directive will result in an error response and no further
+        processing of the request.
+        Valid values are: - All: all dry run stages will be processed
     """
 
     with open(path.abspath(yaml_file)) as f:
-        yml_object = yaml.load(f)
-        # TODO: case of yaml file containing multiple objects
-        group, _, version = yml_object["apiVersion"].partition("/")
-        if version == "":
-            version = group
-            group = "core"
-        # Take care for the case e.g. api_type is "apiextensions.k8s.io"
-        # Only replace the last instance
-        group = "".join(group.rsplit(".k8s.io", 1))
-        fcn_to_call = "{0}{1}Api".format(group.capitalize(),
-                                         version.capitalize())
-        k8s_api = getattr(client, fcn_to_call)(k8s_client)
-        # Replace CamelCased action_type into snake_case
-        kind = yml_object["kind"]
-        kind = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', kind)
-        kind = re.sub('([a-z0-9])([A-Z])', r'\1_\2', kind).lower()
-        # Decide which namespace we are going to put the object in,
-        # if any
-        if "namespace" in yml_object["metadata"]:
-            namespace = yml_object["metadata"]["namespace"]
-        else:
-            namespace = "default"
-        # Expect the user to create namespaced objects more often
-        if hasattr(k8s_api, "create_namespaced_{0}".format(kind)):
-            resp = getattr(k8s_api, "create_namespaced_{0}".format(kind))(
-                body=yml_object, namespace=namespace, **kwargs)
-        else:
-            resp = getattr(k8s_api, "create_{0}".format(kind))(
-                body=yml_object, **kwargs)
-        if verbose:
-            print("{0} created. status='{1}'".format(kind, str(resp.status)))
-        return k8s_api
+        yml_document_all = yaml.safe_load_all(f)
+        api_exceptions = []
+        # Load all documents from a single YAML file
+        for yml_document in yml_document_all:
+            # If it is a list type, will need to iterate its items
+            if "List" in yml_document["kind"]:
+                # Could be "List" or "Pod/Service/...List"
+                # This is a list type. iterate within its items
+                kind = yml_document["kind"].replace("List", "")
+                for yml_object in yml_document["items"]:
+                    # Mitigate cases when server returns a xxxList object
+                    # See kubernetes-client/python#586
+                    if kind is not "":
+                        yml_object["apiVersion"] = yml_document["apiVersion"]
+                        yml_object["kind"] = kind
+                    try:
+                        create_from_yaml_single_item(
+                            k8s_client, yml_object, verbose, **kwargs)
+                    except client.rest.ApiException as api_exception:
+                        api_exceptions.append(api_exception)
+            else:
+                # This is a single object. Call the single item method
+                try:
+                    create_from_yaml_single_item(
+                        k8s_client, yml_document, verbose, **kwargs)
+                except client.rest.ApiException as api_exception:
+                    api_exceptions.append(api_exception)
+    # In case we have exceptions waiting for us, raise them
+    if api_exceptions:
+        raise FailToCreateError(api_exceptions)
+
+
+def create_from_yaml_single_item(
+        k8s_client, yml_object, verbose=False, **kwargs):
+    group, _, version = yml_object["apiVersion"].partition("/")
+    if version == "":
+        version = group
+        group = "core"
+    # Take care for the case e.g. api_type is "apiextensions.k8s.io"
+    # Only replace the last instance
+    group = "".join(group.rsplit(".k8s.io", 1))
+    fcn_to_call = "{0}{1}Api".format(group.capitalize(),
+                                     version.capitalize())
+    k8s_api = getattr(client, fcn_to_call)(k8s_client)
+    # Replace CamelCased action_type into snake_case
+    kind = yml_object["kind"]
+    kind = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', kind)
+    kind = re.sub('([a-z0-9])([A-Z])', r'\1_\2', kind).lower()
+    # Decide which namespace we are going to put the object in,
+    # if any
+    if "namespace" in yml_object["metadata"]:
+        namespace = yml_object["metadata"]["namespace"]
+    else:
+        namespace = "default"
+    # Expect the user to create namespaced objects more often
+    if hasattr(k8s_api, "create_namespaced_{0}".format(kind)):
+        resp = getattr(k8s_api, "create_namespaced_{0}".format(kind))(
+            body=yml_object, namespace=namespace, **kwargs)
+    else:
+        resp = getattr(k8s_api, "create_{0}".format(kind))(
+            body=yml_object, **kwargs)
+    if verbose:
+        print("{0} created. status='{1}'".format(kind, str(resp.status)))
+
+
+class FailToCreateError(Exception):
+    """
+    An exception class for handling error if an error occurred when
+    handling a yaml file.
+    """
+
+    def __init__(self, api_exceptions):
+        self.api_exceptions = api_exceptions
+
+    def __str__(self):
+        msg = ""
+        for api_exception in self.api_exceptions:
+            msg += "Error from server ({0}): {1}".format(
+                api_exception.reason, api_exception.body)
+        return msg