diff --git a/docs/source/user_guide/pipelines.md b/docs/source/user_guide/pipelines.md index 897a5b12f..66d11bf53 100644 --- a/docs/source/user_guide/pipelines.md +++ b/docs/source/user_guide/pipelines.md @@ -79,6 +79,11 @@ The [tutorials](/getting_started/tutorials.md) provide comprehensive step-by-ste - A list of [Persistent Volume Claims](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) (PVC) to be mounted into the container that executes the component. Format: `/mnt/path=existing-pvc-name`. Entries that are empty (`/mnt/path=`) or malformed are ignored. Entries with a PVC name considered to be an [invalid Kubernetes resource name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names) will raise a validation error after pipeline submission or export. - The referenced PVCs must exist in the Kubernetes namespace where the pipeline nodes are executed. - Data volumes are not mounted when the pipeline is executed locally. + - **Kubernetes pod annotations** + - A list of [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#attaching-metadata-to-objects) to be attached to the pod that executes the node. + - Format: `annotation-key=annotation-value`. Entries that are empty (`annotation-key=`) are ignored. Entries with a key considered to be [invalid](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set) will raise a validation error after pipeline submission or export. + - Annotations are ignored when the pipeline is executed locally. + - Properties that apply to every generic pipeline node. In this release the following properties are supported: - **Object storage path prefix**. Elyra stores pipeline input and output artifacts in a cloud object storage bucket. By default these artifacts are located in the `/` path. The example below depicts the artifact location for several pipelines in the `pipeline-examples` bucket: ![artifacts default storage layout on object storage](../images/user_guide/pipelines/node-artifacts-on-object-storage.png) @@ -153,6 +158,11 @@ The [tutorials](/getting_started/tutorials.md) provide comprehensive step-by-ste - Data volumes are not mounted when the pipeline is executed locally. - Example: `/mnt/vol1=data-pvc` + **Kubernetes Pod Annotations** + - Optional. A list of [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#attaching-metadata-to-objects) to be attached to the pod that executes the node. + - Format: `annotation-key=annotation-value`. Entries that are empty (`annotation-key=`) are ignored. Entries with a key considered to be [invalid](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set) will raise a validation error after pipeline submission or export. + - Annotations are ignored when the pipeline is executed locally. + 5. Associate each node with a comment to document its purpose. ![Add comment to node](../images/user_guide/pipelines/add-comment-to-node.gif) diff --git a/elyra/kfp/operator.py b/elyra/kfp/operator.py index 5aee0747e..0e0718470 100644 --- a/elyra/kfp/operator.py +++ b/elyra/kfp/operator.py @@ -32,6 +32,7 @@ from kubernetes.client.models import V1VolumeMount from elyra._version import __version__ +from elyra.pipeline.pipeline import KubernetesAnnotation from elyra.pipeline.pipeline import KubernetesSecret from elyra.pipeline.pipeline import VolumeMount @@ -91,6 +92,7 @@ def __init__( workflow_engine: Optional[str] = "argo", volume_mounts: Optional[List[VolumeMount]] = None, kubernetes_secrets: Optional[List[KubernetesSecret]] = None, + kubernetes_pod_annotations: Optional[List[KubernetesAnnotation]] = None, **kwargs, ): """Create a new instance of ContainerOp. @@ -116,6 +118,7 @@ def __init__( workflow_engine: Kubeflow workflow engine, defaults to 'argo' volume_mounts: data volumes to be mounted kubernetes_secrets: secrets to be made available as environment variables + kubernetes_pod_annotations: annotations to be applied to the pod kwargs: additional key value pairs to pass e.g. name, image, sidecars & is_exit_handler. See Kubeflow pipelines ContainerOp definition for more parameters or how to use https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.ContainerOp @@ -144,6 +147,7 @@ def __init__( self.gpu_limit = gpu_limit self.volume_mounts = volume_mounts # optional data volumes to be mounted to the pod self.kubernetes_secrets = kubernetes_secrets # optional secrets to be made available as env vars + self.kubernetes_pod_annotations = kubernetes_pod_annotations # optional annotations argument_list = [] @@ -267,6 +271,11 @@ def __init__( ) ) + # add user-provided annotations to pod + if self.kubernetes_pod_annotations: + for annotation in self.kubernetes_pod_annotations: + self.add_pod_annotation(annotation.key, annotation.value) + # If crio volume size is found then assume kubeflow pipelines environment is using CRI-o as # its container runtime if self.emptydir_volume_size: diff --git a/elyra/pipeline/airflow/processor_airflow.py b/elyra/pipeline/airflow/processor_airflow.py index 0e535d85c..bc95bd4b1 100644 --- a/elyra/pipeline/airflow/processor_airflow.py +++ b/elyra/pipeline/airflow/processor_airflow.py @@ -334,6 +334,7 @@ def _cc_pipeline(self, pipeline: Pipeline, pipeline_name: str, pipeline_instance "doc": operation.doc, "volumes": operation.mounted_volumes, "secrets": operation.kubernetes_secrets, + "kubernetes_pod_annotations": operation.kubernetes_pod_annotations, } if runtime_image_pull_secret is not None: @@ -447,6 +448,7 @@ def _cc_pipeline(self, pipeline: Pipeline, pipeline_name: str, pipeline_instance "is_generic_operator": operation.is_generic, "doc": operation.doc, "volumes": operation.mounted_volumes, + "kubernetes_pod_annotations": operation.kubernetes_pod_annotations, } target_ops.append(target_op) @@ -499,6 +501,7 @@ def create_pipeline_file( "render_executor_config_for_custom_op": AirflowPipelineProcessor.render_executor_config_for_custom_op, "render_secrets_for_generic_op": AirflowPipelineProcessor.render_secrets_for_generic_op, "render_secrets_for_cos": AirflowPipelineProcessor.render_secrets_for_cos, + "render_executor_config_for_generic_op": AirflowPipelineProcessor.render_executor_config_for_generic_op, } template.globals.update(rendering_functions) @@ -636,18 +639,49 @@ def render_executor_config_for_custom_op(op: Dict) -> Dict[str, Dict[str, List]] :returns: a dict defining the volumes and mounts to be rendered in the DAG """ - executor_config = {"KubernetesExecutor": {"volumes": [], "volume_mounts": []}} - for volume in op.get("volumes", []): - # Define volumes and volume mounts - executor_config["KubernetesExecutor"]["volumes"].append( - { - "name": volume.pvc_name, - "persistentVolumeClaim": {"claimName": volume.pvc_name}, - } - ) - executor_config["KubernetesExecutor"]["volume_mounts"].append( - {"mountPath": volume.path, "name": volume.pvc_name, "read_only": False} - ) + + executor_config = {"KubernetesExecutor": {}} + + # Handle volume mounts + if op.get("volumes"): + executor_config["KubernetesExecutor"]["volumes"] = [] + executor_config["KubernetesExecutor"]["volume_mounts"] = [] + for volume in op.get("volumes", []): + # Add volume and volume mount entry + executor_config["KubernetesExecutor"]["volumes"].append( + { + "name": volume.pvc_name, + "persistentVolumeClaim": {"claimName": volume.pvc_name}, + } + ) + executor_config["KubernetesExecutor"]["volume_mounts"].append( + {"mountPath": volume.path, "name": volume.pvc_name, "read_only": False} + ) + + # Handle annotations + if op.get("kubernetes_pod_annotations"): + executor_config["KubernetesExecutor"]["annotations"] = {} + for annotation in op.get("kubernetes_pod_annotations", []): + # Add Kubernetes annotation entry + executor_config["KubernetesExecutor"]["annotations"][annotation.key] = annotation.value + + return executor_config + + @staticmethod + def render_executor_config_for_generic_op(op: Dict) -> Dict[str, Dict[str, List]]: + """ + Render annotations defined for the specified generic op + for use in the Airflow DAG template + :returns: a dict defining the annotations to be rendered in the DAG + """ + executor_config = {"KubernetesExecutor": {}} + + # Handle annotations + if op.get("kubernetes_pod_annotations"): + executor_config["KubernetesExecutor"]["annotations"] = {} + for annotation in op.get("kubernetes_pod_annotations", []): + # Add Kubernetes annotation entry + executor_config["KubernetesExecutor"]["annotations"][annotation.key] = annotation.value return executor_config diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py index f8c2ab6fa..9f9ec3320 100644 --- a/elyra/pipeline/kfp/processor_kfp.py +++ b/elyra/pipeline/kfp/processor_kfp.py @@ -546,6 +546,7 @@ def _cc_pipeline( }, volume_mounts=operation.mounted_volumes, kubernetes_secrets=operation.kubernetes_secrets, + kubernetes_pod_annotations=operation.kubernetes_pod_annotations, ) if operation.doc: @@ -674,6 +675,14 @@ def _cc_pipeline( V1VolumeMount(mount_path=volume_mount.path, name=volume_mount.pvc_name) ) + # Add user-specified pod annotations + if operation.kubernetes_pod_annotations: + unique_annotations = [] + for annotation in operation.kubernetes_pod_annotations: + if annotation.key not in unique_annotations: + container_op.add_pod_annotation(annotation.key, annotation.value) + unique_annotations.append(annotation.key) + target_ops[operation.id] = container_op except Exception as e: # TODO Fix error messaging and break exceptions down into categories diff --git a/elyra/pipeline/pipeline.py b/elyra/pipeline/pipeline.py index 198c48262..f2f087c53 100644 --- a/elyra/pipeline/pipeline.py +++ b/elyra/pipeline/pipeline.py @@ -25,6 +25,7 @@ from typing import Optional from elyra.pipeline.pipeline_constants import ENV_VARIABLES +from elyra.pipeline.pipeline_constants import KUBERNETES_POD_ANNOTATIONS from elyra.pipeline.pipeline_constants import KUBERNETES_SECRETS from elyra.pipeline.pipeline_constants import MOUNTED_VOLUMES @@ -110,6 +111,17 @@ def __init__( # spec) and must be removed from the component_params dict self._mounted_volumes = self._component_params.pop(MOUNTED_VOLUMES, []) + self._kubernetes_pod_annotations = [] + param_annotations = component_params.get(KUBERNETES_POD_ANNOTATIONS) + if ( + param_annotations is not None + and isinstance(param_annotations, list) + and (len(param_annotations) == 0 or isinstance(param_annotations[0], KubernetesAnnotation)) + ): + # The kubernetes_pod_annotations property is an Elyra system property (ie, not defined in the component + # spec) and must be removed from the component_params dict + self._kubernetes_pod_annotations = self._component_params.pop(KUBERNETES_POD_ANNOTATIONS, []) + # Scrub the inputs and outputs lists self._component_params["inputs"] = Operation._scrub_list(component_params.get("inputs", [])) self._component_params["outputs"] = Operation._scrub_list(component_params.get("outputs", [])) @@ -158,6 +170,10 @@ def component_params_as_dict(self) -> Dict[str, Any]: def mounted_volumes(self) -> List["VolumeMount"]: return self._mounted_volumes + @property + def kubernetes_pod_annotations(self) -> List["KubernetesAnnotation"]: + return self._kubernetes_pod_annotations + @property def inputs(self) -> Optional[List[str]]: return self._component_params.get("inputs") @@ -549,6 +565,12 @@ class KubernetesSecret: key: str +@dataclass +class KubernetesAnnotation: + key: str + value: str + + class DataClassJSONEncoder(json.JSONEncoder): """ A JSON Encoder class to prevent errors during serialization of dataclasses. diff --git a/elyra/pipeline/pipeline_constants.py b/elyra/pipeline/pipeline_constants.py index cc729ca6d..a60e94bd3 100644 --- a/elyra/pipeline/pipeline_constants.py +++ b/elyra/pipeline/pipeline_constants.py @@ -19,7 +19,8 @@ ENV_VARIABLES = "env_vars" MOUNTED_VOLUMES = "mounted_volumes" KUBERNETES_SECRETS = "kubernetes_secrets" +KUBERNETES_POD_ANNOTATIONS = "kubernetes_pod_annotations" PIPELINE_META_PROPERTIES = ["name", "description", "runtime"] # optional static prefix to be used when generating an object name for object storage COS_OBJECT_PREFIX = "cos_object_prefix" -ELYRA_COMPONENT_PROPERTIES = [MOUNTED_VOLUMES] +ELYRA_COMPONENT_PROPERTIES = [MOUNTED_VOLUMES, KUBERNETES_POD_ANNOTATIONS] diff --git a/elyra/pipeline/pipeline_definition.py b/elyra/pipeline/pipeline_definition.py index e973ad0f6..d003bdd84 100644 --- a/elyra/pipeline/pipeline_definition.py +++ b/elyra/pipeline/pipeline_definition.py @@ -27,11 +27,13 @@ from elyra.pipeline.component_catalog import ComponentCache from elyra.pipeline.pipeline import KeyValueList +from elyra.pipeline.pipeline import KubernetesAnnotation from elyra.pipeline.pipeline import KubernetesSecret from elyra.pipeline.pipeline import Operation from elyra.pipeline.pipeline import VolumeMount from elyra.pipeline.pipeline_constants import ELYRA_COMPONENT_PROPERTIES from elyra.pipeline.pipeline_constants import ENV_VARIABLES +from elyra.pipeline.pipeline_constants import KUBERNETES_POD_ANNOTATIONS from elyra.pipeline.pipeline_constants import KUBERNETES_SECRETS from elyra.pipeline.pipeline_constants import MOUNTED_VOLUMES from elyra.pipeline.pipeline_constants import PIPELINE_DEFAULTS @@ -434,6 +436,16 @@ def convert_data_class_properties(self): self.set_component_parameter(KUBERNETES_SECRETS, secret_objects) + kubernetes_pod_annotations = self.get_component_parameter(KUBERNETES_POD_ANNOTATIONS) + if kubernetes_pod_annotations and isinstance(kubernetes_pod_annotations, KeyValueList): + annotations_objects = [] + for annotation_key, annotation_value in kubernetes_pod_annotations.to_dict().items(): + # Validation should have verified that the provided values are valid + # Create a KubernetesAnnotation class instance and add to list + annotations_objects.append(KubernetesAnnotation(annotation_key, annotation_value)) + + self.set_component_parameter(KUBERNETES_POD_ANNOTATIONS, annotations_objects) + class PipelineDefinition(object): """ diff --git a/elyra/pipeline/validation.py b/elyra/pipeline/validation.py index 0d80060f4..5ca82b034 100644 --- a/elyra/pipeline/validation.py +++ b/elyra/pipeline/validation.py @@ -32,12 +32,14 @@ from elyra.pipeline.component_catalog import ComponentCache from elyra.pipeline.pipeline import DataClassJSONEncoder from elyra.pipeline.pipeline import KeyValueList +from elyra.pipeline.pipeline import KubernetesAnnotation from elyra.pipeline.pipeline import KubernetesSecret from elyra.pipeline.pipeline import Operation from elyra.pipeline.pipeline import PIPELINE_CURRENT_SCHEMA from elyra.pipeline.pipeline import PIPELINE_CURRENT_VERSION from elyra.pipeline.pipeline import VolumeMount from elyra.pipeline.pipeline_constants import ENV_VARIABLES +from elyra.pipeline.pipeline_constants import KUBERNETES_POD_ANNOTATIONS from elyra.pipeline.pipeline_constants import KUBERNETES_SECRETS from elyra.pipeline.pipeline_constants import MOUNTED_VOLUMES from elyra.pipeline.pipeline_constants import RUNTIME_IMAGE @@ -45,6 +47,7 @@ from elyra.pipeline.pipeline_definition import PipelineDefinition from elyra.pipeline.processor import PipelineProcessorManager from elyra.pipeline.runtime_type import RuntimeProcessorType +from elyra.util.kubernetes import is_valid_annotation_key from elyra.util.kubernetes import is_valid_kubernetes_key from elyra.util.kubernetes import is_valid_kubernetes_resource_name from elyra.util.path import get_expanded_path @@ -419,6 +422,7 @@ def _validate_generic_node_properties(self, node: Node, response: ValidationResp env_vars = node.get_component_parameter(ENV_VARIABLES) volumes = node.get_component_parameter(MOUNTED_VOLUMES) secrets = node.get_component_parameter(KUBERNETES_SECRETS) + annotations = node.get_component_parameter(KUBERNETES_POD_ANNOTATIONS) self._validate_filepath( node_id=node.id, node_label=node_label, property_name="filename", filename=filename, response=response @@ -442,6 +446,8 @@ def _validate_generic_node_properties(self, node: Node, response: ValidationResp self._validate_mounted_volumes(node.id, node_label, volumes, response=response) if secrets: self._validate_kubernetes_secrets(node.id, node_label, secrets, response=response) + if annotations: + self._validate_kubernetes_pod_annotations(node.id, node_label, annotations, response=response) self._validate_label(node_id=node.id, node_label=node_label, response=response) if dependencies: @@ -490,6 +496,10 @@ async def _validate_custom_component_node_properties( if volumes and MOUNTED_VOLUMES not in node.elyra_properties_to_skip: self._validate_mounted_volumes(node.id, node.label, volumes, response=response) + annotations = node.get_component_parameter(KUBERNETES_POD_ANNOTATIONS) + if annotations and KUBERNETES_POD_ANNOTATIONS not in node.elyra_properties_to_skip: + self._validate_kubernetes_pod_annotations(node.id, node.label, annotations, response=response) + for default_parameter in current_parameter_defaults_list: node_param = node.get_component_parameter(default_parameter) if self._is_required_property(component_property_dict, default_parameter): @@ -707,6 +717,32 @@ def _validate_kubernetes_secrets( }, ) + def _validate_kubernetes_pod_annotations( + self, node_id: str, node_label: str, annotations: List[KubernetesAnnotation], response: ValidationResponse + ) -> None: + """ + Checks the format of the user-provided annotations to ensure they're in the correct form + e.g. annotation_key=annotation_value + :param node_id: the unique ID of the node + :param node_label: the given node name or user customized name/label of the node + :param annotations: a KeyValueList of annotations to check + :param response: ValidationResponse containing the issue list to be updated + """ + for annotation in annotations: + # Ensure the annotation key is valid + if not is_valid_annotation_key(annotation.key): + response.add_message( + severity=ValidationSeverity.Error, + message_type="invalidKubernetesAnnotation", + message=f"'{annotation.key}' is not a valid Kubernetes annotation key.", + data={ + "nodeID": node_id, + "nodeName": node_label, + "propertyName": KUBERNETES_POD_ANNOTATIONS, + "value": KeyValueList.to_str(annotation.key, annotation.value), + }, + ) + def _validate_filepath( self, node_id: str, diff --git a/elyra/templates/airflow/airflow_template.jinja2 b/elyra/templates/airflow/airflow_template.jinja2 index 8c38bccd8..414cbf2b4 100644 --- a/elyra/templates/airflow/airflow_template.jinja2 +++ b/elyra/templates/airflow/airflow_template.jinja2 @@ -31,16 +31,18 @@ dag = DAG( from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator {% endif %} -{% if operation.operator_source %}# Operator source: {{ operation.operator_source }}{% endif %}{% if not operation.is_generic_operator %} +{% if operation.operator_source %}# Operator source: {{ operation.operator_source }} +{% endif %} +{% if not operation.is_generic_operator %} op_{{ operation.id|regex_replace }} = {{ operation.class_name }}( task_id='{{ operation.notebook|regex_replace }}', {% for param, value in operation.component_params.items() %} {{ param }}={{ value }}, {% endfor %} - {% if operation.volumes %} + {% if operation.volumes or operation.kubernetes_pod_annotations %} executor_config={{ render_executor_config_for_custom_op(operation) }}, {% endif %} - {% else %} +{% else %} op_{{ operation.id|regex_replace }} = KubernetesPodOperator(name='{{ operation.notebook|regex_replace }}', namespace='{{ user_namespace }}', image='{{ operation.runtime_image }}', @@ -71,6 +73,9 @@ op_{{ operation.id|regex_replace }} = KubernetesPodOperator(name='{{ operation.n volumes=[{% for volume_var in operation.volume_vars %}{{ volume_var }},{% endfor %}], volume_mounts=[{% for mount_var in operation.volume_mount_vars %}{{ mount_var }},{% endfor %}], {% endif %} + {% if operation.kubernetes_pod_annotations %} + executor_config={{ render_executor_config_for_generic_op(operation) }}, + {% endif %} in_cluster={{ in_cluster }}, config_file="{{ kube_config_path }}", {% endif %} diff --git a/elyra/templates/components/canvas_properties_template.jinja2 b/elyra/templates/components/canvas_properties_template.jinja2 index ca5b0eb47..db52bdc4b 100644 --- a/elyra/templates/components/canvas_properties_template.jinja2 +++ b/elyra/templates/components/canvas_properties_template.jinja2 @@ -27,6 +27,9 @@ {% endfor %} {% if "mounted_volumes" not in elyra_property_collisions_list %} "elyra_mounted_volumes": [], +{% endif %} +{% if "kubernetes_pod_annotations" not in elyra_property_collisions_list %} + "elyra_kubernetes_pod_annotations": [], {% endif %} "component_source": {{ component.component_source|tojson|safe }} }, @@ -46,6 +49,11 @@ { "id": "elyra_mounted_volumes" }, +{% endif %} +{% if "kubernetes_pod_annotations" not in elyra_property_collisions_list %} + { + "id": "elyra_kubernetes_pod_annotations" + }, {% endif %} { "id": "component_source" @@ -152,6 +160,25 @@ "keyValueEntries": true } }, +{% endif %} +{% if "kubernetes_pod_annotations" not in elyra_property_collisions_list %} + { + "parameter_ref": "elyra_kubernetes_pod_annotations", + "control": "custom", + "custom_control_id": "StringArrayControl", + "label": { + "default": "Kubernetes Pod Annotations" + }, + "description": { + "default": "Metadata to be added to this node. The metadata is exposed as annotation in the Kubernetes pod that executes this node.", + "placement": "on_panel" + }, + "data": { + "required": false, + "placeholder": "annotation_key=annotation_value", + "keyValueEntries": true + } + }, {% endif %} { "parameter_ref": "component_source", @@ -215,7 +242,8 @@ }, {% endfor %} {% endif %} -{% if "mounted_volumes" not in elyra_property_collisions_list %} +{% if (("mounted_volumes" not in elyra_property_collisions_list) or + ("kubernetes_pod_annotations" not in elyra_property_collisions_list)) %} { "id": "elyra_other_propertiesCategoryHeader", "type": "textPanel", @@ -228,11 +256,20 @@ "placement": "on_panel" } }, +{% endif %} +{% if "mounted_volumes" not in elyra_property_collisions_list %} { "id": "elyra_mounted_volumes", "type": "controls", "parameter_refs": ["elyra_mounted_volumes"] }, +{% endif %} +{% if "kubernetes_pod_annotations" not in elyra_property_collisions_list %} + { + "id": "elyra_kubernetes_pod_annotations", + "type": "controls", + "parameter_refs": ["elyra_kubernetes_pod_annotations"] + }, {% endif %} { "id": "elyra_component_sourceCategoryHeader", diff --git a/elyra/templates/components/generic_properties_template.jinja2 b/elyra/templates/components/generic_properties_template.jinja2 index dcd16dc45..e0ce5defe 100644 --- a/elyra/templates/components/generic_properties_template.jinja2 +++ b/elyra/templates/components/generic_properties_template.jinja2 @@ -9,6 +9,7 @@ "elyra_outputs": [], "elyra_env_vars": [], "elyra_kubernetes_secrets": [], + "elyra_kubernetes_pod_annotations": [], "elyra_dependencies": [], "elyra_include_subdirectories": false, "elyra_mounted_volumes": [] @@ -44,6 +45,9 @@ { "id": "elyra_kubernetes_secrets" }, + { + "id": "elyra_kubernetes_pod_annotations" + }, { "id": "elyra_outputs" }, @@ -237,6 +241,22 @@ "placeholder": "/mount/path=pvc-name", "keyValueEntries": true } + }, + { + "parameter_ref": "elyra_kubernetes_pod_annotations", + "control": "custom", + "custom_control_id": "StringArrayControl", + "label": { + "default": "Kubernetes Pod Annotations" + }, + "description": { + "default": "Metadata to be added to this node. The metadata is exposed as annotation in the Kubernetes pod that executes this node.", + "placement": "on_panel" + }, + "data": { + "placeholder": "annotation_key=annotation_value", + "keyValueEntries": true + } } ], "group_info": [ @@ -293,6 +313,11 @@ "id": "elyra_mounted_volumes", "type": "controls", "parameter_refs": ["elyra_mounted_volumes"] + }, + { + "id": "elyra_kubernetes_pod_annotations", + "type": "controls", + "parameter_refs": ["elyra_kubernetes_pod_annotations"] } ] } diff --git a/elyra/templates/pipeline/pipeline_properties_template.jinja2 b/elyra/templates/pipeline/pipeline_properties_template.jinja2 index 8a57b112e..0c269a8f7 100644 --- a/elyra/templates/pipeline/pipeline_properties_template.jinja2 +++ b/elyra/templates/pipeline/pipeline_properties_template.jinja2 @@ -7,7 +7,8 @@ "elyra_runtime_image": null, "elyra_env_vars": [], "elyra_kubernetes_secrets": [], - "elyra_mounted_volumes": [] + "elyra_mounted_volumes": [], + "elyra_kubernetes_pod_annotations": [] }, "parameters": [ { @@ -33,7 +34,10 @@ }, { "id": "elyra_mounted_volumes" - } + }, + { + "id": "elyra_kubernetes_pod_annotations" + } ], "uihints": { "id": "nodeProperties", @@ -134,6 +138,22 @@ "placeholder": "/mount/path=pvc-name", "keyValueEntries": true } + }, + { + "parameter_ref": "elyra_kubernetes_pod_annotations", + "control": "custom", + "custom_control_id": "StringArrayControl", + "label": { + "default": "Kubernetes Pod Annotations" + }, + "description": { + "default": "Metadata to be added to all nodes. The metadata is exposed as annotations in the Kubernetes pods that execute the nodes.", + "placement": "on_panel" + }, + "data": { + "placeholder": "annotation_key=annotation_value", + "keyValueEntries": true + } } ], "group_info": [ @@ -178,6 +198,11 @@ "type": "controls", "parameter_refs": ["elyra_mounted_volumes"] }, + { + "id": "elyra_kubernetes_pod_annotations", + "type": "controls", + "parameter_refs": ["elyra_kubernetes_pod_annotations"] + }, { "id": "elyra_generic_nodesCategoryHeader", "type": "textPanel", diff --git a/elyra/tests/pipeline/airflow/test_component_parser_airflow.py b/elyra/tests/pipeline/airflow/test_component_parser_airflow.py index 43e34fdcd..093e0de97 100644 --- a/elyra/tests/pipeline/airflow/test_component_parser_airflow.py +++ b/elyra/tests/pipeline/airflow/test_component_parser_airflow.py @@ -441,9 +441,9 @@ def test_parse_airflow_component_file_no_inputs(): no_input_op = parser.parse(catalog_entry)[0] properties_json = ComponentCache.to_canvas_properties(no_input_op) - # Properties JSON should only include the two parameters common to every - # component: ('label', 'component_source' and 'mounted_volumes') - num_common_params = 3 + # Properties JSON should only include the four parameters common to every + # component: ('label', 'component_source', 'mounted_volumes', and 'kubernetes_pod_annotations') + num_common_params = 4 assert len(properties_json["current_parameters"].keys()) == num_common_params assert len(properties_json["parameters"]) == num_common_params assert len(properties_json["uihints"]["parameter_info"]) == num_common_params diff --git a/elyra/tests/pipeline/kfp/test_component_parser_kfp.py b/elyra/tests/pipeline/kfp/test_component_parser_kfp.py index 1d49fcdff..e5694c9cd 100644 --- a/elyra/tests/pipeline/kfp/test_component_parser_kfp.py +++ b/elyra/tests/pipeline/kfp/test_component_parser_kfp.py @@ -430,12 +430,12 @@ def test_parse_kfp_component_file_no_inputs(): component = parser.parse(catalog_entry)[0] properties_json = ComponentCache.to_canvas_properties(component) - # Properties JSON should only include the three parameters common to every - # component ('label', 'component_source' and 'mounted_volumes'), the component + # Properties JSON should only include the four parameters common to every + # component ('label', 'component_source', 'mounted_volumes', and 'kubernetes_pod_annotations'), the component # description if it exists (which it does for this component), and the output # parameter for this component - num_common_params = 5 - assert len(properties_json["current_parameters"].keys()) == num_common_params + num_common_params = 6 + assert len(properties_json["current_parameters"].keys()) == num_common_params, properties_json["current_parameters"] assert len(properties_json["parameters"]) == num_common_params assert len(properties_json["uihints"]["parameter_info"]) == num_common_params diff --git a/elyra/tests/pipeline/resources/properties.json b/elyra/tests/pipeline/resources/properties.json index 3b7498187..98ae0f136 100644 --- a/elyra/tests/pipeline/resources/properties.json +++ b/elyra/tests/pipeline/resources/properties.json @@ -10,6 +10,7 @@ "elyra_outputs": [], "elyra_env_vars": [], "elyra_kubernetes_secrets": [], + "elyra_kubernetes_pod_annotations": [], "elyra_dependencies": [], "elyra_include_subdirectories": false }, @@ -44,6 +45,9 @@ { "id": "elyra_kubernetes_secrets" }, + { + "id": "elyra_kubernetes_pod_annotations" + }, { "id": "elyra_outputs" }, @@ -237,6 +241,22 @@ "placeholder": "/mount/path=pvc-name", "keyValueEntries": true } + }, + { + "parameter_ref": "elyra_kubernetes_pod_annotations", + "control": "custom", + "custom_control_id": "StringArrayControl", + "label": { + "default": "Kubernetes Pod Annotations" + }, + "description": { + "default": "Metadata to be added to this node. The metadata is exposed as annotation in the Kubernetes pod that executes this node.", + "placement": "on_panel" + }, + "data": { + "placeholder": "annotation_key=annotation_value", + "keyValueEntries": true + } } ], "group_info": [ @@ -293,6 +313,11 @@ "id": "elyra_mounted_volumes", "type": "controls", "parameter_refs": ["elyra_mounted_volumes"] + }, + { + "id": "elyra_kubernetes_pod_annotations", + "type": "controls", + "parameter_refs": ["elyra_kubernetes_pod_annotations"] } ] } diff --git a/elyra/tests/pipeline/test_handlers.py b/elyra/tests/pipeline/test_handlers.py index 7a5b1e4e0..94b0133d7 100644 --- a/elyra/tests/pipeline/test_handlers.py +++ b/elyra/tests/pipeline/test_handlers.py @@ -220,4 +220,5 @@ async def test_get_pipeline_properties_definition(jp_fetch): {"id": "elyra_env_vars"}, {"id": "elyra_kubernetes_secrets"}, {"id": "elyra_mounted_volumes"}, + {"id": "elyra_kubernetes_pod_annotations"}, ] diff --git a/elyra/tests/pipeline/test_validation.py b/elyra/tests/pipeline/test_validation.py index 58ffbd3cd..9b531e80b 100644 --- a/elyra/tests/pipeline/test_validation.py +++ b/elyra/tests/pipeline/test_validation.py @@ -20,9 +20,11 @@ from conftest import KFP_COMPONENT_CACHE_INSTANCE import pytest +from elyra.pipeline.pipeline import KubernetesAnnotation from elyra.pipeline.pipeline import KubernetesSecret from elyra.pipeline.pipeline import PIPELINE_CURRENT_VERSION from elyra.pipeline.pipeline import VolumeMount +from elyra.pipeline.pipeline_constants import KUBERNETES_POD_ANNOTATIONS from elyra.pipeline.pipeline_constants import KUBERNETES_SECRETS from elyra.pipeline.pipeline_constants import MOUNTED_VOLUMES from elyra.pipeline.pipeline_definition import PipelineDefinition @@ -435,6 +437,113 @@ def test_invalid_node_property_volumes(validation_manager): assert "not a valid Kubernetes resource name" in issues[0]["message"] +def test_valid_node_property_kubernetes_pod_annotation(validation_manager): + """ + Validate that valid kubernetes pod annotation definitions are not flagged as invalid. + Constraints are documented in + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + """ + response = ValidationResponse() + node = {"id": "test-id", "app_data": {"label": "test"}} + # The following annotations are valid + annotations = [ + # parameters are key and value + KubernetesAnnotation("k", ""), + KubernetesAnnotation("key", "value"), + KubernetesAnnotation("n-a-m-e", "value"), + KubernetesAnnotation("n.a.m.e", "value"), + KubernetesAnnotation("n_a_m_e", "value"), + KubernetesAnnotation("n-a.m_e", "value"), + KubernetesAnnotation("prefix/name", "value"), + KubernetesAnnotation("abc.def/name", "value"), + KubernetesAnnotation("abc.def.ghi/n-a-m-e", "value"), + KubernetesAnnotation("abc.def.ghi.jkl/n.a.m.e", "value"), + KubernetesAnnotation("abc.def.ghi.jkl.mno/n_a_m_e", "value"), + KubernetesAnnotation("abc.def.ghijklmno.pqr/n-a.m_e", "value"), + ] + validation_manager._validate_kubernetes_pod_annotations( + node_id=node["id"], node_label=node["app_data"]["label"], annotations=annotations, response=response + ) + issues = response.to_json().get("issues") + assert len(issues) == 0, response.to_json() + + +def test_invalid_node_property_kubernetes_pod_annotation(validation_manager): + """ + Validate that valid kubernetes pod annotation definitions are not flagged as invalid. + Constraints are documented in + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + """ + response = ValidationResponse() + node = {"id": "test-id", "app_data": {"label": "test"}} + TOO_SHORT_LENGTH = 0 + MAX_PREFIX_LENGTH = 253 + MAX_NAME_LENGTH = 63 + TOO_LONG_LENGTH = MAX_PREFIX_LENGTH + 1 + MAX_NAME_LENGTH + 1 # prefix + '/' + name + + # The following annotations are invalid + invalid_annotations = [ + # parameters are key and value + # test length violations (key name and prefix) + KubernetesAnnotation("a" * (TOO_SHORT_LENGTH), ""), # empty key (min 1) + KubernetesAnnotation("a" * (TOO_LONG_LENGTH), ""), # key too long + KubernetesAnnotation(f"{'a' * (MAX_PREFIX_LENGTH + 1)}/b", ""), # key prefix too long + KubernetesAnnotation(f"{'a' * (MAX_NAME_LENGTH + 1)}", ""), # key name too long + KubernetesAnnotation(f"prefix/{'a' * (MAX_NAME_LENGTH + 1)}", ""), # key name too long + KubernetesAnnotation(f"{'a' * (MAX_PREFIX_LENGTH + 1)}/name", ""), # key prefix too long + # test character violations (key name) + KubernetesAnnotation("-", ""), # name must start and end with alphanum + KubernetesAnnotation("-a", ""), # name must start with alphanum + KubernetesAnnotation("a-", ""), # name must start with alphanum + KubernetesAnnotation("prefix/-b", ""), # name start with alphanum + KubernetesAnnotation("prefix/b-", ""), # name must end with alphanum + # test character violations (key prefix) + KubernetesAnnotation("PREFIX/name", ""), # prefix must be lowercase + KubernetesAnnotation("pref!x/name", ""), # prefix must contain alnum, '-' or '.' + KubernetesAnnotation("pre.fx./name", ""), # prefix must contain alnum, '-' or '.' + KubernetesAnnotation("-pre.fx.com/name", ""), # prefix must contain alnum, '-' or '.' + KubernetesAnnotation("pre.fx-./name", ""), # prefix must contain alnum, '-' or '.' + KubernetesAnnotation("a/b/c", ""), # only one separator char + ] + expected_error_messages = [ + "'' is not a valid Kubernetes annotation key.", + f"'{'a' * (TOO_LONG_LENGTH)}' is not a valid Kubernetes annotation key.", + f"'{'a' * (MAX_PREFIX_LENGTH + 1)}/b' is not a valid Kubernetes annotation key.", + f"'{'a' * (MAX_NAME_LENGTH + 1)}' is not a valid Kubernetes annotation key.", + f"'prefix/{'a' * (MAX_NAME_LENGTH + 1)}' is not a valid Kubernetes annotation key.", + f"'{'a' * (MAX_PREFIX_LENGTH + 1)}/name' is not a valid Kubernetes annotation key.", + "'-' is not a valid Kubernetes annotation key.", + "'-a' is not a valid Kubernetes annotation key.", + "'a-' is not a valid Kubernetes annotation key.", + "'prefix/-b' is not a valid Kubernetes annotation key.", + "'prefix/b-' is not a valid Kubernetes annotation key.", + "'PREFIX/name' is not a valid Kubernetes annotation key.", + "'pref!x/name' is not a valid Kubernetes annotation key.", + "'pre.fx./name' is not a valid Kubernetes annotation key.", + "'-pre.fx.com/name' is not a valid Kubernetes annotation key.", + "'pre.fx-./name' is not a valid Kubernetes annotation key.", + "'a/b/c' is not a valid Kubernetes annotation key.", + ] + + # verify that the number of annotations in this test matches the number of error messages + assert len(invalid_annotations) == len(expected_error_messages), "Test implementation error. " + + validation_manager._validate_kubernetes_pod_annotations( + node_id=node["id"], node_label=node["app_data"]["label"], annotations=invalid_annotations, response=response + ) + issues = response.to_json().get("issues") + assert len(issues) == len( + invalid_annotations + ), f"validation returned unexpected results: {response.to_json()['issues']}" + index = 0 + for issue in issues: + assert issue["type"] == "invalidKubernetesAnnotation" + assert issue["data"]["propertyName"] == KUBERNETES_POD_ANNOTATIONS + assert issue["data"]["nodeID"] == "test-id" + assert issue["message"] == expected_error_messages[index], f"Index is {index}" + index = index + 1 + + def test_invalid_node_property_secrets(validation_manager): response = ValidationResponse() node = {"id": "test-id", "app_data": {"label": "test"}} diff --git a/elyra/util/kubernetes.py b/elyra/util/kubernetes.py index 887dbee90..c53d0f2cf 100644 --- a/elyra/util/kubernetes.py +++ b/elyra/util/kubernetes.py @@ -40,6 +40,19 @@ def is_valid_kubernetes_resource_name(name: str) -> bool: return True +def is_valid_dns_subdomain_name(name: str) -> bool: + """ + Returns a truthy value indicating whether name meets the kubernetes + naming constraints for DNS subdomains, as outlined in the link below. + + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names + """ + if name is None or len(name) > 253: + return False + + return re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$", name) is not None + + def is_valid_kubernetes_key(name: str) -> bool: """ Returns a truthy value indicating whether name meets the kubernetes @@ -51,3 +64,34 @@ def is_valid_kubernetes_key(name: str) -> bool: return False return re.match(r"^[\w\-_.]+$", name) is not None + + +def is_valid_annotation_key(key: str) -> bool: + """ + Returns a truthy value indicating whether name meets the kubernetes + naming constraints for annotation keys, as outlined in the link below. + + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + """ + if key is None or (len(key) == 0): + return False + + parts = key.split("/") + if len(parts) == 1: + prefix = "" + name = parts[0] + elif len(parts) == 2: + prefix = parts[0] + name = parts[1] + else: + return False + + # validate optional prefix + if len(prefix) > 0 and not is_valid_dns_subdomain_name(prefix): + return False + + # validate name + if len(name) > 63 or not name[0].isalnum() or not name[-1].isalnum(): + return False + + return re.match(r"^[\w\-_.]+$", name) is not None diff --git a/tests/snapshots/pipeline-editor-tests/matches-complex-pipeline-snapshot.1.snap b/tests/snapshots/pipeline-editor-tests/matches-complex-pipeline-snapshot.1.snap index 0925d880d..9d0725210 100644 --- a/tests/snapshots/pipeline-editor-tests/matches-complex-pipeline-snapshot.1.snap +++ b/tests/snapshots/pipeline-editor-tests/matches-complex-pipeline-snapshot.1.snap @@ -25,6 +25,7 @@ Object { ], "filename": "producer.ipynb", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [ @@ -81,6 +82,7 @@ Object { "env_vars": Array [], "filename": "consumer.ipynb", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [], @@ -136,6 +138,7 @@ Object { "env_vars": Array [], "filename": "../scripts/setup.py", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [], @@ -189,6 +192,7 @@ Object { "env_vars": Array [], "filename": "create-source-files.py", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [ @@ -245,6 +249,7 @@ Object { "env_vars": Array [], "filename": "producer-script.py", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [ diff --git a/tests/snapshots/pipeline-editor-tests/matches-simple-pipeline-snapshot.1.snap b/tests/snapshots/pipeline-editor-tests/matches-simple-pipeline-snapshot.1.snap index 6ac51a949..ab51de376 100644 --- a/tests/snapshots/pipeline-editor-tests/matches-simple-pipeline-snapshot.1.snap +++ b/tests/snapshots/pipeline-editor-tests/matches-simple-pipeline-snapshot.1.snap @@ -26,6 +26,7 @@ Object { ], "filename": "helloworld.ipynb", "include_subdirectories": false, + "kubernetes_pod_annotations": Array [], "kubernetes_secrets": Array [], "mounted_volumes": Array [], "outputs": Array [],