Skip to content

Commit

Permalink
Improve JSON output of describe command (#2878)
Browse files Browse the repository at this point in the history 
Render custom component dependencies as list of dict instead of list of strings.
  • Loading branch information
@ptitzler
ptitzler authored Aug 17, 2022
1 parent 61ceb4f commit e357dff
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 62 deletions.
10 changes: 7 additions & 3 deletions elyra/cli/pipeline_app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -644,9 +644,7 @@ def describe(json_option, pipeline_path):
describe_dict["notebook_dependencies"]["value"]
),
describe_dict["file_dependencies"]["json_name"]: list(describe_dict["file_dependencies"]["value"]),
describe_dict["component_dependencies"]["json_name"]: list(
describe_dict["component_dependencies"]["value"]
),
describe_dict["component_dependencies"]["json_name"]: [],
describe_dict["container_image_dependencies"]["json_name"]: list(
describe_dict["container_image_dependencies"]["value"]
),
Expand All @@ -656,6 +654,12 @@ def describe(json_option, pipeline_path):
),
},
}

for component_dependency in describe_dict["component_dependencies"]["value"]:
output_dict["dependencies"][describe_dict["component_dependencies"]["json_name"]].append(
json.loads(component_dependency)
)

click.echo(json.dumps(output_dict, indent=indent_length))
else:
# produce human-readable output
Expand Down
114 changes: 61 additions & 53 deletions elyra/tests/cli/resources/pipelines/kfp_3_node_custom.pipeline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,28 +9,30 @@
"id": "primary",
"nodes": [
{
"id": "151538a2-4279-49a4-8742-c3ed50c31b2d",
"id": "4bde6fbe-3935-4fd8-a338-5d15d39d0ddc",
"type": "execution_node",
"op": "elyra-kfp-examples-catalog:d68ec7fcdf46",
"op": "elyra-kfp-examples-catalog:a08014f9252f",
"app_data": {
"component_parameters": {
"hash_algorithm": {
"url": {
"activeControl": "StringControl",
"StringControl": "SHA256"
"StringControl": "https://raw.githubusercontent.com/elyra-ai/elyra/main/elyra/metadata/error.py"
},
"output_hash": "",
"data": {
"value": "b7d9fe00-2a2d-4bd8-8fc4-73a9d8917052",
"option": "elyra_output_filtered_text"
}
"curl_options": {
"activeControl": "StringControl",
"StringControl": "--location"
},
"output_data": "",
"mounted_volumes": [],
"kubernetes_pod_annotations": []
},
"label": "",
"component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml",
"component_source": "{\"catalog_type\": \"elyra-kfp-examples-catalog\", \"component_ref\": {\"component-id\": \"download_data.yaml\"}}",
"ui_data": {
"label": "Calculate data hash",
"label": "Download data",
"image": "/static/elyra/kubeflow.svg",
"x_pos": 90,
"y_pos": 238.5
"x_pos": 84.00000762939453,
"y_pos": 90.5
}
},
"inputs": [
Expand All @@ -44,14 +46,7 @@
},
"label": "Input Port"
}
},
"links": [
{
"id": "05c3101a-9f63-4a0d-9184-ec011fa38511",
"node_id_ref": "b7d9fe00-2a2d-4bd8-8fc4-73a9d8917052",
"port_id_ref": "outPort"
}
]
}
}
],
"outputs": [
Expand All @@ -70,28 +65,32 @@
]
},
{
"id": "321b18d9-1585-4a08-9ca7-9128ee3df527",
"id": "ce91f17a-f5d7-497d-810a-1950260bf60e",
"type": "execution_node",
"op": "elyra-kfp-examples-catalog:a08014f9252f",
"op": "elyra-kfp-examples-catalog:737915b826e9",
"app_data": {
"component_parameters": {
"url": {
"pattern": {
"activeControl": "StringControl",
"StringControl": "https://raw.githubusercontent.com/elyra-ai/elyra/main/elyra/metadata/error.py"
"StringControl": ".*Error"
},
"curl_options": {
"activeControl": "StringControl",
"StringControl": "--location"
},
"output_data": ""
"output_filtered_text": "",
"mounted_volumes": [],
"kubernetes_pod_annotations": [],
"text": {
"value": "4bde6fbe-3935-4fd8-a338-5d15d39d0ddc",
"option": "elyra_output_data"
}
},
"component_description": "Filter input text according to the given regex pattern using shell and grep.",
"label": "",
"component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml",
"component_source": "{\"catalog_type\": \"elyra-kfp-examples-catalog\", \"component_ref\": {\"component-id\": \"filter_text_using_shell_and_grep.yaml\"}}",
"ui_data": {
"label": "Download data",
"label": "Filter text",
"image": "/static/elyra/kubeflow.svg",
"x_pos": 30,
"y_pos": 10.5
"x_pos": 291,
"y_pos": 189.5,
"description": "Filter input text according to the given regex pattern using shell and grep."
}
},
"inputs": [
Expand All @@ -105,7 +104,14 @@
},
"label": "Input Port"
}
}
},
"links": [
{
"id": "9c75642a-b391-44f6-97af-448569774584",
"node_id_ref": "4bde6fbe-3935-4fd8-a338-5d15d39d0ddc",
"port_id_ref": "outPort"
}
]
}
],
"outputs": [
Expand All @@ -124,29 +130,30 @@
]
},
{
"id": "b7d9fe00-2a2d-4bd8-8fc4-73a9d8917052",
"id": "c44f244d-3ca1-4548-8cf0-a76c676849f1",
"type": "execution_node",
"op": "elyra-kfp-examples-catalog:737915b826e9",
"op": "elyra-kfp-examples-catalog:d68ec7fcdf46",
"app_data": {
"component_parameters": {
"component_description": "Filter input text according to the given regex pattern using shell and grep.",
"pattern": {
"hash_algorithm": {
"activeControl": "StringControl",
"StringControl": "*Error"
"StringControl": "SHA256"
},
"output_filtered_text": "",
"text": {
"value": "321b18d9-1585-4a08-9ca7-9128ee3df527",
"option": "elyra_output_data"
"output_hash": "",
"mounted_volumes": [],
"kubernetes_pod_annotations": [],
"data": {
"value": "ce91f17a-f5d7-497d-810a-1950260bf60e",
"option": "elyra_output_filtered_text"
}
},
"component_source": "/opt/anaconda3/envs/elyra-dev/share/jupyter/components/kfp/filter_text_using_shell_and_grep.yaml",
"label": "",
"component_source": "{\"catalog_type\": \"elyra-kfp-examples-catalog\", \"component_ref\": {\"component-id\": \"calculate_hash.yaml\"}}",
"ui_data": {
"label": "Filter text",
"label": "Calculate data hash",
"image": "/static/elyra/kubeflow.svg",
"x_pos": 33,
"y_pos": 89.5,
"description": "Filter input text according to the given regex pattern using shell and grep."
"x_pos": 529,
"y_pos": 263.5
}
},
"inputs": [
Expand All @@ -163,8 +170,8 @@
},
"links": [
{
"id": "99d621f2-63ac-4cef-8f42-7505ccc40792",
"node_id_ref": "321b18d9-1585-4a08-9ca7-9128ee3df527",
"id": "13bf1e1f-971e-4384-a937-dd78cca6d843",
"node_id_ref": "ce91f17a-f5d7-497d-810a-1950260bf60e",
"port_id_ref": "outPort"
}
]
Expand Down Expand Up @@ -193,9 +200,10 @@
"version": 7,
"runtime_type": "KUBEFLOW_PIPELINES",
"properties": {
"pipeline_defaults": {},
"name": "kfp_3_node_custom",
"description": "3-node custom component pipeline",
"runtime": "Kubeflow Pipelines"
"runtime": "Kubeflow Pipelines",
"description": "3-node custom component pipeline"
}
},
"runtime_ref": ""
Expand Down
43 changes: 37 additions & 6 deletions elyra/tests/cli/test_pipeline_app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,16 +264,16 @@ def test_describe_with_kfp_components():
assert "Number of generic nodes: 0" in result.output
assert "Local file dependencies: None specified" in result.output
assert (
"- https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/"
"basics/Calculate_hash/component.yaml" in result.output
'- {"catalog_type": "elyra-kfp-examples-catalog", "component_ref": {"component-id": "download_data.yaml"}}'
in result.output
)
assert (
"- /opt/anaconda3/envs/elyra-dev/share/jupyter/components/"
"kfp/filter_text_using_shell_and_grep.yaml" in result.output
'- {"catalog_type": "elyra-kfp-examples-catalog", "component_ref": '
'{"component-id": "filter_text_using_shell_and_grep.yaml"}}' in result.output
)
assert (
"- https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/"
"web/Download/component.yaml" in result.output
'- {"catalog_type": "elyra-kfp-examples-catalog", "component_ref": {"component-id": "calculate_hash.yaml"}}'
in result.output
)
assert result.exit_code == 0

Expand Down Expand Up @@ -845,6 +845,37 @@ def test_describe_kubernetes_secrets_json():
assert "secret-3" in dependencies["kubernetes_secrets"], dependencies["kubernetes_secrets"]


def test_describe_custom_component_dependencies_json():
"""
Test JSON output for the 'custom_components' dependency property
"""
runner = CliRunner()

#
# - Pipeline contains only custom components

pipeline_file_path = Path(__file__).parent / "resources" / "pipelines" / "kfp_3_node_custom.pipeline"
result = runner.invoke(pipeline, ["describe", "--json", str(pipeline_file_path)])
assert result.exit_code == 0
result_json = json.loads(result.output)
assert result_json["generic_node_count"] == 0
assert result_json["custom_node_count"] == 3
assert isinstance(result_json.get("dependencies"), dict)
dependencies = result_json["dependencies"]
assert isinstance(dependencies["custom_components"], list)
assert len(dependencies["custom_components"]) == 3
assert dependencies["custom_components"][0]["catalog_type"] == "elyra-kfp-examples-catalog"
assert dependencies["custom_components"][1]["catalog_type"] == "elyra-kfp-examples-catalog"
assert dependencies["custom_components"][2]["catalog_type"] == "elyra-kfp-examples-catalog"
expected_component_ids = ["download_data.yaml", "filter_text_using_shell_and_grep.yaml", "calculate_hash.yaml"]
assert dependencies["custom_components"][0]["component_ref"]["component-id"] in expected_component_ids
expected_component_ids.remove(dependencies["custom_components"][0]["component_ref"]["component-id"])
assert dependencies["custom_components"][1]["component_ref"]["component-id"] in expected_component_ids
expected_component_ids.remove(dependencies["custom_components"][1]["component_ref"]["component-id"])
assert dependencies["custom_components"][2]["component_ref"]["component-id"] in expected_component_ids
expected_component_ids.remove(dependencies["custom_components"][2]["component_ref"]["component-id"])


# ------------------------------------------------------------------
# end tests for 'describe' command
# ------------------------------------------------------------------
Expand Down

0 comments on commit e357dff

Please sign in to comment.