From c36affdc63ed105c35268fa48384a808c4ce0438 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Tue, 21 Nov 2023 18:04:00 -0800 Subject: [PATCH 01/13] add opcua check --- azext_edge/edge/params.py | 2 +- azext_edge/edge/providers/check/opcua.py | 253 +++++++++++++++++++++++ azext_edge/edge/providers/checks.py | 13 +- 3 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 azext_edge/edge/providers/check/opcua.py diff --git a/azext_edge/edge/params.py b/azext_edge/edge/params.py index 5b03bf890..448e1f668 100644 --- a/azext_edge/edge/params.py +++ b/azext_edge/edge/params.py @@ -93,7 +93,7 @@ def load_iotops_arguments(self, _): context.argument( "ops_service", options_list=["--ops-service", "--svc"], - choices=CaseInsensitiveList(["mq", "dataprocessor", "lnm"]), + choices=CaseInsensitiveList(["mq", "dataprocessor", "lnm", "opcua"]), help="The IoT Operations service deployment that will be evaluated.", ) context.argument( diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py new file mode 100644 index 000000000..df37e3d7b --- /dev/null +++ b/azext_edge/edge/providers/check/opcua.py @@ -0,0 +1,253 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + +from typing import Any, Dict, List, Tuple + +from azext_edge.edge.providers.base import get_namespaced_pods_by_prefix + +from .base import ( + CheckManager, + add_display_and_eval, + check_post_deployment, + decorate_pod_phase, + evaluate_pod_health, + process_properties, + resources_grouped_by_namespace, +) + +from rich.padding import Padding +from kubernetes.client.models import V1Pod + +from ...common import CheckTaskStatus + +from .common import ( + AIO_LNM_PREFIX, + LNM_ALLOWLIST_PROPERTIES, + LNM_EXCLUDED_SUBRESOURCE, + LNM_IMAGE_PROPERTIES, + LNM_POD_CONDITION_TEXT_MAP, + LNM_REST_PROPERTIES, + ResourceOutputDetailLevel, +) + +from ..edge_api import ( + OPCUA_API_V1, + OpcuaResourceKinds, +) + +from ..support.opcua import OPC_APP_LABEL, OPC_PREFIX + + +def check_opcua_deployment( + result: Dict[str, Any], + as_list: bool = False, + detail_level: int = ResourceOutputDetailLevel.summary.value, + resource_kinds: List[str] = None +) -> None: + evaluate_funcs = { + OpcuaResourceKinds.ASSET_TYPE: evaluate_asset_types, + } + + check_post_deployment( + api_info=OPCUA_API_V1, + check_name="enumerateOpcuaApi", + check_desc="Enumerate OPCUA API resources", + result=result, + resource_kinds_enum=OpcuaResourceKinds, + evaluate_funcs=evaluate_funcs, + as_list=as_list, + detail_level=detail_level, + resource_kinds=resource_kinds, + ) + + +def evaluate_asset_types( + as_list: bool = False, + detail_level: int = ResourceOutputDetailLevel.summary.value, +) -> Dict[str, Any]: + check_manager = CheckManager(check_name="evalAssetTypes", check_desc="Evaluate OPCUA asset types") + + target_asset_types = "assettypes.opcuabroker.iotoperations.azure.com" + asset_type_namespace_conditions = ["len(asset_types)>=0", "status.configStatusLevel", "spec.allowList", "spec.image"] + + all_asset_types: dict = OPCUA_API_V1.get_resources(OpcuaResourceKinds.ASSET_TYPE).get("items", []) + + if not all_asset_types: + fetch_asset_types_error_text = "Unable to fetch OPCUA asset types in any namespaces." + check_manager.add_target_eval( + target_name=target_asset_types, + status=CheckTaskStatus.skipped.value, + value={"asset_types": fetch_asset_types_error_text} + ) + check_manager.add_display(target_name=target_asset_types, display=Padding(fetch_asset_types_error_text, (0, 0, 0, 8))) + + for (namespace, asset_types) in resources_grouped_by_namespace(all_asset_types): + check_manager.add_target(target_name=target_asset_types, namespace=namespace, conditions=asset_type_namespace_conditions) + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"OPCUA asset types in namespace {{[purple]{namespace}[/purple]}}", + (0, 0, 0, 8) + ) + ) + + asset_types: List[dict] = list(asset_types) + asset_types_count = len(asset_types) + asset_types_count_text = "- Expecting [bright_blue]>=1[/bright_blue] instance resource per namespace. {}." + + if asset_types_count >= 1: + asset_types_count_text = asset_types_count_text.format(f"[green]Detected {asset_types_count}[/green]") + else: + asset_types_count_text = asset_types_count_text.format(f"[red]Detected {asset_types_count}[/red]") + check_manager.set_target_status(target_name=all_asset_types, status=CheckTaskStatus.error.value) + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding(asset_types_count_text, (0, 0, 0, 10)) + ) + + for asset_type in asset_types: + asset_type_name = asset_type["metadata"]["name"] + + lnm_text = ( + f"- Opcua asset type {{[bright_blue]{asset_type_name}[/bright_blue]}} detected." + ) + + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding(lnm_text, (0, 0, 0, 12)) + ) + + spec = asset_type["spec"] + if detail_level >= ResourceOutputDetailLevel.detail.value: + # label summarize + labels = spec["labels"] + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"Detected [cyan]{len(labels)}[/cyan] labels", + (0, 0, 0, 16), + ), + ) + + if detail_level == ResourceOutputDetailLevel.verbose.value: + # remove repeated labels + non_repeated_labels = list(set(labels)) + + if len(non_repeated_labels) > 0: + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + "[yellow](Only non repeatative labels will be displayed)[/yellow]", + (0, 0, 0, 20), + ), + ) + + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"[cyan]{', '.join(non_repeated_labels)}[/cyan]", + (0, 0, 0, 20), + ), + ) + + # schema summarize + schema = spec["schema"] + _process_schema( + check_manager=check_manager, + target_asset_types=target_asset_types, + namespace=namespace, + schema=schema, + detail_level=detail_level + ) + + if asset_types_count > 0: + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + "\nRuntime Health", + (0, 0, 0, 10), + ), + ) + + evaluate_pod_health( + check_manager=check_manager, + target=target_asset_types, + pod=OPC_PREFIX, + display_padding=12, + service_label=OPC_APP_LABEL, + namespace=namespace, + ) + + return check_manager.as_dict(as_list) + + +def _process_schema(check_manager: CheckManager, target_asset_types: str, namespace: str, schema: str, detail_level: int = ResourceOutputDetailLevel.summary.value) -> None: + # convert JSON string to dict + import json + schema_dict = json.loads(schema) + + if detail_level == ResourceOutputDetailLevel.detail.value: + # get the schema id + schema_id = schema_dict["@id"] + + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"Schema {{[cyan]{schema_id}[/cyan]}} detected", + (0, 0, 0, 16), + ), + ) + + # get the schema version in @context and the string looks like "dtmi:dtdl:context;3" + schema_version = schema_dict["@context"].split(";")[1] + + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"DTDL version: [cyan]{schema_version}[/cyan]", + (0, 0, 0, 20), + ), + ) + + # get the schema type + schema_type = schema_dict["@type"] + + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + f"Type: [cyan]{schema_type}[/cyan]", + (0, 0, 0, 20), + ), + ) + elif detail_level == ResourceOutputDetailLevel.verbose.value: + from rich.json import JSON + schema_json = JSON(schema, indent=2) + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + "Schema: ", + (0, 0, 0, 16), + ), + ) + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding( + schema_json, + (0, 0, 0, 20), + ), + ) diff --git a/azext_edge/edge/providers/checks.py b/azext_edge/edge/providers/checks.py index e4595eeb4..cce2a0a52 100644 --- a/azext_edge/edge/providers/checks.py +++ b/azext_edge/edge/providers/checks.py @@ -9,9 +9,6 @@ from azure.cli.core.azclierror import ArgumentUsageError from rich.console import Console -from azext_edge.edge.providers.check.lnm import check_lnm_deployment -from azext_edge.edge.providers.edge_api.lnm import LnmResourceKinds - from ..common import ListableEnum, OpsServiceType from .check.base import check_pre_deployment, process_as_list from .check.common import ResourceOutputDetailLevel @@ -19,6 +16,10 @@ from .check.mq import check_mq_deployment from .edge_api.dataprocessor import DataProcessorResourceKinds from .edge_api.mq import MqResourceKinds +from .check.lnm import check_lnm_deployment +from .edge_api.lnm import LnmResourceKinds +from .check.opcua import check_opcua_deployment +from .edge_api.opcua import OpcuaResourceKinds console = Console(width=100, highlight=False) @@ -51,7 +52,8 @@ def run_checks( service_check_dict = { OpsServiceType.mq.value: check_mq_deployment, OpsServiceType.dataprocessor.value: check_dataprocessor_deployment, - OpsServiceType.lnm.value: check_lnm_deployment + OpsServiceType.lnm.value: check_lnm_deployment, + OpsServiceType.opcua.value: check_opcua_deployment, } service_check_dict[ops_service]( detail_level=detail_level, @@ -69,7 +71,8 @@ def _validate_resource_kinds_under_service(ops_service: str, resource_kinds: Lis service_kinds_dict: Dict[str, ListableEnum] = { OpsServiceType.dataprocessor.value: DataProcessorResourceKinds, OpsServiceType.mq.value: MqResourceKinds, - OpsServiceType.lnm.value: LnmResourceKinds + OpsServiceType.lnm.value: LnmResourceKinds, + OpsServiceType.opcua.value: OpcuaResourceKinds } valid_resource_kinds = service_kinds_dict[ops_service].list() if ops_service in service_kinds_dict else [] From d221725663143a090897813607c3e558e9df7ae5 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Wed, 22 Nov 2023 12:38:59 -0800 Subject: [PATCH 02/13] add unit tests --- azext_edge/edge/providers/check/opcua.py | 110 ++++++++-------- azext_edge/tests/edge/checks/conftest.py | 13 ++ .../edge/checks/test_opcua_checks_unit.py | 119 ++++++++++++++++++ 3 files changed, 184 insertions(+), 58 deletions(-) create mode 100644 azext_edge/tests/edge/checks/test_opcua_checks_unit.py diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index df37e3d7b..3acb62416 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -4,32 +4,20 @@ # Licensed under the MIT License. See License file in the project root for license information. # ---------------------------------------------------------------------------------------------- -from typing import Any, Dict, List, Tuple - -from azext_edge.edge.providers.base import get_namespaced_pods_by_prefix +from typing import Any, Dict, List from .base import ( CheckManager, - add_display_and_eval, check_post_deployment, - decorate_pod_phase, evaluate_pod_health, - process_properties, resources_grouped_by_namespace, ) from rich.padding import Padding -from kubernetes.client.models import V1Pod from ...common import CheckTaskStatus from .common import ( - AIO_LNM_PREFIX, - LNM_ALLOWLIST_PROPERTIES, - LNM_EXCLUDED_SUBRESOURCE, - LNM_IMAGE_PROPERTIES, - LNM_POD_CONDITION_TEXT_MAP, - LNM_REST_PROPERTIES, ResourceOutputDetailLevel, ) @@ -38,7 +26,7 @@ OpcuaResourceKinds, ) -from ..support.opcua import OPC_APP_LABEL, OPC_PREFIX +from ..support.opcua import OPC_APP_LABEL, OPC_NAME_LABEL, OPC_PREFIX def check_opcua_deployment( @@ -71,7 +59,8 @@ def evaluate_asset_types( check_manager = CheckManager(check_name="evalAssetTypes", check_desc="Evaluate OPCUA asset types") target_asset_types = "assettypes.opcuabroker.iotoperations.azure.com" - asset_type_namespace_conditions = ["len(asset_types)>=0", "status.configStatusLevel", "spec.allowList", "spec.image"] + asset_type_conditions = ["len(asset_types)>=0"] + check_manager.add_target(target_name=target_asset_types, conditions=asset_type_conditions) all_asset_types: dict = OPCUA_API_V1.get_resources(OpcuaResourceKinds.ASSET_TYPE).get("items", []) @@ -85,7 +74,7 @@ def evaluate_asset_types( check_manager.add_display(target_name=target_asset_types, display=Padding(fetch_asset_types_error_text, (0, 0, 0, 8))) for (namespace, asset_types) in resources_grouped_by_namespace(all_asset_types): - check_manager.add_target(target_name=target_asset_types, namespace=namespace, conditions=asset_type_namespace_conditions) + check_manager.add_target(target_name=target_asset_types, namespace=namespace, conditions=asset_type_conditions) check_manager.add_display( target_name=target_asset_types, namespace=namespace, @@ -113,14 +102,14 @@ def evaluate_asset_types( for asset_type in asset_types: asset_type_name = asset_type["metadata"]["name"] - lnm_text = ( + asset_type_text = ( f"- Opcua asset type {{[bright_blue]{asset_type_name}[/bright_blue]}} detected." ) check_manager.add_display( target_name=target_asset_types, namespace=namespace, - display=Padding(lnm_text, (0, 0, 0, 12)) + display=Padding(asset_type_text, (0, 0, 0, 12)) ) spec = asset_type["spec"] @@ -166,6 +155,7 @@ def evaluate_asset_types( target_asset_types=target_asset_types, namespace=namespace, schema=schema, + padding=16, detail_level=detail_level ) @@ -179,68 +169,72 @@ def evaluate_asset_types( ), ) - evaluate_pod_health( - check_manager=check_manager, - target=target_asset_types, - pod=OPC_PREFIX, - display_padding=12, - service_label=OPC_APP_LABEL, - namespace=namespace, - ) + for pod in ["", OPC_PREFIX]: + evaluate_pod_health( + check_manager=check_manager, + target=target_asset_types, + pod=pod, + display_padding=12, + service_label=OPC_NAME_LABEL if pod == "" else OPC_APP_LABEL, + namespace=namespace, + ) return check_manager.as_dict(as_list) -def _process_schema(check_manager: CheckManager, target_asset_types: str, namespace: str, schema: str, detail_level: int = ResourceOutputDetailLevel.summary.value) -> None: - # convert JSON string to dict - import json - schema_dict = json.loads(schema) +def _process_schema( + check_manager: CheckManager, + target_asset_types: str, + namespace: str, + schema: str, + padding: int, + detail_level: int = ResourceOutputDetailLevel.summary.value +) -> None: if detail_level == ResourceOutputDetailLevel.detail.value: - # get the schema id - schema_id = schema_dict["@id"] + # convert JSON string to dict + import json - check_manager.add_display( - target_name=target_asset_types, - namespace=namespace, - display=Padding( - f"Schema {{[cyan]{schema_id}[/cyan]}} detected", - (0, 0, 0, 16), - ), - ) + schema_dict = json.loads(schema) - # get the schema version in @context and the string looks like "dtmi:dtdl:context;3" - schema_version = schema_dict["@context"].split(";")[1] + schema_items = { + "DTDL version": ("@context", lambda x: x.split(";")[1] if ';' in x else None), + "Type": ("@type", lambda x: x) + } + schema_id = schema_dict["@id"] check_manager.add_display( target_name=target_asset_types, namespace=namespace, - display=Padding( - f"DTDL version: [cyan]{schema_version}[/cyan]", - (0, 0, 0, 20), - ), + display=Padding(f"Schema {{[cyan]{schema_id} detected:}}", (0, 0, 0, padding)), ) - # get the schema type - schema_type = schema_dict["@type"] + padding += 4 + # Loop over the map and add each item to the display + for item_label, (schema_key, value_extractor) in schema_items.items(): + # Extract value using the defined lambda function + item_value = value_extractor(schema_dict[schema_key]) - check_manager.add_display( - target_name=target_asset_types, - namespace=namespace, - display=Padding( - f"Type: [cyan]{schema_type}[/cyan]", - (0, 0, 0, 20), - ), - ) + # Skip adding the display if the extracted value is None + if item_value is None: + continue + + message = f"{item_label}: [cyan]{item_value}[/cyan]" + check_manager.add_display( + target_name=target_asset_types, + namespace=namespace, + display=Padding(message, (0, 0, 0, padding)), + ) elif detail_level == ResourceOutputDetailLevel.verbose.value: from rich.json import JSON + schema_json = JSON(schema, indent=2) check_manager.add_display( target_name=target_asset_types, namespace=namespace, display=Padding( "Schema: ", - (0, 0, 0, 16), + (0, 0, 0, padding), ), ) check_manager.add_display( @@ -248,6 +242,6 @@ def _process_schema(check_manager: CheckManager, target_asset_types: str, namesp namespace=namespace, display=Padding( schema_json, - (0, 0, 0, 20), + (0, 0, 0, padding + 4), ), ) diff --git a/azext_edge/tests/edge/checks/conftest.py b/azext_edge/tests/edge/checks/conftest.py index 37f8b7128..f620f3c52 100644 --- a/azext_edge/tests/edge/checks/conftest.py +++ b/azext_edge/tests/edge/checks/conftest.py @@ -33,6 +33,12 @@ def mock_evaluate_lnm_pod_health(mocker): yield patched +@pytest.fixture +def mock_evaluate_opcua_pod_health(mocker): + patched = mocker.patch("azext_edge.edge.providers.check.opcua.evaluate_pod_health", return_value={}) + yield patched + + @pytest.fixture def mock_get_namespaced_pods_by_prefix(mocker): patched = mocker.patch("azext_edge.edge.providers.check.lnm.get_namespaced_pods_by_prefix", return_value=[]) @@ -70,6 +76,13 @@ def mock_resource_types(mocker, ops_service): "Lnm": [{}] } ) + elif ops_service == "opcua": + patched.return_value = ( + {}, + { + "AssetType": [{}] + } + ) yield patched diff --git a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py new file mode 100644 index 000000000..763e88e5e --- /dev/null +++ b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py @@ -0,0 +1,119 @@ +# coding=utf-8 +# ---------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License file in the project root for license information. +# ---------------------------------------------------------------------------------------------- + + +from azext_edge.edge.providers.check.common import ResourceOutputDetailLevel +import pytest +from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds +from azext_edge.edge.providers.check.opcua import evaluate_asset_types + +from .conftest import ( + assert_check_by_resource_types, + assert_conditions, + assert_evaluations +) +from ...generators import generate_generic_id + + +@pytest.mark.parametrize( + "resource_kinds", + [ + None, + [], + [OpcuaResourceKinds.ASSET_TYPE.value], + ], +) +@pytest.mark.parametrize('ops_service', ['opcua']) +def test_check_opcua_by_resource_types(ops_service, mocker, mock_resource_types, resource_kinds): + eval_lookup = { + OpcuaResourceKinds.ASSET_TYPE.value: "azext_edge.edge.providers.check.opcua.evaluate_asset_types", + } + + assert_check_by_resource_types(ops_service, mocker, mock_resource_types, resource_kinds, eval_lookup) + + +@pytest.mark.parametrize("detail_level", ResourceOutputDetailLevel.list()) +@pytest.mark.parametrize( + "asset_types, namespace_conditions, namespace_evaluations", + [ + ( + # asset_types + [ + { + "metadata": { + "name": "boiler-1", + }, + "spec": { + "description": "", + "labels": [], + "name": "boiler_1", + "schema": '''{"@context":"dtmi:dtdl:context;3","@type":"Interface", + "@id":"dtmi:microsoft:opcuabroker:Boiler__2;1","contents":[]}''' + } + }, + { + "metadata": { + "name": "boiler-2", + }, + "spec": { + "description": "", + "labels": [], + "name": "boiler_2", + "schema": '''{"@context":"dtmi:dtdl:context;3","@type":"Interface", + "@id":"dtmi:microsoft:opcuabroker:Boiler__2;1","contents":[]}''' + } + }, + ], + # namespace conditions str + ["len(asset_types)>=0"], + # namespace evaluations str + [ + [], + ] + ), + ( + # asset_types + [], + # namespace conditions str + ["len(asset_types)>=0"], + # namespace evaluations str + [ + [ + ("status", "skipped"), + ("value/asset_types", "Unable to fetch OPCUA asset types in any namespaces.") + ], + ] + ), + ] +) +def test_asset_types_checks( + mocker, + asset_types, + namespace_conditions, + namespace_evaluations, + detail_level, + mock_evaluate_opcua_pod_health +): + mocker = mocker.patch( + "azext_edge.edge.providers.edge_api.base.EdgeResourceApi.get_resources", + side_effect=[{"items": asset_types}], + ) + + namespace = generate_generic_id() + for asset_type in asset_types: + asset_type['metadata']['namespace'] = namespace + result = evaluate_asset_types(detail_level=detail_level) + + assert result["name"] == "evalAssetTypes" + assert result["targets"]["assettypes.opcuabroker.iotoperations.azure.com"] + target = result["targets"]["assettypes.opcuabroker.iotoperations.azure.com"] + + for namespace in target: + assert namespace in result["targets"]["assettypes.opcuabroker.iotoperations.azure.com"] + + target[namespace]["conditions"] = [] if not target[namespace]["conditions"] else target[namespace]["conditions"] + assert_conditions(target[namespace], namespace_conditions) + assert_evaluations(target[namespace], namespace_evaluations) From 8d4bad531516603bd59f2d861f583bf9936f2810 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 27 Nov 2023 11:54:00 -0800 Subject: [PATCH 03/13] fix style --- azext_edge/edge/providers/check/opcua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index 3acb62416..05ed4b917 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -206,7 +206,7 @@ def _process_schema( check_manager.add_display( target_name=target_asset_types, namespace=namespace, - display=Padding(f"Schema {{[cyan]{schema_id} detected:}}", (0, 0, 0, padding)), + display=Padding(f"Schema {{[cyan]{schema_id}[/cyan]}} detected.", (0, 0, 0, padding)), ) padding += 4 From 45828eee90164517d30878bd9832c7809116a47a Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Mon, 27 Nov 2023 11:55:15 -0800 Subject: [PATCH 04/13] fix --- azext_edge/edge/providers/check/opcua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index 05ed4b917..a89051871 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -210,7 +210,7 @@ def _process_schema( ) padding += 4 - # Loop over the map and add each item to the display + for item_label, (schema_key, value_extractor) in schema_items.items(): # Extract value using the defined lambda function item_value = value_extractor(schema_dict[schema_key]) From 53c3747c4bc4cf9414b9fd435f7d2038c5c59143 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Wed, 29 Nov 2023 09:21:41 -0800 Subject: [PATCH 05/13] address coment --- azext_edge/edge/providers/check/opcua.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index a89051871..5ede624a4 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -116,29 +116,20 @@ def evaluate_asset_types( if detail_level >= ResourceOutputDetailLevel.detail.value: # label summarize labels = spec["labels"] + + # remove repeated labels + non_repeated_labels = list(set(labels)) check_manager.add_display( target_name=target_asset_types, namespace=namespace, display=Padding( - f"Detected [cyan]{len(labels)}[/cyan] labels", + f"Detected [cyan]{len(non_repeated_labels)}[/cyan] unique labels", (0, 0, 0, 16), ), ) if detail_level == ResourceOutputDetailLevel.verbose.value: - # remove repeated labels - non_repeated_labels = list(set(labels)) - if len(non_repeated_labels) > 0: - check_manager.add_display( - target_name=target_asset_types, - namespace=namespace, - display=Padding( - "[yellow](Only non repeatative labels will be displayed)[/yellow]", - (0, 0, 0, 20), - ), - ) - check_manager.add_display( target_name=target_asset_types, namespace=namespace, From e3ee04c1e45383ee1e887ade69038ea4f5f768d1 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Wed, 29 Nov 2023 09:57:35 -0800 Subject: [PATCH 06/13] update --- azext_edge/edge/providers/check/opcua.py | 8 ++++++-- azext_edge/tests/edge/checks/test_opcua_checks_unit.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index 5ede624a4..c073e7b15 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -60,12 +60,12 @@ def evaluate_asset_types( target_asset_types = "assettypes.opcuabroker.iotoperations.azure.com" asset_type_conditions = ["len(asset_types)>=0"] - check_manager.add_target(target_name=target_asset_types, conditions=asset_type_conditions) all_asset_types: dict = OPCUA_API_V1.get_resources(OpcuaResourceKinds.ASSET_TYPE).get("items", []) if not all_asset_types: fetch_asset_types_error_text = "Unable to fetch OPCUA asset types in any namespaces." + check_manager.add_target(target_name=target_asset_types) check_manager.add_target_eval( target_name=target_asset_types, status=CheckTaskStatus.skipped.value, @@ -74,7 +74,11 @@ def evaluate_asset_types( check_manager.add_display(target_name=target_asset_types, display=Padding(fetch_asset_types_error_text, (0, 0, 0, 8))) for (namespace, asset_types) in resources_grouped_by_namespace(all_asset_types): - check_manager.add_target(target_name=target_asset_types, namespace=namespace, conditions=asset_type_conditions) + check_manager.add_target( + target_name=target_asset_types, + namespace=namespace, + conditions=asset_type_conditions + ) check_manager.add_display( target_name=target_asset_types, namespace=namespace, diff --git a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py index 763e88e5e..c9bd553ff 100644 --- a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py +++ b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py @@ -78,7 +78,7 @@ def test_check_opcua_by_resource_types(ops_service, mocker, mock_resource_types, # asset_types [], # namespace conditions str - ["len(asset_types)>=0"], + [], # namespace evaluations str [ [ From 9294687548d6de8a98cda5c082efb6c81871ed6c Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Thu, 30 Nov 2023 18:23:03 -0800 Subject: [PATCH 07/13] did some restructure --- azext_edge/edge/providers/check/base.py | 30 ++++++-- azext_edge/edge/providers/check/common.py | 4 +- azext_edge/edge/providers/check/opcua.py | 90 ++++++++++++++++------- 3 files changed, 90 insertions(+), 34 deletions(-) diff --git a/azext_edge/edge/providers/check/base.py b/azext_edge/edge/providers/check/base.py index d9fe9ec12..4cd084176 100644 --- a/azext_edge/edge/providers/check/base.py +++ b/azext_edge/edge/providers/check/base.py @@ -17,7 +17,7 @@ from rich.console import Console, NewLine from rich.padding import Padding -from .common import ALL_NAMESPACES_TARGET, ResourceOutputDetailLevel +from .common import ALL_NAMESPACES_TARGET, CORE_SERVICE_RUNTIME_RESOURCE, ResourceOutputDetailLevel from ...common import CheckTaskStatus, ListableEnum from ...providers.edge_api import EdgeResourceApi @@ -70,8 +70,9 @@ def check_post_deployment( lowercase_api_resources = {k.lower(): v for k, v in api_resources.items()} if lowercase_api_resources: - for api_resource, evaluate_func in evaluate_funcs.items(): - if api_resource.value in lowercase_api_resources and check_resources[api_resource]: + for resource, evaluate_func in evaluate_funcs.items(): + if (resource == CORE_SERVICE_RUNTIME_RESOURCE) or\ + (resource.value in lowercase_api_resources and check_resources[resource]): result["postDeployment"].append(evaluate_func(detail_level=detail_level, as_list=as_list)) @@ -522,7 +523,25 @@ def evaluate_pod_health( target_service_pod = f"pod/{pod}" check_manager.add_target_conditions(target_name=target, namespace=namespace, conditions=[f"{target_service_pod}.status.phase"]) diagnostics_pods = get_namespaced_pods_by_prefix(prefix=pod, namespace=namespace, label_selector=service_label) - if not diagnostics_pods: + process_pods_status( + check_manager=check_manager, + namespace=namespace, + target=target, + target_service_pod=target_service_pod, + pods=diagnostics_pods, + display_padding=display_padding, + ) + + +def process_pods_status( + check_manager: CheckManager, + namespace: str, + target: str, + target_service_pod: str, + pods: List[dict], + display_padding: int, +) -> None: + if not pods: add_display_and_eval( check_manager=check_manager, target_name=target, @@ -535,11 +554,12 @@ def evaluate_pod_health( ) else: - for pod in diagnostics_pods: + for pod in pods: pod_dict = pod.to_dict() pod_name = pod_dict["metadata"]["name"] pod_phase = pod_dict.get("status", {}).get("phase") pod_phase_deco, status = decorate_pod_phase(pod_phase) + target_service_pod = f"pod/{pod_name}" check_manager.add_target_eval( target_name=target, diff --git a/azext_edge/edge/providers/check/common.py b/azext_edge/edge/providers/check/common.py index d3f7536ae..ca8aebcb7 100644 --- a/azext_edge/edge/providers/check/common.py +++ b/azext_edge/edge/providers/check/common.py @@ -163,7 +163,9 @@ class DataprocessorDestinationStageType(ListableEnum): # Check constants ALL_NAMESPACES_TARGET = '_all_' - +# when there are runtime resources related to the service but not +# related to any service resource, use this constant as the resource name +CORE_SERVICE_RUNTIME_RESOURCE = "coreServiceRuntimeResources" # MQ connector enums class KafkaTopicMapRouteType(Enum): diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index c073e7b15..dc4d852a1 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -4,20 +4,26 @@ # Licensed under the MIT License. See License file in the project root for license information. # ---------------------------------------------------------------------------------------------- +from itertools import groupby from typing import Any, Dict, List +from azext_edge.edge.providers.base import get_namespaced_pods_by_prefix + from .base import ( CheckManager, check_post_deployment, evaluate_pod_health, + process_pods_status, resources_grouped_by_namespace, ) +from kubernetes.client.models import V1Pod from rich.padding import Padding from ...common import CheckTaskStatus from .common import ( + CORE_SERVICE_RUNTIME_RESOURCE, ResourceOutputDetailLevel, ) @@ -26,7 +32,7 @@ OpcuaResourceKinds, ) -from ..support.opcua import OPC_APP_LABEL, OPC_NAME_LABEL, OPC_PREFIX +from ..support.opcua import OPC_APP_LABEL, OPC_NAME_LABEL, OPC_PREFIX, SIMULATOR_PREFIX def check_opcua_deployment( @@ -36,13 +42,14 @@ def check_opcua_deployment( resource_kinds: List[str] = None ) -> None: evaluate_funcs = { + CORE_SERVICE_RUNTIME_RESOURCE: evaluate_core_service_runtime, OpcuaResourceKinds.ASSET_TYPE: evaluate_asset_types, } check_post_deployment( api_info=OPCUA_API_V1, - check_name="enumerateOpcuaApi", - check_desc="Enumerate OPCUA API resources", + check_name="enumerateOpcUaBrokerApi", + check_desc="Enumerate OPC UA Broker API resources", result=result, resource_kinds_enum=OpcuaResourceKinds, evaluate_funcs=evaluate_funcs, @@ -52,19 +59,66 @@ def check_opcua_deployment( ) +def evaluate_core_service_runtime( + as_list: bool = False, + detail_level: int = ResourceOutputDetailLevel.summary.value, +) -> Dict[str, Any]: + check_manager = CheckManager(check_name="evalCoreServiceRuntime", check_desc="Evaluate OPC UA broker core service runtime resources") + + opcua_runtime_resources = get_namespaced_pods_by_prefix( + prefix="", + namespace="", + label_selector=OPC_APP_LABEL, + ) + opcua_runtime_resources.extend( + get_namespaced_pods_by_prefix( + prefix="", + namespace="", + label_selector=OPC_NAME_LABEL, + ) + ) + + def get_namespace(pod: V1Pod) -> str: + return pod.metadata.namespace + + opcua_runtime_resources.sort(key=get_namespace) + + for (namespace, pods) in groupby(opcua_runtime_resources, get_namespace): + check_manager.add_target(target_name=CORE_SERVICE_RUNTIME_RESOURCE, namespace=namespace) + check_manager.add_display( + target_name=CORE_SERVICE_RUNTIME_RESOURCE, + namespace=namespace, + display=Padding( + f"OPC UA runtime resources for namespace {{[purple]{namespace}[/purple]}}", + (0, 0, 0, 6) + ) + ) + + process_pods_status( + check_manager=check_manager, + target_service_pod="", + target=CORE_SERVICE_RUNTIME_RESOURCE, + pods=list(pods), + namespace=namespace, + display_padding=10, + ) + + return check_manager.as_dict(as_list) + + def evaluate_asset_types( as_list: bool = False, detail_level: int = ResourceOutputDetailLevel.summary.value, ) -> Dict[str, Any]: - check_manager = CheckManager(check_name="evalAssetTypes", check_desc="Evaluate OPCUA asset types") + check_manager = CheckManager(check_name="evalAssetTypes", check_desc="Evaluate OPC UA broker asset types") - target_asset_types = "assettypes.opcuabroker.iotoperations.azure.com" + target_asset_types = f"{OPCUA_API_V1._kinds}.{OPCUA_API_V1.group}" asset_type_conditions = ["len(asset_types)>=0"] all_asset_types: dict = OPCUA_API_V1.get_resources(OpcuaResourceKinds.ASSET_TYPE).get("items", []) if not all_asset_types: - fetch_asset_types_error_text = "Unable to fetch OPCUA asset types in any namespaces." + fetch_asset_types_error_text = "Unable to fetch OPC UA broker asset types in any namespaces." check_manager.add_target(target_name=target_asset_types) check_manager.add_target_eval( target_name=target_asset_types, @@ -83,7 +137,7 @@ def evaluate_asset_types( target_name=target_asset_types, namespace=namespace, display=Padding( - f"OPCUA asset types in namespace {{[purple]{namespace}[/purple]}}", + f"OPC UA broker asset types in namespace {{[purple]{namespace}[/purple]}}", (0, 0, 0, 8) ) ) @@ -107,7 +161,7 @@ def evaluate_asset_types( asset_type_name = asset_type["metadata"]["name"] asset_type_text = ( - f"- Opcua asset type {{[bright_blue]{asset_type_name}[/bright_blue]}} detected." + f"- Asset type {{[bright_blue]{asset_type_name}[/bright_blue]}} detected." ) check_manager.add_display( @@ -154,26 +208,6 @@ def evaluate_asset_types( detail_level=detail_level ) - if asset_types_count > 0: - check_manager.add_display( - target_name=target_asset_types, - namespace=namespace, - display=Padding( - "\nRuntime Health", - (0, 0, 0, 10), - ), - ) - - for pod in ["", OPC_PREFIX]: - evaluate_pod_health( - check_manager=check_manager, - target=target_asset_types, - pod=pod, - display_padding=12, - service_label=OPC_NAME_LABEL if pod == "" else OPC_APP_LABEL, - namespace=namespace, - ) - return check_manager.as_dict(as_list) From 5de139c28e10424a8345854bcea7e6f6f6c41eba Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 14:27:36 -0800 Subject: [PATCH 08/13] move core service check to new section and address pr comments --- azext_edge/edge/providers/check/base.py | 7 +- azext_edge/edge/providers/check/common.py | 1 + .../edge/providers/check/dataprocessor.py | 9 +- azext_edge/edge/providers/check/lnm.py | 3 +- azext_edge/edge/providers/check/opcua.py | 36 ++++---- azext_edge/tests/edge/checks/conftest.py | 48 ++++++++++- .../tests/edge/checks/test_lnm_checks_unit.py | 1 + .../edge/checks/test_opcua_checks_unit.py | 83 +++++++++++++++++-- 8 files changed, 154 insertions(+), 34 deletions(-) diff --git a/azext_edge/edge/providers/check/base.py b/azext_edge/edge/providers/check/base.py index 4cd084176..b25c40deb 100644 --- a/azext_edge/edge/providers/check/base.py +++ b/azext_edge/edge/providers/check/base.py @@ -72,7 +72,7 @@ def check_post_deployment( if lowercase_api_resources: for resource, evaluate_func in evaluate_funcs.items(): if (resource == CORE_SERVICE_RUNTIME_RESOURCE) or\ - (resource.value in lowercase_api_resources and check_resources[resource]): + (resource.value in lowercase_api_resources and check_resources[resource]): result["postDeployment"].append(evaluate_func(detail_level=detail_level, as_list=as_list)) @@ -710,3 +710,8 @@ def resources_grouped_by_namespace(resources: List[dict]): def filter_by_namespace(resources: List[dict], namespace: str) -> List[dict]: return [resource for resource in resources if get_resource_namespace(resource) == namespace] + + +def generate_target_resource_name(api_info: EdgeResourceApi, resource_kind: str) -> str: + resource_plural = api_info._kinds[resource_kind] if api_info._kinds else f"{resource_kind}s" + return f"{resource_plural}.{api_info.group}" diff --git a/azext_edge/edge/providers/check/common.py b/azext_edge/edge/providers/check/common.py index ca8aebcb7..b3ef55824 100644 --- a/azext_edge/edge/providers/check/common.py +++ b/azext_edge/edge/providers/check/common.py @@ -167,6 +167,7 @@ class DataprocessorDestinationStageType(ListableEnum): # related to any service resource, use this constant as the resource name CORE_SERVICE_RUNTIME_RESOURCE = "coreServiceRuntimeResources" + # MQ connector enums class KafkaTopicMapRouteType(Enum): """ diff --git a/azext_edge/edge/providers/check/dataprocessor.py b/azext_edge/edge/providers/check/dataprocessor.py index 3642f3767..081725604 100644 --- a/azext_edge/edge/providers/check/dataprocessor.py +++ b/azext_edge/edge/providers/check/dataprocessor.py @@ -11,6 +11,7 @@ decorate_resource_status, check_post_deployment, evaluate_pod_health, + generate_target_resource_name, process_properties, process_property_by_type, resources_grouped_by_namespace, @@ -73,12 +74,12 @@ def evaluate_instances( ) -> Dict[str, Any]: check_manager = CheckManager(check_name="evalInstances", check_desc="Evaluate Data processor instance") - target_instances = "instances.dataprocessor.iotoperations.azure.com" instance_namespace_conditions = ["len(instances)==1", "provisioningStatus"] instance_all_conditions = ["instances"] - check_manager.add_target(target_name=target_instances, conditions=instance_all_conditions) instance_resources: dict = DATA_PROCESSOR_API_V1.get_resources(DataProcessorResourceKinds.INSTANCE) + target_instances = generate_target_resource_name(api_info=DATA_PROCESSOR_API_V1, resource_kind=DataProcessorResourceKinds.INSTANCE.value) + check_manager.add_target(target_name=target_instances, conditions=instance_all_conditions) all_instances: list = instance_resources.get("items", []) if instance_resources else [] if not all_instances: @@ -211,7 +212,7 @@ def evaluate_pipelines( ) -> Dict[str, Any]: check_manager = CheckManager(check_name="evalPipelines", check_desc="Evaluate Data processor pipeline") - target_pipelines = "pipelines.dataprocessor.iotoperations.azure.com" + target_pipelines = generate_target_resource_name(api_info=DATA_PROCESSOR_API_V1, resource_kind=DataProcessorResourceKinds.PIPELINE.value) pipeline_all_conditions = ["pipelines"] pipeline_namespace_conditions = [ "len(pipelines)>=1", @@ -394,7 +395,7 @@ def evaluate_datasets( ) -> Dict[str, Any]: check_manager = CheckManager(check_name="evalDatasets", check_desc="Evaluate Data processor dataset") - target_datasets = "datasets.dataprocessor.iotoperations.azure.com" + target_datasets = generate_target_resource_name(api_info=DATA_PROCESSOR_API_V1, resource_kind=DataProcessorResourceKinds.DATASET.value) dataset_all_conditions = ["datasets"] dataset_namespace_conditions = ["provisioningState"] check_manager.add_target(target_name=target_datasets, conditions=dataset_all_conditions) diff --git a/azext_edge/edge/providers/check/lnm.py b/azext_edge/edge/providers/check/lnm.py index 117f618dd..cafeaea3d 100644 --- a/azext_edge/edge/providers/check/lnm.py +++ b/azext_edge/edge/providers/check/lnm.py @@ -13,6 +13,7 @@ add_display_and_eval, check_post_deployment, decorate_pod_phase, + generate_target_resource_name, process_properties, resources_grouped_by_namespace, ) @@ -70,10 +71,10 @@ def evaluate_lnms( ) -> Dict[str, Any]: check_manager = CheckManager(check_name="evalLnms", check_desc="Evaluate LNM instances") - target_lnms = "lnmz.layerednetworkmgmt.iotoperations.azure.com" lnm_namespace_conditions = ["len(lnms)>=1", "status.configStatusLevel", "spec.allowList", "spec.image"] all_lnms: dict = LNM_API_V1B1.get_resources(LnmResourceKinds.LNM).get("items", []) + target_lnms = generate_target_resource_name(api_info=LNM_API_V1B1, resource_kind=LnmResourceKinds.LNM.value) if not all_lnms: fetch_lnms_error_text = "Unable to fetch LNM instances in any namespaces." diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index dc4d852a1..bcfc2fdac 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -12,7 +12,7 @@ from .base import ( CheckManager, check_post_deployment, - evaluate_pod_health, + generate_target_resource_name, process_pods_status, resources_grouped_by_namespace, ) @@ -32,7 +32,7 @@ OpcuaResourceKinds, ) -from ..support.opcua import OPC_APP_LABEL, OPC_NAME_LABEL, OPC_PREFIX, SIMULATOR_PREFIX +from ..support.opcua import OPC_APP_LABEL, OPC_NAME_LABEL def check_opcua_deployment( @@ -49,7 +49,7 @@ def check_opcua_deployment( check_post_deployment( api_info=OPCUA_API_V1, check_name="enumerateOpcUaBrokerApi", - check_desc="Enumerate OPC UA Broker API resources", + check_desc="Enumerate OPC UA broker API resources", result=result, resource_kinds_enum=OpcuaResourceKinds, evaluate_funcs=evaluate_funcs, @@ -63,20 +63,17 @@ def evaluate_core_service_runtime( as_list: bool = False, detail_level: int = ResourceOutputDetailLevel.summary.value, ) -> Dict[str, Any]: - check_manager = CheckManager(check_name="evalCoreServiceRuntime", check_desc="Evaluate OPC UA broker core service runtime resources") - - opcua_runtime_resources = get_namespaced_pods_by_prefix( - prefix="", - namespace="", - label_selector=OPC_APP_LABEL, - ) - opcua_runtime_resources.extend( - get_namespaced_pods_by_prefix( - prefix="", - namespace="", - label_selector=OPC_NAME_LABEL, + check_manager = CheckManager(check_name="evalCoreServiceRuntime", check_desc="Evaluate OPC UA broker core service") + + opcua_runtime_resources: List[dict] = [] + for label_selector in [OPC_APP_LABEL, OPC_NAME_LABEL]: + opcua_runtime_resources.extend( + get_namespaced_pods_by_prefix( + prefix="", + namespace="", + label_selector=label_selector, + ) ) - ) def get_namespace(pod: V1Pod) -> str: return pod.metadata.namespace @@ -89,7 +86,7 @@ def get_namespace(pod: V1Pod) -> str: target_name=CORE_SERVICE_RUNTIME_RESOURCE, namespace=namespace, display=Padding( - f"OPC UA runtime resources for namespace {{[purple]{namespace}[/purple]}}", + f"OPC UA broker runtime resources in namespace {{[purple]{namespace}[/purple]}}", (0, 0, 0, 6) ) ) @@ -111,11 +108,10 @@ def evaluate_asset_types( detail_level: int = ResourceOutputDetailLevel.summary.value, ) -> Dict[str, Any]: check_manager = CheckManager(check_name="evalAssetTypes", check_desc="Evaluate OPC UA broker asset types") - - target_asset_types = f"{OPCUA_API_V1._kinds}.{OPCUA_API_V1.group}" asset_type_conditions = ["len(asset_types)>=0"] all_asset_types: dict = OPCUA_API_V1.get_resources(OpcuaResourceKinds.ASSET_TYPE).get("items", []) + target_asset_types = generate_target_resource_name(api_info=OPCUA_API_V1, resource_kind=OpcuaResourceKinds.ASSET_TYPE.value) if not all_asset_types: fetch_asset_types_error_text = "Unable to fetch OPC UA broker asset types in any namespaces." @@ -144,7 +140,7 @@ def evaluate_asset_types( asset_types: List[dict] = list(asset_types) asset_types_count = len(asset_types) - asset_types_count_text = "- Expecting [bright_blue]>=1[/bright_blue] instance resource per namespace. {}." + asset_types_count_text = "- Expecting [bright_blue]>=1[/bright_blue] asset type resource per namespace. {}." if asset_types_count >= 1: asset_types_count_text = asset_types_count_text.format(f"[green]Detected {asset_types_count}[/green]") diff --git a/azext_edge/tests/edge/checks/conftest.py b/azext_edge/tests/edge/checks/conftest.py index f620f3c52..393e0baa3 100644 --- a/azext_edge/tests/edge/checks/conftest.py +++ b/azext_edge/tests/edge/checks/conftest.py @@ -4,7 +4,9 @@ # Licensed under the MIT License. See License file in the project root for license information. # ---------------------------------------------------------------------------------------------- from typing import List, Dict, Any +from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE import pytest +from kubernetes.client import V1Pod, V1ObjectMeta, V1PodStatus from azext_edge.edge.providers.checks import run_checks @@ -35,7 +37,7 @@ def mock_evaluate_lnm_pod_health(mocker): @pytest.fixture def mock_evaluate_opcua_pod_health(mocker): - patched = mocker.patch("azext_edge.edge.providers.check.opcua.evaluate_pod_health", return_value={}) + patched = mocker.patch("azext_edge.edge.providers.check.opcua.get_namespaced_pods_by_prefix", return_value={}) yield patched @@ -45,6 +47,30 @@ def mock_get_namespaced_pods_by_prefix(mocker): yield patched +@pytest.fixture +def mock_generate_lnm_target_resources(mocker): + patched = mocker.patch( + "azext_edge.edge.providers.check.lnm.generate_target_resource_name", + return_value="lnmz.layerednetworkmgmt.iotoperations.azure.com" + ) + yield patched + + +@pytest.fixture +def mock_generate_opcua_target_resources(mocker): + patched = mocker.patch( + "azext_edge.edge.providers.check.opcua.generate_target_resource_name", + return_value="assettypes.opcuabroker.iotoperations.azure.com" + ) + yield patched + + +@pytest.fixture +def mock_opcua_get_namespaced_pods_by_prefix(mocker): + patched = mocker.patch("azext_edge.edge.providers.check.opcua.get_namespaced_pods_by_prefix", return_value=[]) + yield patched + + @pytest.fixture def mock_resource_types(mocker, ops_service): patched = mocker.patch("azext_edge.edge.providers.check.base.enumerate_ops_service_resources") @@ -80,7 +106,7 @@ def mock_resource_types(mocker, ops_service): patched.return_value = ( {}, { - "AssetType": [{}] + "AssetType": [{}], } ) @@ -131,6 +157,16 @@ def generate_resource_stub( return resource +def generate_pod_stub( + name: str, + phase: str, +): + metadata = V1ObjectMeta(name=name) + pod_status = V1PodStatus(phase=phase) + pod = V1Pod(metadata=metadata, status=pod_status) + return pod + + def assert_check_by_resource_types(ops_service, mocker, mock_resource_types, resource_kinds, eval_lookup): # Mock the functions for key, value in eval_lookup.items(): @@ -153,7 +189,13 @@ def assert_check_by_resource_types(ops_service, mocker, mock_resource_types, res for resource_kind in resource_kinds: eval_lookup[resource_kind].assert_called_once() del eval_lookup[resource_kind] - # ensure no other checks were run + + # ensure core service runtime check was run once when it exists + if CORE_SERVICE_RUNTIME_RESOURCE in eval_lookup: + eval_lookup[CORE_SERVICE_RUNTIME_RESOURCE].assert_called_once() + del eval_lookup[CORE_SERVICE_RUNTIME_RESOURCE] + + # ensure no other checks were run except core service runtime [eval_lookup[evaluator].assert_not_called() for evaluator in eval_lookup] diff --git a/azext_edge/tests/edge/checks/test_lnm_checks_unit.py b/azext_edge/tests/edge/checks/test_lnm_checks_unit.py index 945c777fb..16df580ba 100644 --- a/azext_edge/tests/edge/checks/test_lnm_checks_unit.py +++ b/azext_edge/tests/edge/checks/test_lnm_checks_unit.py @@ -202,6 +202,7 @@ def test_check_lnm_by_resource_types(ops_service, mocker, mock_resource_types, r def test_lnm_checks( mocker, mock_get_namespaced_pods_by_prefix, + mock_generate_lnm_target_resources, lnms, namespace_conditions, namespace_evaluations, diff --git a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py index c9bd553ff..5cceba7f7 100644 --- a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py +++ b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py @@ -5,15 +5,16 @@ # ---------------------------------------------------------------------------------------------- -from azext_edge.edge.providers.check.common import ResourceOutputDetailLevel +from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE, ResourceOutputDetailLevel import pytest from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds -from azext_edge.edge.providers.check.opcua import evaluate_asset_types +from azext_edge.edge.providers.check.opcua import evaluate_asset_types, evaluate_core_service_runtime from .conftest import ( assert_check_by_resource_types, assert_conditions, - assert_evaluations + assert_evaluations, + generate_pod_stub ) from ...generators import generate_generic_id @@ -29,6 +30,7 @@ @pytest.mark.parametrize('ops_service', ['opcua']) def test_check_opcua_by_resource_types(ops_service, mocker, mock_resource_types, resource_kinds): eval_lookup = { + CORE_SERVICE_RUNTIME_RESOURCE: "azext_edge.edge.providers.check.opcua.evaluate_core_service_runtime", OpcuaResourceKinds.ASSET_TYPE.value: "azext_edge.edge.providers.check.opcua.evaluate_asset_types", } @@ -83,7 +85,7 @@ def test_check_opcua_by_resource_types(ops_service, mocker, mock_resource_types, [ [ ("status", "skipped"), - ("value/asset_types", "Unable to fetch OPCUA asset types in any namespaces.") + ("value/asset_types", "Unable to fetch OPC UA broker asset types in any namespaces.") ], ] ), @@ -94,8 +96,8 @@ def test_asset_types_checks( asset_types, namespace_conditions, namespace_evaluations, + mock_generate_opcua_target_resources, detail_level, - mock_evaluate_opcua_pod_health ): mocker = mocker.patch( "azext_edge.edge.providers.edge_api.base.EdgeResourceApi.get_resources", @@ -117,3 +119,74 @@ def test_asset_types_checks( target[namespace]["conditions"] = [] if not target[namespace]["conditions"] else target[namespace]["conditions"] assert_conditions(target[namespace], namespace_conditions) assert_evaluations(target[namespace], namespace_evaluations) + + +@pytest.mark.parametrize("detail_level", ResourceOutputDetailLevel.list()) +@pytest.mark.parametrize( + "pods, namespace_conditions, namespace_evaluations", + [ + ( + # pods + [ + generate_pod_stub( + name="opcua-broker-1", + phase="Running", + ) + ], + # namespace conditions str + [], + # namespace evaluations str + [ + [ + ("status", "success"), + ("value/status.phase", "Running"), + ], + ] + ), + ( + # pods + [ + generate_pod_stub( + name="opcua-broker-1", + phase="Failed", + ) + ], + # namespace conditions str + [], + # namespace evaluations str + [ + [ + ("status", "error") + ], + ] + ), + ] +) +def test_evaluate_core_service_runtime( + mocker, + pods, + namespace_conditions, + namespace_evaluations, + mock_generate_opcua_target_resources, + detail_level, +): + mocker = mocker.patch( + "azext_edge.edge.providers.check.opcua.get_namespaced_pods_by_prefix", + return_value=pods, + ) + + namespace = generate_generic_id() + for pod in pods: + pod.metadata.namespace = namespace + result = evaluate_core_service_runtime(detail_level=detail_level) + + assert result["name"] == "evalCoreServiceRuntime" + assert result["targets"][CORE_SERVICE_RUNTIME_RESOURCE] + target = result["targets"][CORE_SERVICE_RUNTIME_RESOURCE] + + for namespace in target: + assert namespace in result["targets"][CORE_SERVICE_RUNTIME_RESOURCE] + + target[namespace]["conditions"] = [] if not target[namespace]["conditions"] else target[namespace]["conditions"] + assert_conditions(target[namespace], namespace_conditions) + assert_evaluations(target[namespace], namespace_evaluations) From c49cf9bcf97c8792ec14d2c0fa848fcf55121cd0 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 15:28:21 -0800 Subject: [PATCH 09/13] clean up imports --- azext_edge/edge/providers/check/opcua.py | 7 +++---- azext_edge/edge/providers/checks.py | 6 +++--- azext_edge/tests/edge/checks/conftest.py | 5 ++--- azext_edge/tests/edge/checks/test_opcua_checks_unit.py | 5 ++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/azext_edge/edge/providers/check/opcua.py b/azext_edge/edge/providers/check/opcua.py index bcfc2fdac..b3652e8a3 100644 --- a/azext_edge/edge/providers/check/opcua.py +++ b/azext_edge/edge/providers/check/opcua.py @@ -5,9 +5,11 @@ # ---------------------------------------------------------------------------------------------- from itertools import groupby +from kubernetes.client.models import V1Pod +from rich.padding import Padding from typing import Any, Dict, List -from azext_edge.edge.providers.base import get_namespaced_pods_by_prefix +from ..base import get_namespaced_pods_by_prefix from .base import ( CheckManager, @@ -17,9 +19,6 @@ resources_grouped_by_namespace, ) -from kubernetes.client.models import V1Pod -from rich.padding import Padding - from ...common import CheckTaskStatus from .common import ( diff --git a/azext_edge/edge/providers/checks.py b/azext_edge/edge/providers/checks.py index cce2a0a52..7537f92e9 100644 --- a/azext_edge/edge/providers/checks.py +++ b/azext_edge/edge/providers/checks.py @@ -13,12 +13,12 @@ from .check.base import check_pre_deployment, process_as_list from .check.common import ResourceOutputDetailLevel from .check.dataprocessor import check_dataprocessor_deployment +from .check.lnm import check_lnm_deployment from .check.mq import check_mq_deployment +from .check.opcua import check_opcua_deployment from .edge_api.dataprocessor import DataProcessorResourceKinds -from .edge_api.mq import MqResourceKinds -from .check.lnm import check_lnm_deployment from .edge_api.lnm import LnmResourceKinds -from .check.opcua import check_opcua_deployment +from .edge_api.mq import MqResourceKinds from .edge_api.opcua import OpcuaResourceKinds console = Console(width=100, highlight=False) diff --git a/azext_edge/tests/edge/checks/conftest.py b/azext_edge/tests/edge/checks/conftest.py index 393e0baa3..a18b82476 100644 --- a/azext_edge/tests/edge/checks/conftest.py +++ b/azext_edge/tests/edge/checks/conftest.py @@ -3,12 +3,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License file in the project root for license information. # ---------------------------------------------------------------------------------------------- -from typing import List, Dict, Any -from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE import pytest from kubernetes.client import V1Pod, V1ObjectMeta, V1PodStatus - +from typing import List, Dict, Any from azext_edge.edge.providers.checks import run_checks +from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE @pytest.fixture diff --git a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py index 5cceba7f7..b9371c005 100644 --- a/azext_edge/tests/edge/checks/test_opcua_checks_unit.py +++ b/azext_edge/tests/edge/checks/test_opcua_checks_unit.py @@ -4,11 +4,10 @@ # Licensed under the MIT License. See License file in the project root for license information. # ---------------------------------------------------------------------------------------------- - -from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE, ResourceOutputDetailLevel import pytest -from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds +from azext_edge.edge.providers.check.common import CORE_SERVICE_RUNTIME_RESOURCE, ResourceOutputDetailLevel from azext_edge.edge.providers.check.opcua import evaluate_asset_types, evaluate_core_service_runtime +from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds from .conftest import ( assert_check_by_resource_types, From aa4b1ce7f4e3513844ac542dbb2db804bfa5311d Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 16:24:32 -0800 Subject: [PATCH 10/13] pr comment --- azext_edge/edge/providers/check/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azext_edge/edge/providers/check/common.py b/azext_edge/edge/providers/check/common.py index b3ef55824..05ae4ba15 100644 --- a/azext_edge/edge/providers/check/common.py +++ b/azext_edge/edge/providers/check/common.py @@ -165,7 +165,7 @@ class DataprocessorDestinationStageType(ListableEnum): ALL_NAMESPACES_TARGET = '_all_' # when there are runtime resources related to the service but not # related to any service resource, use this constant as the resource name -CORE_SERVICE_RUNTIME_RESOURCE = "coreServiceRuntimeResources" +CORE_SERVICE_RUNTIME_RESOURCE = "coreServiceRuntimeResource" # MQ connector enums From 1ac0b07bbb6daae423d9677b3575dc1146411056 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 16:30:01 -0800 Subject: [PATCH 11/13] address comment --- azext_edge/edge/providers/check/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azext_edge/edge/providers/check/base.py b/azext_edge/edge/providers/check/base.py index b25c40deb..3ae1c1a8b 100644 --- a/azext_edge/edge/providers/check/base.py +++ b/azext_edge/edge/providers/check/base.py @@ -522,13 +522,13 @@ def evaluate_pod_health( ) -> None: target_service_pod = f"pod/{pod}" check_manager.add_target_conditions(target_name=target, namespace=namespace, conditions=[f"{target_service_pod}.status.phase"]) - diagnostics_pods = get_namespaced_pods_by_prefix(prefix=pod, namespace=namespace, label_selector=service_label) + pods = get_namespaced_pods_by_prefix(prefix=pod, namespace=namespace, label_selector=service_label) process_pods_status( check_manager=check_manager, namespace=namespace, target=target, target_service_pod=target_service_pod, - pods=diagnostics_pods, + pods=pods, display_padding=display_padding, ) From d5b333d8b0ac2b544e35b0ca553a2ffca1f2ed5a Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 16:39:18 -0800 Subject: [PATCH 12/13] comment --- azext_edge/edge/params.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azext_edge/edge/params.py b/azext_edge/edge/params.py index 448e1f668..5e96b436c 100644 --- a/azext_edge/edge/params.py +++ b/azext_edge/edge/params.py @@ -8,6 +8,8 @@ CLI parameter definitions. """ +from azext_edge.edge.providers.edge_api.lnm import LnmResourceKinds +from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds from knack.arguments import CaseInsensitiveList from azure.cli.core.commands.parameters import get_three_state_flag, get_enum_type, tags_type @@ -111,6 +113,7 @@ def load_iotops_arguments(self, _): DataProcessorResourceKinds.DATASET.value, DataProcessorResourceKinds.PIPELINE.value, DataProcessorResourceKinds.INSTANCE.value, + OpcuaResourceKinds.ASSET_TYPE.value, ] ), help="Only run checks on specific resource kinds. Use space-separated values.", From 1e4f809bdeb7392c6326ac7efb902b0211412483 Mon Sep 17 00:00:00 2001 From: Elsie4ever <3467996@gmail.com> Date: Fri, 1 Dec 2023 16:52:21 -0800 Subject: [PATCH 13/13] fix lint --- azext_edge/edge/params.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azext_edge/edge/params.py b/azext_edge/edge/params.py index 5e96b436c..c191a534d 100644 --- a/azext_edge/edge/params.py +++ b/azext_edge/edge/params.py @@ -8,7 +8,6 @@ CLI parameter definitions. """ -from azext_edge.edge.providers.edge_api.lnm import LnmResourceKinds from azext_edge.edge.providers.edge_api.opcua import OpcuaResourceKinds from knack.arguments import CaseInsensitiveList from azure.cli.core.commands.parameters import get_three_state_flag, get_enum_type, tags_type