From 8a7e129cc22ff395f0a4a018f1a970d9580ba0e2 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 11:32:55 -0500 Subject: [PATCH 1/7] initial POC for application insights debug --- src/cli/onefuzz/debug.py | 85 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index 075ab50741..8331027577 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -3,15 +3,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import json import logging import os import shutil import subprocess # nosec import tempfile -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse from uuid import UUID +from azure.applicationinsights import ApplicationInsightsDataClient +from azure.applicationinsights.models import QueryBody +from azure.common.client_factory import get_azure_cli_credentials from onefuzztypes.enums import ContainerType, TaskType from onefuzztypes.models import BlobRef, NodeAssignment, Report, Task from onefuzztypes.primitives import Directory @@ -262,6 +266,84 @@ def download_files(self, job_id: UUID_EXPANSION, output: Directory) -> None: subprocess.check_output([azcopy, "sync", to_download[name], outdir]) +class DebugLog(Command): + def _convert(self, raw_data: Any) -> Dict[str, List[Dict[str, Any]]]: + results = {} + for table in raw_data.tables: + result = [] + for row in table.rows: + converted = { + table.columns[x].name: y + for (x, y) in enumerate(row) + if y not in [None, ""] + } + if "customDimensions" in converted: + converted["customDimensions"] = json.loads( + converted["customDimensions"] + ) + result.append(converted) + results[table.name] = result + return results + + def query( + self, log_query: str, *, timespan: str = "PT24H", raw: bool = False + ) -> Any: + """ + Perform an Application Insights query + + :param str log_query: Query to send to Application Insights + :param str timespan: ISO 8601 duration format + :param bool raw: Do not simplify the data result + """ + creds, _ = get_azure_cli_credentials( + resource="https://api.applicationinsights.io" + ) + client = ApplicationInsightsDataClient(creds) + raw_data = client.query.execute( + os.environ["APP_ID"], body=QueryBody(query=log_query, timespan=timespan) + ) + if "error" in raw_data.additional_properties: + raise Exception( + "Error performing query: %s" % raw_data.additional_properties["error"] + ) + if raw: + return raw_data + return self._convert(raw_data) + + def keyword( + self, + value: str, + *, + timespan: str = "PT24H", + limit: Optional[int] = None, + raw: bool = False, + ) -> Any: + """ + Perform an Application Insights keyword query akin to "Transaction Search" + + :param str value: Keyword to query Application Insights + :param str timespan: ISO 8601 duration format + :param int limit: Limit the number of records returned + :param bool raw: Do not simplify the data result + """ + components = ["union isfuzzy=true exceptions, traces, customEvents"] + + value = value.strip() + keywords = ['* has "%s"' % (x.replace('"', '\\"')) for x in value.split(" ")] + if keywords: + components.append("where " + " and ".join(keywords)) + + components.append("order by timestamp desc") + + if limit is not None: + components.append(f"take {limit}") + + log_query = " | ".join(components) + self.logger.debug("query: %s", log_query) + + return self.query(log_query, timespan=timespan) + + class DebugNotification(Command): """ Debug notification integrations """ @@ -366,3 +448,4 @@ def __init__(self, onefuzz: Any, logger: logging.Logger): self.job = DebugJob(onefuzz, logger) self.notification = DebugNotification(onefuzz, logger) self.task = DebugTask(onefuzz, logger) + self.logs = DebugLog(onefuzz, logger) From e9a3f843858d551d43ad270f2bf34ba49fec5f2b Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 11:35:19 -0500 Subject: [PATCH 2/7] include azure-applicationinsights prereq --- src/cli/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/requirements.txt b/src/cli/requirements.txt index 33152acc2d..27f1ad4392 100644 --- a/src/cli/requirements.txt +++ b/src/cli/requirements.txt @@ -9,6 +9,7 @@ dataclasses~=0.6 pydantic~=1.6.1 --no-binary=pydantic memoization~=0.3.1 azure-storage-blob~=12.3 +azure-applicationinsights==0.1.0 tenacity==6.2.0 docstring_parser==0.7.3 # onefuzztypes version is set during build From 40f30539e5cdbe53418d534c68e0500d695bf3d1 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 11:48:15 -0500 Subject: [PATCH 3/7] adds later versions of azure-common --- src/cli/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/requirements.txt b/src/cli/requirements.txt index 27f1ad4392..2db134ec80 100644 --- a/src/cli/requirements.txt +++ b/src/cli/requirements.txt @@ -10,6 +10,7 @@ pydantic~=1.6.1 --no-binary=pydantic memoization~=0.3.1 azure-storage-blob~=12.3 azure-applicationinsights==0.1.0 +azure-common~=1.1.25 tenacity==6.2.0 docstring_parser==0.7.3 # onefuzztypes version is set during build From bfd38710d2e971a1cc7e84b55b53fd6f5616e1c3 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 11:50:21 -0500 Subject: [PATCH 4/7] specify adal --- src/cli/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/requirements.txt b/src/cli/requirements.txt index 2db134ec80..60eb7eff17 100644 --- a/src/cli/requirements.txt +++ b/src/cli/requirements.txt @@ -10,7 +10,7 @@ pydantic~=1.6.1 --no-binary=pydantic memoization~=0.3.1 azure-storage-blob~=12.3 azure-applicationinsights==0.1.0 -azure-common~=1.1.25 +adal~=1.2.5 tenacity==6.2.0 docstring_parser==0.7.3 # onefuzztypes version is set during build From b9bc835c6444827f5388adba2f134ad6285c6972 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 11:56:52 -0500 Subject: [PATCH 5/7] add msrestazure prereq --- src/cli/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/requirements.txt b/src/cli/requirements.txt index 60eb7eff17..2874713068 100644 --- a/src/cli/requirements.txt +++ b/src/cli/requirements.txt @@ -8,6 +8,7 @@ asciimatics~=1.11.0 dataclasses~=0.6 pydantic~=1.6.1 --no-binary=pydantic memoization~=0.3.1 +msrestazure==0.6.4 azure-storage-blob~=12.3 azure-applicationinsights==0.1.0 adal~=1.2.5 From fe6e9b21ad0490176db25d987c98696a31554740 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 10 Nov 2020 15:09:50 -0500 Subject: [PATCH 6/7] add insights_appid --- src/api-service/__app__/info/__init__.py | 2 ++ src/api-service/__app__/onefuzzlib/azure/creds.py | 5 +++++ src/cli/onefuzz/debug.py | 6 +++++- src/deployment/azuredeploy.json | 4 ++++ src/pytypes/onefuzztypes/responses.py | 1 + 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/api-service/__app__/info/__init__.py b/src/api-service/__app__/info/__init__.py index 48527c9594..3a7b9ccc38 100644 --- a/src/api-service/__app__/info/__init__.py +++ b/src/api-service/__app__/info/__init__.py @@ -9,6 +9,7 @@ from ..onefuzzlib.azure.creds import ( get_base_region, get_base_resource_group, + get_insights_appid, get_instance_id, get_subscription, ) @@ -24,5 +25,6 @@ def main(req: func.HttpRequest) -> func.HttpResponse: subscription=get_subscription(), versions=versions(), instance_id=get_instance_id(), + insights_appid=get_insights_appid(), ) ) diff --git a/src/api-service/__app__/onefuzzlib/azure/creds.py b/src/api-service/__app__/onefuzzlib/azure/creds.py index fdb80d91e1..f72a247793 100644 --- a/src/api-service/__app__/onefuzzlib/azure/creds.py +++ b/src/api-service/__app__/onefuzzlib/azure/creds.py @@ -82,6 +82,11 @@ def get_subscription() -> Any: # should be str return parse_resource_id(os.environ["ONEFUZZ_DATA_STORAGE"])["subscription"] +@cached +def get_insights_appid() -> str: + return os.environ["APPINSIGHTS_APPID"] + + @cached def get_fuzz_storage() -> str: return os.environ["ONEFUZZ_DATA_STORAGE"] diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index 8331027577..4ba6c3d885 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -299,8 +299,12 @@ def query( resource="https://api.applicationinsights.io" ) client = ApplicationInsightsDataClient(creds) + + app_id = self.onefuzz.info.get().insights_appid + if app_id is None: + raise Exception("instance does not have an insights_appid") raw_data = client.query.execute( - os.environ["APP_ID"], body=QueryBody(query=log_query, timespan=timespan) + app_id, body=QueryBody(query=log_query, timespan=timespan) ) if "error" in raw_data.additional_properties: raise Exception( diff --git a/src/deployment/azuredeploy.json b/src/deployment/azuredeploy.json index 0f91bfeb2c..91a5a43501 100644 --- a/src/deployment/azuredeploy.json +++ b/src/deployment/azuredeploy.json @@ -161,6 +161,10 @@ "name": "APPINSIGHTS_INSTRUMENTATIONKEY", "value": "[reference(resourceId('microsoft.insights/components/', parameters('name')), '2015-05-01').InstrumentationKey]" }, + { + "name": "APPINSIGHTS_APPID", + "value": "[reference(resourceId('microsoft.insights/components/', parameters('name')), '2015-05-01').AppId]" + }, { "name": "ONEFUZZ_TELEMETRY", "value": "[variables('telemetry')]" diff --git a/src/pytypes/onefuzztypes/responses.py b/src/pytypes/onefuzztypes/responses.py index 069182df9c..dfc50cc206 100644 --- a/src/pytypes/onefuzztypes/responses.py +++ b/src/pytypes/onefuzztypes/responses.py @@ -37,6 +37,7 @@ class Info(BaseResponse): subscription: str versions: Dict[str, Version] instance_id: Optional[UUID] + insights_appid: Optional[str] class ContainerInfoBase(BaseResponse): From 07805bdf52ad18b67ee901435a02e5582908d632 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Nov 2020 05:55:32 -0500 Subject: [PATCH 7/7] document that we're building KQL --- src/cli/onefuzz/debug.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index 4ba6c3d885..f4235b3eee 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -291,6 +291,9 @@ def query( """ Perform an Application Insights query + Queries should be well formed Kusto Queries. + Ref https://docs.microsoft.com/en-us/azure/data-explorer/kql-quick-reference + :param str log_query: Query to send to Application Insights :param str timespan: ISO 8601 duration format :param bool raw: Do not simplify the data result @@ -330,6 +333,9 @@ def keyword( :param int limit: Limit the number of records returned :param bool raw: Do not simplify the data result """ + + # See https://docs.microsoft.com/en-us/azure/data-explorer/kql-quick-reference + components = ["union isfuzzy=true exceptions, traces, customEvents"] value = value.strip()