From 20cb083c80127249b9219a7867ceaeba41a5951e Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Aug 2021 16:51:03 -0400 Subject: [PATCH 1/6] enable testing of ADO work item rendering This takes an ADO config, a crash report, and renders it into Patch objects used by the ADO API. This allows us to verify the rendering is as we expect --- .../__app__/onefuzzlib/notifications/ado.py | 20 +++++-- .../onefuzzlib/notifications/common.py | 53 +++++++++++++------ 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py index cd71aa99af..79d769fa8a 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/ado.py +++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Iterator, List, Optional, Union +from typing import Iterator, List, Optional, Union, Tuple from azure.devops.connection import Connection from azure.devops.credentials import BasicAuthentication @@ -60,12 +60,19 @@ def __init__( filename: str, config: ADOTemplate, report: Report, + *, + renderer: Optional[Render] = None, ): self.config = config - self.renderer = Render(container, filename, report) + if renderer: + self.renderer = renderer + else: + self.renderer = Render(container, filename, report) + self.project = self.render(self.config.project) + + def connect(self) -> None: auth_token = get_secret_string_value(self.config.auth_token) self.client = get_ado_client(self.config.base_url, auth_token) - self.project = self.render(self.config.project) def render(self, template: str) -> str: return self.renderer.render(template) @@ -169,9 +176,8 @@ def update_existing(self, item: WorkItem) -> None: if document: self.client.update_work_item(document, item.id, project=self.project) - def create_new(self) -> None: + def render_new(self) -> Tuple[str, List[JsonPatchOperation]]: task_type = self.render(self.config.type) - document = [] if "System.Tags" not in self.config.ado_fields: document.append( @@ -187,6 +193,10 @@ def create_new(self) -> None: document.append( JsonPatchOperation(op="Add", path="/fields/%s" % field, value=value) ) + return (task_type, document) + + def create_new(self) -> None: + task_type, document = self.render_new() entry = self.client.create_work_item( document=document, project=self.project, type=task_type diff --git a/src/api-service/__app__/onefuzzlib/notifications/common.py b/src/api-service/__app__/onefuzzlib/notifications/common.py index a76d0381fd..19263942e0 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/common.py +++ b/src/api-service/__app__/onefuzzlib/notifications/common.py @@ -37,34 +37,53 @@ def fail_task(report: Report, error: Exception) -> None: class Render: - def __init__(self, container: Container, filename: str, report: Report): + def __init__( + self, + container: Container, + filename: str, + report: Report, + *, + task: Optional[Task] = None, + job: Optional[Job] = None, + target_url: Optional[str] = None, + input_url: Optional[str] = None, + report_url: Optional[str] = None, + ): self.report = report self.container = container self.filename = filename - task = Task.get(report.job_id, report.task_id) if not task: - raise ValueError(f"invalid task {report.task_id}") - job = Job.get(report.job_id) + task = Task.get(report.job_id, report.task_id) + if not task: + raise ValueError(f"invalid task {report.task_id}") if not job: - raise ValueError(f"invalid job {report.job_id}") + job = Job.get(report.job_id) + if not job: + raise ValueError(f"invalid job {report.job_id}") self.task_config = task.config self.job_config = job.config self.env = SandboxedEnvironment() - self.target_url: Optional[str] = None - setup_container = get_setup_container(task.config) - if setup_container: - self.target_url = auth_download_url( - setup_container, self.report.executable.replace("setup/", "", 1) - ) + self.target_url = target_url + if not self.target_url: + setup_container = get_setup_container(task.config) + if setup_container: + self.target_url = auth_download_url( + setup_container, self.report.executable.replace("setup/", "", 1) + ) - self.report_url = auth_download_url(container, filename) - self.input_url: Optional[str] = None - if self.report.input_blob: - self.input_url = auth_download_url( - self.report.input_blob.container, self.report.input_blob.name - ) + if report_url: + self.report_url = report_url + else: + self.report_url = auth_download_url(container, filename) + + self.input_url = input_url + if not self.input_url: + if self.report.input_blob: + self.input_url = auth_download_url( + self.report.input_blob.container, self.report.input_blob.name + ) def render(self, template: str) -> str: return self.env.from_string(template).render( From 364751db82a157f8c21c986c4da4415d1c4cc903 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Aug 2021 16:57:11 -0400 Subject: [PATCH 2/6] test new ADO work item rendering --- src/api-service/tests/data/ado-config.json | 32 ++++++++ src/api-service/tests/data/ado-rendered.json | 42 ++++++++++ .../tests/data/crash-report-with-html.json | 23 ++++++ src/api-service/tests/test_ado_render.py | 81 +++++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 src/api-service/tests/data/ado-config.json create mode 100644 src/api-service/tests/data/ado-rendered.json create mode 100644 src/api-service/tests/data/crash-report-with-html.json create mode 100755 src/api-service/tests/test_ado_render.py diff --git a/src/api-service/tests/data/ado-config.json b/src/api-service/tests/data/ado-config.json new file mode 100644 index 0000000000..4dc5bb2cff --- /dev/null +++ b/src/api-service/tests/data/ado-config.json @@ -0,0 +1,32 @@ +{ + "base_url": "https://dev.azure.com/contoso", + "auth_token": "DO NOT PUT YOUR PAT HERE", + "type": "Bug", + "project": "Contoso", + "unique_fields": [ + "Microsoft.VSTS.Build.FoundIn" + ], + "comment": "my comment", + "ado_fields": { + "System.AssignedTo": "example@contoso.com", + "System.Tags": "OneFuzz; OneFuzz-Pipeline; example@example.com", + "System.AreaPath": "MY\\AREA Path\\Here", + "System.IterationPath": "MY\\ITERATION", + "Microsoft.VSTS.Scheduling.StoryPoints": "1", + "Microsoft.VSTS.Build.FoundIn": "{{ report.input_sha256 }}", + "System.Title": "[OneFuzz] - {{ report.crash_site }}", + "Microsoft.VSTS.TCM.ReproSteps": "This is the call stack. You may wish to confirm: " + }, + "on_duplicate": { + "comment": "DEDUPED!", + "set_state": { + "Resolved": "Active" + }, + "ado_fields": { + "System.IterationPath": "MY\\ITERATION" + }, + "increment": [ + "Microsoft.VSTS.Scheduling.StoryPoints" + ] + } +} diff --git a/src/api-service/tests/data/ado-rendered.json b/src/api-service/tests/data/ado-rendered.json new file mode 100644 index 0000000000..431278b2bf --- /dev/null +++ b/src/api-service/tests/data/ado-rendered.json @@ -0,0 +1,42 @@ +[ + { + "op": "Add", + "path": "/fields/System.AssignedTo", + "value": "example@contoso.com" + }, + { + "op": "Add", + "path": "/fields/System.Tags", + "value": "OneFuzz; OneFuzz-Pipeline; example@example.com;Onefuzz" + }, + { + "op": "Add", + "path": "/fields/System.AreaPath", + "value": "MY\\AREA Path\\Here" + }, + { + "op": "Add", + "path": "/fields/System.IterationPath", + "value": "MY\\ITERATION" + }, + { + "op": "Add", + "path": "/fields/Microsoft.VSTS.Scheduling.StoryPoints", + "value": "1" + }, + { + "op": "Add", + "path": "/fields/Microsoft.VSTS.Build.FoundIn", + "value": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "op": "Add", + "path": "/fields/System.Title", + "value": "[OneFuzz] - callfunc" + }, + { + "op": "Add", + "path": "/fields/Microsoft.VSTS.TCM.ReproSteps", + "value": "This is the call stack. You may wish to confirm: " + } +] diff --git a/src/api-service/tests/data/crash-report-with-html.json b/src/api-service/tests/data/crash-report-with-html.json new file mode 100644 index 0000000000..f7fb0609fc --- /dev/null +++ b/src/api-service/tests/data/crash-report-with-html.json @@ -0,0 +1,23 @@ +{ + "call_stack": [ + "test.exe.exe+0x2", + "test.exe.exe+0x9", + "test.exe.exe+0x3", + "test.exe.exe+0xd", + "test.exe.exe+0x8", + "test.exe.exe+0x8", + "test.exe.exe+0x2" + ], + "call_stack_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "crash_site": "callfunc", + "crash_type": "customtype", + "executable": "test2.exe", + "input_blob": { + "account": "storageaccount1", + "container": "storagecontainer1", + "name": "%23A_B.exe.exe%2B0x0_a_b_c_0x5b1e7_a_515c01e.tar" + }, + "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "job_id": "581e0ba6-9757-47e5-a077-dba21e604961", + "task_id": "4a949007-b59f-4a47-8418-8a2c94b4aef1" +} \ No newline at end of file diff --git a/src/api-service/tests/test_ado_render.py b/src/api-service/tests/test_ado_render.py new file mode 100755 index 0000000000..cce0ca8e00 --- /dev/null +++ b/src/api-service/tests/test_ado_render.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +import os +import json +import unittest +from pathlib import Path + +from onefuzztypes.models import Report, ADOTemplate, TaskConfig, JobConfig, TaskDetails +from onefuzztypes.primitives import Container +from onefuzztypes.enums import OS, TaskType + +from __app__.onefuzzlib.notifications.ado import ADO +from __app__.onefuzzlib.notifications.common import Render +from __app__.onefuzzlib.tasks.main import Task +from __app__.onefuzzlib.jobs import Job + + +os.environ["ONEFUZZ_INSTANCE_NAME"] = "contoso-test" + + +class TestReportParse(unittest.TestCase): + def test_sample(self) -> None: + expected_path = Path(__file__).parent / "data" / "ado-rendered.json" + with open(expected_path , "r") as handle: + expected_document = json.load(handle) + + report_path = Path(__file__).parent / "data" / "crash-report-with-html.json" + with open(report_path, "r") as handle: + report_raw = json.load(handle) + + ado_path = Path(__file__).parent / "data" / "ado-config.json" + with open(ado_path, "r") as handle: + ado_raw = json.load(handle) + + print(ado_raw) + + report = Report.parse_obj(report_raw) + config = ADOTemplate.parse_obj(ado_raw) + + container = Container("containername") + filename = "test.json" + + job = Job( + config=JobConfig(project="project", name="name", build="build", duration=1) + ) + task = Task( + config=TaskConfig( + job_id=job.job_id, + tags={}, + containers=[], + task=TaskDetails(type=TaskType.libfuzzer_fuzz, duration=1), + ), + job_id=job.job_id, + os=OS.linux, + ) + renderer = Render( + container, + filename, + report, + task=task, + job=job, + target_url="https://contoso.com/1", + input_url="https://contoso.com/2", + report_url="https://contoso.com/3", + ) + + ado = ADO(container, filename, config, report, renderer=renderer) + work_item_type, document = ado.render_new() + self.assertEqual(work_item_type, "Bug") + + as_obj = [x.as_dict() for x in document] + + self.assertEqual(as_obj, expected_document) + + +if __name__ == "__main__": + unittest.main() From f3d4450936f7c2777d6bea33502e52647a461ea6 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Aug 2021 17:23:46 -0400 Subject: [PATCH 3/6] lint --- .../__app__/onefuzzlib/notifications/ado.py | 2 +- src/api-service/tests/test_ado_render.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py index 79d769fa8a..79b5153e4c 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/ado.py +++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Iterator, List, Optional, Union, Tuple +from typing import Iterator, List, Optional, Tuple, Union from azure.devops.connection import Connection from azure.devops.credentials import BasicAuthentication diff --git a/src/api-service/tests/test_ado_render.py b/src/api-service/tests/test_ado_render.py index cce0ca8e00..6e546ccb38 100755 --- a/src/api-service/tests/test_ado_render.py +++ b/src/api-service/tests/test_ado_render.py @@ -4,20 +4,19 @@ # Licensed under the MIT License. -import os import json +import os import unittest from pathlib import Path -from onefuzztypes.models import Report, ADOTemplate, TaskConfig, JobConfig, TaskDetails -from onefuzztypes.primitives import Container from onefuzztypes.enums import OS, TaskType +from onefuzztypes.models import ADOTemplate, JobConfig, Report, TaskConfig, TaskDetails +from onefuzztypes.primitives import Container +from __app__.onefuzzlib.jobs import Job from __app__.onefuzzlib.notifications.ado import ADO from __app__.onefuzzlib.notifications.common import Render from __app__.onefuzzlib.tasks.main import Task -from __app__.onefuzzlib.jobs import Job - os.environ["ONEFUZZ_INSTANCE_NAME"] = "contoso-test" @@ -25,9 +24,9 @@ class TestReportParse(unittest.TestCase): def test_sample(self) -> None: expected_path = Path(__file__).parent / "data" / "ado-rendered.json" - with open(expected_path , "r") as handle: + with open(expected_path, "r") as handle: expected_document = json.load(handle) - + report_path = Path(__file__).parent / "data" / "crash-report-with-html.json" with open(report_path, "r") as handle: report_raw = json.load(handle) From 72397dc75a242c058e179bad7cecc1ed0590d908 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Aug 2021 17:53:18 -0400 Subject: [PATCH 4/6] address feedback --- src/api-service/tests/test_ado_render.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/api-service/tests/test_ado_render.py b/src/api-service/tests/test_ado_render.py index 6e546ccb38..3165c3e340 100755 --- a/src/api-service/tests/test_ado_render.py +++ b/src/api-service/tests/test_ado_render.py @@ -8,6 +8,7 @@ import os import unittest from pathlib import Path +from unittest.mock import patch from onefuzztypes.enums import OS, TaskType from onefuzztypes.models import ADOTemplate, JobConfig, Report, TaskConfig, TaskDetails @@ -18,10 +19,17 @@ from __app__.onefuzzlib.notifications.common import Render from __app__.onefuzzlib.tasks.main import Task -os.environ["ONEFUZZ_INSTANCE_NAME"] = "contoso-test" - class TestReportParse(unittest.TestCase): + def setUp(self): + self.env_patch = patch.dict( + "os.environ", {"ONEFUZZ_INSTANCE_NAME": "contoso-test"} + ) + self.env_patch.start() + + def tearDown(self): + self.env_patch.stop() + def test_sample(self) -> None: expected_path = Path(__file__).parent / "data" / "ado-rendered.json" with open(expected_path, "r") as handle: @@ -35,8 +43,6 @@ def test_sample(self) -> None: with open(ado_path, "r") as handle: ado_raw = json.load(handle) - print(ado_raw) - report = Report.parse_obj(report_raw) config = ADOTemplate.parse_obj(ado_raw) @@ -56,6 +62,7 @@ def test_sample(self) -> None: job_id=job.job_id, os=OS.linux, ) + renderer = Render( container, filename, From 76e06dadf18ae1ecad261a78a0db57db0176b87d Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Wed, 11 Aug 2021 18:46:00 -0400 Subject: [PATCH 5/6] lint --- src/api-service/tests/test_ado_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api-service/tests/test_ado_render.py b/src/api-service/tests/test_ado_render.py index 3165c3e340..67048144c3 100755 --- a/src/api-service/tests/test_ado_render.py +++ b/src/api-service/tests/test_ado_render.py @@ -5,7 +5,6 @@ import json -import os import unittest from pathlib import Path from unittest.mock import patch From 69e16ca79697564dc5b78bab3b2421e24302e8e3 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Thu, 12 Aug 2021 09:20:38 -0400 Subject: [PATCH 6/6] add return typing --- src/api-service/tests/test_ado_render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api-service/tests/test_ado_render.py b/src/api-service/tests/test_ado_render.py index 67048144c3..cd569bb03e 100755 --- a/src/api-service/tests/test_ado_render.py +++ b/src/api-service/tests/test_ado_render.py @@ -20,13 +20,13 @@ class TestReportParse(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.env_patch = patch.dict( "os.environ", {"ONEFUZZ_INSTANCE_NAME": "contoso-test"} ) self.env_patch.start() - def tearDown(self): + def tearDown(self) -> None: self.env_patch.stop() def test_sample(self) -> None: