diff --git a/docs/webhook_events.md b/docs/webhook_events.md index cdb7b631db..85d8889607 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -330,9 +330,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -2239,9 +2237,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -2969,9 +2965,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -3490,9 +3484,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -3954,9 +3946,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -4392,9 +4382,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -4857,9 +4845,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" @@ -6606,9 +6592,7 @@ If webhook is set to have Event Grid message format then the payload will look a }, "required": [ "job_id", - "task", - "containers", - "tags" + "task" ], "title": "TaskConfig", "type": "object" diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 4fc2753a2a..d6a0dcd099 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -529,10 +529,11 @@ def _download_tasks( ) -> None: to_download: Dict[str, str] = {} for task in tasks: - for container in task.config.containers: - info = self.onefuzz.containers.get(container.name) - name = os.path.join(container.type.name, container.name) - to_download[name] = info.sas_url + if task.config.containers is not None: + for container in task.config.containers: + info = self.onefuzz.containers.get(container.name) + name = os.path.join(container.type.name, container.name) + to_download[name] = info.sas_url if output is None: output = primitives.Directory(os.getcwd()) @@ -1099,9 +1100,14 @@ def list( containers = set() tasks = self.onefuzz.tasks.list(job_id=job_id, state=[]) for task in tasks: - containers.update( - set(x.name for x in task.config.containers if x.type == container_type) - ) + if task.config.containers is not None: + containers.update( + set( + x.name + for x in task.config.containers + if x.type == container_type + ) + ) results: Dict[str, List[str]] = {} for container in containers: @@ -1133,23 +1139,24 @@ def delete( containers = set() to_delete = set() for task in self.onefuzz.jobs.tasks.list(job_id=job.job_id): - for container in task.config.containers: - containers.add(container.name) - if container.type not in SAFE_TO_REMOVE: - continue - elif not only_job_specific: - to_delete.add(container.name) - elif only_job_specific and ( - self.onefuzz.utils.build_container_name( - container_type=container.type, - project=job.config.project, - name=job.config.name, - build=job.config.build, - platform=task.os, - ) - == container.name - ): - to_delete.add(container.name) + if task.config.containers is not None: + for container in task.config.containers: + containers.add(container.name) + if container.type not in SAFE_TO_REMOVE: + continue + elif not only_job_specific: + to_delete.add(container.name) + elif only_job_specific and ( + self.onefuzz.utils.build_container_name( + container_type=container.type, + project=job.config.project, + name=job.config.name, + build=job.config.build, + platform=task.os, + ) + == container.name + ): + to_delete.add(container.name) to_keep = containers - to_delete for container_name in to_keep: @@ -1713,6 +1720,17 @@ def update(self, config: models.InstanceConfig) -> models.InstanceConfig: ) +class ValidateScriban(Endpoint): + """Interact with Validate Scriban""" + + endpoint = "ValidateScriban" + + def post( + self, req: requests.TemplateValidationPost + ) -> responses.TemplateValidationResponse: + return self._req_model("POST", responses.TemplateValidationResponse, data=req) + + class Command: def __init__(self, onefuzz: "Onefuzz", logger: logging.Logger): self.onefuzz = onefuzz @@ -1803,6 +1821,7 @@ def __init__( self.webhooks = Webhooks(self) self.tools = Tools(self) self.instance_config = InstanceConfigCmd(self) + self.validate_scriban = ValidateScriban(self) if self._backend.is_feature_enabled(PreviewFeature.job_templates.name): self.job_templates = JobTemplates(self) diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index f62c0d3ba2..aefefc3ea0 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -18,9 +18,11 @@ from azure.applicationinsights.models import QueryBody from azure.identity import AzureCliCredential from azure.storage.blob import ContainerClient +from onefuzztypes import models, requests from onefuzztypes.enums import ContainerType, TaskType from onefuzztypes.models import BlobRef, Job, NodeAssignment, Report, Task, TaskConfig from onefuzztypes.primitives import Container, Directory, PoolName +from onefuzztypes.responses import TemplateValidationResponse from onefuzz.api import UUID_EXPANSION, Command, Onefuzz @@ -721,9 +723,10 @@ class DebugNotification(Command): def _get_container( self, task: Task, container_type: ContainerType ) -> Optional[Container]: - for container in task.config.containers: - if container.type == container_type: - return container.name + if task.config.containers is not None: + for container in task.config.containers: + if container.type == container_type: + return container.name return None def _get_storage_account(self, container_name: Container) -> str: @@ -731,6 +734,13 @@ def _get_storage_account(self, container_name: Container) -> str: _, netloc, _, _, _, _ = urlparse(sas_url) return netloc.split(".")[0] + def template( + self, template: str, context: Optional[models.TemplateRenderContext] + ) -> TemplateValidationResponse: + """Validate scriban rendering of notification config""" + req = requests.TemplateValidationPost(template=template, context=context) + return self.onefuzz.validate_scriban.post(req) + def job( self, job_id: UUID_EXPANSION, diff --git a/src/cli/onefuzz/status/cmd.py b/src/cli/onefuzz/status/cmd.py index e986029101..bf1dd2f2bd 100644 --- a/src/cli/onefuzz/status/cmd.py +++ b/src/cli/onefuzz/status/cmd.py @@ -115,10 +115,11 @@ def job(self, job_id: UUID_EXPANSION) -> None: containers: DefaultDict[ContainerType, Set[Container]] = defaultdict(set) for task in tasks: - for container in task.config.containers: - if container.type not in containers: - containers[container.type] = set() - containers[container.type].add(container.name) + if task.config.containers is not None: + for container in task.config.containers: + if container.type not in containers: + containers[container.type] = set() + containers[container.type].add(container.name) print("\ncontainers:") for container_type in containers: diff --git a/src/cli/onefuzz/status/top.py b/src/cli/onefuzz/status/top.py index e344dce823..0ee539213f 100644 --- a/src/cli/onefuzz/status/top.py +++ b/src/cli/onefuzz/status/top.py @@ -68,8 +68,9 @@ def setup(self) -> Stream: for task in self.onefuzz.tasks.list(job_id=job.job_id): self.cache.add_task(task) - for container in task.config.containers: - self.add_container(container.name) + if task.config.containers is not None: + for container in task.config.containers: + self.add_container(container.name) nodes = self.onefuzz.nodes.list() for node in nodes: diff --git a/src/cli/onefuzz/template.py b/src/cli/onefuzz/template.py index 3cfc5bcf22..21fabdd271 100644 --- a/src/cli/onefuzz/template.py +++ b/src/cli/onefuzz/template.py @@ -63,7 +63,11 @@ def stop( self.onefuzz.tasks.delete(task.task_id) if stop_notifications: - container_names = [x.name for x in task.config.containers] + container_names = ( + [x.name for x in task.config.containers] + if task.config.containers is not None + else [] + ) notifications = self.onefuzz.notifications.list( container=container_names ) diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 60fa68cc12..fd5493c2e7 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Generic, List, Optional, TypeVar, Union from uuid import UUID, uuid4 -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import AnyHttpUrl, BaseModel, Field, root_validator, validator from pydantic.dataclasses import dataclass from ._monkeypatch import _check_hotfix @@ -193,8 +193,8 @@ class TaskConfig(BaseModel): task: TaskDetails vm: Optional[TaskVm] pool: Optional[TaskPool] - containers: List[TaskContainers] - tags: Dict[str, str] + containers: Optional[List[TaskContainers]] + tags: Optional[Dict[str, str]] debug: Optional[List[TaskDebugFlag]] colocate: Optional[bool] @@ -870,6 +870,18 @@ class ApiAccessRule(BaseModel): allowed_groups: List[UUID] +class TemplateRenderContext(BaseModel): + report: Report + task: TaskConfig + job: JobConfig + report_url: AnyHttpUrl + input_url: AnyHttpUrl + target_url: AnyHttpUrl + report_container: Container + report_filename: str + repro_cmd: str + + Endpoint = str # json dumps doesn't support UUID as dictionary key PrincipalID = str diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 1bdf37dff3..b2ceba3a44 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -20,7 +20,12 @@ TaskState, ) from .events import EventType -from .models import AutoScaleConfig, InstanceConfig, NotificationConfig +from .models import ( + AutoScaleConfig, + InstanceConfig, + NotificationConfig, + TemplateRenderContext, +) from .primitives import Container, PoolName, Region from .webhooks import WebhookMessageFormat @@ -252,4 +257,9 @@ class InstanceConfigUpdate(BaseModel): config: InstanceConfig +class TemplateValidationPost(BaseModel): + template: str + context: Optional[TemplateRenderContext] + + _check_hotfix() diff --git a/src/pytypes/onefuzztypes/responses.py b/src/pytypes/onefuzztypes/responses.py index 7926deb2dc..b59712c281 100644 --- a/src/pytypes/onefuzztypes/responses.py +++ b/src/pytypes/onefuzztypes/responses.py @@ -9,7 +9,7 @@ from pydantic import BaseModel from .enums import VmState -from .models import Forward, NodeCommandEnvelope +from .models import Forward, NodeCommandEnvelope, TemplateRenderContext from .primitives import Region @@ -84,3 +84,8 @@ class PendingNodeCommand(BaseResponse): class CanSchedule(BaseResponse): allowed: bool work_stopped: bool + + +class TemplateValidationResponse(BaseResponse): + rendered_template: str + available_context: TemplateRenderContext