Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Log redirection, service side #1727

Merged
merged 5 commits into from
Mar 29, 2022
Merged

Log redirection, service side #1727

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0715fb4
Setting the service side of the log management
chkeita Mar 16, 2022
519aa59
regenerate doces
chkeita Mar 28, 2022
5e1d3b7
including job_id in the container name
chkeita Mar 28, 2022
2617b87
regenerating docs
chkeita Mar 28, 2022
5cf049c
Merge branch 'main' into 1711_service
chkeita Mar 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6,857 changes: 6,857 additions & 0 deletions docs/webhook_events
View file
Open in desktop

Large diffs are not rendered by default.

71 changes: 58 additions & 13 deletions docs/webhook_events.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -287,6 +288,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -1110,7 +1115,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
Expand Down Expand Up @@ -1722,7 +1728,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
}
Expand Down Expand Up @@ -1908,7 +1915,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -2117,6 +2125,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -2609,7 +2621,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
}
Expand Down Expand Up @@ -2795,7 +2808,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -2830,6 +2844,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -3250,7 +3268,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -3299,7 +3318,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
Expand Down Expand Up @@ -3334,6 +3354,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -3747,7 +3771,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -3782,6 +3807,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -4169,7 +4198,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -4204,6 +4234,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -4618,7 +4652,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -4653,6 +4688,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down Expand Up @@ -5197,7 +5236,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"tools",
"unique_inputs",
"unique_reports",
"regression_reports"
"regression_reports",
"logs"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -5258,7 +5298,8 @@ If webhook is set to have Event Grid message format then the payload will look a
470,
471,
472,
473
473,
474
],
"title": "ErrorCode"
},
Expand Down Expand Up @@ -6356,6 +6397,10 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Job Id",
"type": "string"
},
"logs": {
"title": "Logs",
"type": "string"
},
"pool": {
"$ref": "#/definitions/TaskPool"
},
Expand Down
29 changes: 28 additions & 1 deletion src/api-service/__app__/jobs/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,13 @@
# Licensed under the MIT License.

import azure.functions as func
from onefuzztypes.enums import ErrorCode, JobState
from onefuzztypes.enums import ContainerType, ErrorCode, JobState
from onefuzztypes.models import Error, JobConfig, JobTaskInfo
from onefuzztypes.primitives import Container
from onefuzztypes.requests import JobGet, JobSearch

from ..onefuzzlib.azure.containers import create_container
from ..onefuzzlib.azure.storage import StorageType
from ..onefuzzlib.endpoint_authorization import call_if_user
from ..onefuzzlib.jobs import Job
from ..onefuzzlib.request import not_ok, ok, parse_request
Expand Down Expand Up @@ -52,6 +55,30 @@ def post(req: func.HttpRequest) -> func.HttpResponse:

job = Job(config=request, user_info=user_info)
job.save()

# create the job logs container
log_container_sas = create_container(
stishkin marked this conversation as resolved.
Show resolved Hide resolved
Container(f"logs-{job.job_id}"),
StorageType.corpus,
metadata={"container_type": ContainerType.logs.name},
)
if not log_container_sas:
return not_ok(
Error(
code=ErrorCode.UNABLE_TO_CREATE_CONTAINER,
errors=["unable to create logs container"],
),
context="logs",
)
sep_index = log_container_sas.find("?")
if sep_index > 0:
log_container = log_container_sas[:sep_index]
else:
log_container = log_container_sas

job.config.logs = log_container
job.save()

return ok(job)


Expand Down
34 changes: 32 additions & 2 deletions src/api-service/__app__/onefuzzlib/azure/containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,31 @@ def get_container_metadata(
return cast(Dict[str, str], result)


def create_container(
def add_container_sas_url(container_url: str) -> str:
parsed = urllib.parse.urlparse(container_url)
query = urllib.parse.parse_qs(parsed.query)
if "sig" in query:
return container_url
else:
account_name = parsed.netloc.split(".")[0]
account_key = get_storage_account_name_key_by_name(account_name)
sas_token = generate_container_sas(
account_name=account_name,
container_name=parsed.path.split("/")[1],
account_key=account_key,
permission=ContainerSasPermissions(
read=True, write=True, delete=True, list=True
),
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1),
)
return f"{container_url}?{sas_token}"


def get_or_create_container_client(
container: Container,
storage_type: StorageType,
metadata: Optional[Dict[str, str]],
) -> Optional[str]:
) -> Optional[ContainerClient]:
client = find_container(container, storage_type)
if client is None:
account = choose_account(storage_type)
Expand All @@ -134,7 +154,17 @@ def create_container(
err,
)
return None
return client


def create_container(
container: Container,
storage_type: StorageType,
metadata: Optional[Dict[str, str]],
) -> Optional[str]:
client = get_or_create_container_client(container, storage_type, metadata)
if client is None:
return None
return get_container_sas_url_service(
client,
read=True,
Expand Down
23 changes: 17 additions & 6 deletions src/api-service/__app__/onefuzzlib/tasks/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,17 @@
import os
import pathlib
from typing import Dict, List, Optional
from uuid import UUID

from onefuzztypes.enums import Compare, ContainerPermission, ContainerType, TaskFeature
from onefuzztypes.models import TaskConfig, TaskDefinition, TaskUnitConfig
from onefuzztypes.models import Job, Task, TaskConfig, TaskDefinition, TaskUnitConfig
from onefuzztypes.primitives import Container

from ..azure.containers import blob_exists, container_exists, get_container_sas_url
from ..azure.containers import (
add_container_sas_url,
blob_exists,
container_exists,
get_container_sas_url,
)
from ..azure.creds import get_instance_id
from ..azure.queue import get_queue_sas
from ..azure.storage import StorageType
Expand Down Expand Up @@ -255,18 +259,25 @@ def check_config(config: TaskConfig) -> None:
raise TaskConfigError(err)


def build_task_config(
job_id: UUID, task_id: UUID, task_config: TaskConfig
) -> TaskUnitConfig:
def build_task_config(job: Job, task: Task) -> TaskUnitConfig:
job_id = job.job_id
task_id = task.task_id
task_config = task.config

if task_config.task.type not in TASK_DEFINITIONS:
raise TaskConfigError("unsupported task type: %s" % task_config.task.type.name)

if job.config.logs is None:
raise TaskConfigError(
"Missing log container: job_id %s, task_id %s", job_id, task_id
)

definition = TASK_DEFINITIONS[task_config.task.type]

config = TaskUnitConfig(
job_id=job_id,
task_id=task_id,
logs=add_container_sas_url(job.config.logs),
task_type=task_config.task.type,
instance_telemetry_key=os.environ.get("APPINSIGHTS_INSTRUMENTATIONKEY"),
microsoft_telemetry_key=os.environ.get("ONEFUZZ_TELEMETRY"),
Expand Down
7 changes: 6 additions & 1 deletion src/api-service/__app__/onefuzzlib/tasks/scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from ..azure.containers import blob_exists, get_container_sas_url
from ..azure.storage import StorageType
from ..jobs import Job
from ..workers.pools import Pool
from .config import build_task_config, get_setup_container
from .main import Task
Expand Down Expand Up @@ -116,7 +117,11 @@ def build_work_unit(task: Task) -> Optional[Tuple[BucketConfig, WorkUnit]]:

logging.info("scheduling task: %s", task.task_id)

task_config = build_task_config(task.job_id, task.task_id, task.config)
job = Job.get(task.job_id)
if not job:
raise Exception(f"invalid job_id {task.job_id} for task {task.task_id}")

task_config = build_task_config(job, task)

setup_container = get_setup_container(task.config)
setup_script = None
Expand Down
Loading