Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Expose coverage/exec_sec for libfuzzer targets via CLI (#325)
Browse files Browse the repository at this point in the history
Adds debug subcommands to the SDK/CLI that simplify querying Application Insights for libfuzzer telemetry.  

Querying for the latest execs_sec for a job, by job_id fragment.
```
$ onefuzz debug job libfuzzer_execs_sec 88 --limit 1
[
    {
        "execs_sec": "191035",
        "machine_id": "b2dbe720-4fd8-4342-957a-6cb0979d2187",
        "timestamp": "2020-11-18T00:08:53.98Z",
        "worker_id": "0"
    }
]
```

Querying for the latest coverage for a job, by job_id fragment.
```
$ onefuzz debug job libfuzzer_coverage 88 --limit 1
[
    {
        "covered": "10",
        "features": "21",
        "rate": "0.47619047619047616",
        "timestamp": "2020-11-18T00:09:40.793Z"
    }
]
```
  • Loading branch information
bmc-msft authored Nov 19, 2020
1 parent bb2b18a commit 31a661f
Showing 1 changed file with 137 additions and 5 deletions.
142 changes: 137 additions & 5 deletions src/cli/onefuzz/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
ZERO_SHA256 = "0" * len(EMPTY_SHA256)
DAY_TIMESPAN = "PT24H"


class DebugRepro(Command):
Expand Down Expand Up @@ -213,6 +214,42 @@ def rdp(
scaleset_id, node_id = self._get_node(task_id, node_id)
return self.onefuzz.debug.scalesets.rdp(scaleset_id, node_id, duration=duration)

def libfuzzer_coverage(
self,
task_id: UUID_EXPANSION,
timespan: str = DAY_TIMESPAN,
limit: Optional[int] = None,
) -> Any:
"""
Get the coverage for the specified task
:param task_id value: Task ID
:param str timespan: ISO 8601 duration format
:param int limit: Limit the number of records returned
"""
task = self.onefuzz.tasks.get(task_id)
query = f"where customDimensions.task_id == '{task.task_id}'"
return self.onefuzz.debug.logs._query_libfuzzer_coverage(query, timespan, limit)

def libfuzzer_execs_sec(
self,
task_id: UUID_EXPANSION,
timespan: str = DAY_TIMESPAN,
limit: Optional[int] = None,
) -> Any:
"""
Get the executions per second for the specified task
:param task_id value: Task ID
:param str timespan: ISO 8601 duration format
:param int limit: Limit the number of records returned
"""
task = self.onefuzz.tasks.get(task_id)
query = f"where customDimensions.task_id == '{task.task_id}'"
return self.onefuzz.debug.logs._query_libfuzzer_execs_sec(
query, timespan, limit
)


class DebugJobTask(Command):
""" Debug a task for a specific job """
Expand Down Expand Up @@ -258,6 +295,42 @@ def __init__(self, onefuzz: Any, logger: logging.Logger):
super().__init__(onefuzz, logger)
self.task = DebugJobTask(onefuzz, logger)

def libfuzzer_coverage(
self,
job_id: UUID_EXPANSION,
timespan: str = DAY_TIMESPAN,
limit: Optional[int] = None,
) -> Any:
"""
Get the coverage for the specified job
:param job_id value: Job ID
:param str timespan: ISO 8601 duration format
:param int limit: Limit the number of records returned
"""
job = self.onefuzz.jobs.get(job_id)
query = f"where customDimensions.job_id == '{job.job_id}'"
return self.onefuzz.debug.logs._query_libfuzzer_coverage(query, timespan, limit)

def libfuzzer_execs_sec(
self,
job_id: UUID_EXPANSION,
timespan: str = DAY_TIMESPAN,
limit: Optional[int] = None,
) -> Any:
"""
Get the executions per second for the specified job
:param job_id value: Job ID
:param str timespan: ISO 8601 duration format
:param int limit: Limit the number of records returned
"""
job = self.onefuzz.jobs.get(job_id)
query = f"where customDimensions.job_id == '{job.job_id}'"
return self.onefuzz.debug.logs._query_libfuzzer_execs_sec(
query, timespan, limit
)

def download_files(self, job_id: UUID_EXPANSION, output: Directory) -> None:
""" Download the containers by container type for each task in the specified job """

Expand Down Expand Up @@ -306,7 +379,7 @@ def _convert(self, raw_data: Any) -> Dict[str, List[Dict[str, Any]]]:
return results

def query(
self, log_query: str, *, timespan: str = "PT24H", raw: bool = False
self, log_query: str, *, timespan: str = DAY_TIMESPAN, raw: bool = False
) -> Any:
"""
Perform an Application Insights query
Expand All @@ -326,6 +399,7 @@ def query(
app_id = self.onefuzz.info.get().insights_appid
if app_id is None:
raise Exception("instance does not have an insights_appid")
self.logger.debug("query: %s", log_query)
raw_data = client.query.execute(
app_id, body=QueryBody(query=log_query, timespan=timespan)
)
Expand All @@ -337,11 +411,17 @@ def query(
return raw_data
return self._convert(raw_data)

def _query_parts(
self, parts: List[str], timespan: str, *, raw: bool = False
) -> Any:
log_query = " | ".join(parts)
return self.query(log_query, timespan=timespan, raw=raw)

def keyword(
self,
value: str,
*,
timespan: str = "PT24H",
timespan: str = DAY_TIMESPAN,
limit: Optional[int] = None,
raw: bool = False,
) -> Any:
Expand All @@ -368,10 +448,62 @@ def keyword(
if limit is not None:
components.append(f"take {limit}")

log_query = " | ".join(components)
self.logger.debug("query: %s", log_query)
return self._query_parts(components, timespan=timespan, raw=raw)

return self.query(log_query, timespan=timespan)
def _query_libfuzzer_coverage(
self, query: str, timespan: str, limit: Optional[int] = None
) -> Any:
project_fields = [
"rate=customDimensions.rate",
"covered=customDimensions.covered",
"features=customDimensions.features",
"timestamp",
]

query_parts = [
"customEvents",
"where name == 'coverage_data'",
query,
"order by timestamp desc",
f"project {','.join(project_fields)}",
]

if limit:
query_parts.append(f"take {limit}")

results = self.onefuzz.debug.logs._query_parts(query_parts, timespan=timespan)
if "PrimaryResult" in results:
return results["PrimaryResult"]
return results

def _query_libfuzzer_execs_sec(
self,
query: str,
timespan: str,
limit: Optional[int] = None,
) -> Any:
project_fields = [
"machine_id=customDimensions.machine_id",
"worker_id=customDimensions.worker_id",
"execs_sec=customDimensions.execs_sec",
"timestamp",
]

query_parts = [
"customEvents",
"where name == 'runtime_stats'",
query,
"where customDimensions.execs_sec > 0",
"order by timestamp desc",
f"project {','.join(project_fields)}",
]
if limit:
query_parts.append(f"take {limit}")

results = self.onefuzz.debug.logs._query_parts(query_parts, timespan=timespan)
if "PrimaryResult" in results:
return results["PrimaryResult"]
return results


class DebugNotification(Command):
Expand Down

0 comments on commit 31a661f

Please sign in to comment.