Skip to content
This repository has been archived by the owner on Nov 1, 2023. It is now read-only.

Commit

Permalink
Adding extra container to tasks (#2847)
Browse files Browse the repository at this point in the history 
* adding extra container to tasks

* setup expand

* build fix

* generate docs

* build fix

* build fix

* build fix

* format

* format

* build fix

* fix extra container references

* format

* Update "Needs Triage" label to the one we use. (#2845)

* Report extension errors (#2846)

Old failure message:
```
failed to launch extension
```

New failure message:

```
failed to launch extension(s): Errors for extension 'CustomScriptExtension':
:Error: ProvisioningState/failed/3 (Provisioning failed) - Failed to download all specified files. Exiting. Error Message: The remote server returned an error: (400) Bad Request.
```

* Sematically validate notification configs (#2850)

* Add new command

* Update remaining jinja templates and references to use scriban

* Add ado template validation

* Validate ado and github templates

* Remove unnecessary function

* Update src/ApiService/ApiService/OneFuzzTypes/Model.cs

Co-authored-by: Cheick Keita <kcheick@gmail.com>

---------

Co-authored-by: Cheick Keita <kcheick@gmail.com>

* adding extra container to integration tests

* adding doc

* update tests

* format

* build and clippy fix

* Update src/agent/onefuzz-task/src/tasks/report/generic.rs

Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>

---------

Co-authored-by: Marc Greisen <mgreisen@microsoft.com>
Co-authored-by: George Pollard <gpollard@microsoft.com>
Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>
  • Loading branch information
@chkeita @mgreisen
@Porges @tevoinea
4 people authored Feb 23, 2023
1 parent dfb0db8 commit b848968
Show file tree
Hide file tree
Showing 47 changed files with 367 additions and 27 deletions.
1 change: 1 addition & 0 deletions docs/command-replacements.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ The following values are replaced with the specific values at runtime.
* `{crashes_container}`: Container name for the `crashes` container
* `{microsoft_telemetry_key}`: Application Insights key used for collecting [non-attributable telemetry](telemetry.md) to improve OneFuzz.
* `{instance_telemetry_key}`: Application Insights key used for private, instance-owned telemetry and logging (See [OneFuzz Telemetry](telemetry.md).
* `{extra}`: Path to the optionally provided `extra` directory

## Example

Expand Down
24 changes: 16 additions & 8 deletions docs/webhook_events.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -2001,7 +2002,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -2927,7 +2929,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -3408,7 +3411,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -3932,7 +3936,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -4380,7 +4385,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -4855,7 +4861,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down Expand Up @@ -5460,7 +5467,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"unique_inputs",
"unique_reports",
"regression_reports",
"logs"
"logs",
"extra"
],
"title": "ContainerType"
},
Expand Down
3 changes: 2 additions & 1 deletion src/ApiService/ApiService/OneFuzzTypes/Enums.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ public enum ContainerType {
UniqueInputs,
UniqueReports,
RegressionReports,
Logs
Logs,
Extra
}


Expand Down
2 changes: 2 additions & 0 deletions src/ApiService/ApiService/OneFuzzTypes/Model.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,7 @@ public record TaskDefinition(
public record WorkSet(
bool Reboot,
Uri SetupUrl,
Uri? ExtraUrl,
bool Script,
List<WorkUnit> WorkUnits
);
Expand Down Expand Up @@ -1020,6 +1021,7 @@ Uri HeartbeatQueue
public IContainerDef? UniqueInputs { get; set; }
public IContainerDef? UniqueReports { get; set; }
public IContainerDef? RegressionReports { get; set; }
public IContainerDef? Extra { get; set; }

}

Expand Down
3 changes: 3 additions & 0 deletions src/ApiService/ApiService/onefuzzlib/Config.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ await _containers.GetContainerSasUrl(x.Item2.Name, StorageType.Corpus, ConvertPe
case ContainerType.RegressionReports:
config.RegressionReports = def;
break;
case ContainerType.Extra:
config.Extra = def;
break;
}
}

Expand Down
98 changes: 96 additions & 2 deletions src/ApiService/ApiService/onefuzzlib/Defs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,14 @@ public static class Defs {
ContainerPermission.List |
ContainerPermission.Read |
ContainerPermission.Write
)},
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.ReadonlyInputs)
},
{
Expand Down Expand Up @@ -83,6 +90,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.ReadonlyInputs)
},
Expand Down Expand Up @@ -135,6 +148,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.Crashes
)
Expand Down Expand Up @@ -185,6 +204,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)},
{ TaskType.GenericAnalysis ,
Expand Down Expand Up @@ -222,6 +247,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.Crashes)
},
Expand Down Expand Up @@ -264,6 +295,12 @@ public static class Defs {
Value: 0,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)},
{
Expand Down Expand Up @@ -309,6 +346,12 @@ public static class Defs {
Value: 1,
Permissions: ContainerPermission.Write
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.Crashes
)
Expand Down Expand Up @@ -343,7 +386,16 @@ public static class Defs {
ContainerPermission.List |
ContainerPermission.Read |
ContainerPermission.Write
)},
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),


},
MonitorQueue: ContainerType.ReadonlyInputs
)},
{
Expand Down Expand Up @@ -377,6 +429,12 @@ public static class Defs {
Value: 0,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)
},
Expand Down Expand Up @@ -445,6 +503,12 @@ public static class Defs {
Value: 1,
Permissions: ContainerPermission.Write | ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)
},
Expand Down Expand Up @@ -486,6 +550,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Write| ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)
},
Expand Down Expand Up @@ -532,6 +602,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
}
)
},
Expand Down Expand Up @@ -579,6 +655,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Write
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
},
MonitorQueue: ContainerType.Crashes
)
Expand Down Expand Up @@ -640,6 +722,12 @@ public static class Defs {
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
})
},
{
Expand Down Expand Up @@ -699,6 +787,12 @@ public static class Defs {
Permissions:
ContainerPermission.Read | ContainerPermission.List
),
new ContainerDefinition(
Type:ContainerType.Extra,
Compare: Compare.AtMost,
Value:1,
Permissions: ContainerPermission.Read | ContainerPermission.List
),
})
},
};
Expand Down
7 changes: 6 additions & 1 deletion src/ApiService/ApiService/onefuzzlib/Scheduler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,12 @@ private async Async.Task<bool> ScheduleWorkset(WorkSet workSet, Pool pool, long

if (bucketConfig is not null) {
var setupUrl = await _containers.GetContainerSasUrl(bucketConfig.setupContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List);
var extraUrl = bucketConfig.extraContainer != null ? await _containers.GetContainerSasUrl(bucketConfig.extraContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List) : null;
var workSet = new WorkSet(
Reboot: bucketConfig.reboot,
Script: bucketConfig.setupScript is not null,
SetupUrl: setupUrl,
ExtraUrl: extraUrl,
WorkUnits: workUnits
);

Expand All @@ -118,7 +120,7 @@ private async Async.Task<bool> ScheduleWorkset(WorkSet workSet, Pool pool, long
}


sealed record BucketConfig(long count, bool reboot, Container setupContainer, string? setupScript, Pool pool);
sealed record BucketConfig(long count, bool reboot, Container setupContainer, Container? extraContainer, string? setupScript, Pool pool);

sealed record PoolKey(
PoolName? poolName = null,
Expand Down Expand Up @@ -172,6 +174,8 @@ sealed record PoolKey(
}
var setupContainer = task.Config.Containers?.FirstOrDefault(c => c.Type == ContainerType.Setup) ?? throw new Exception($"task missing setup container: task_type = {task.Config.Task.Type}");

var extraContainer = task.Config.Containers?.FirstOrDefault(c => c.Type == ContainerType.Extra);

string? setupScript = null;
if (task.Os == Os.Windows) {
if (await _containers.BlobExists(setupContainer.Name, "setup.ps1", StorageType.Corpus)) {
Expand Down Expand Up @@ -209,6 +213,7 @@ sealed record PoolKey(
count,
reboot,
setupContainer.Name,
extraContainer?.Name,
setupScript,
pool with { ETag = default, TimeStamp = default });

Expand Down
2 changes: 1 addition & 1 deletion src/agent/coverage/src/binary.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ pub fn find_coverage_sites(

// Apply allowlists per block, to account for inlining. The `location` values
// here describe the top of the inline-inclusive call stack.
if !allowlist.source_files.is_allowed(&path) {
if !allowlist.source_files.is_allowed(path) {
continue;
}

Expand Down
1 change: 1 addition & 0 deletions src/agent/onefuzz-agent/src/agent/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ impl Fixture {
WorkSet {
reboot: false,
setup_url: self.setup_url(),
extra_url: None,
script: false,
work_units: vec![self.work_unit()],
}
Expand Down
Loading

0 comments on commit b848968

Please sign in to comment.