Merge branch 'main' into user/noharper/updated-threat-model

.github/workflows/ci.yml

@@ -94,6 +94,8 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           directory: artifacts
+          # aarch64 must be explicitly set; see: https://github.com/codecov/codecov-action/issues/650#issuecomment-1528860991
+          os: ${{ contains(join(matrix.os,':'), 'arm64') && 'aarch64' || '' }}
       - uses: actions/upload-artifact@v3
         with:
           name: build-artifacts
@@ -146,7 +148,7 @@ jobs:
           python setup.py sdist bdist_wheel
           pip install -r ./requirements.txt ../../artifacts/sdk/*.whl
           pip install six
-          pyinstaller onefuzz/__main__.py --onefile --name onefuzz --additional-hooks-dir extra/pyinstaller --hidden-import='pkg_resources.py2_warn' --exclude-module tkinter --exclude-module PySide2 --exclude-module PIL.ImageDraw --exclude-module Pillow --clean --add-data "onefuzz/data/privacy.txt;onefuzz/data" --add-data "onefuzz/data/licenses.json;onefuzz/data"
+          pyinstaller onefuzz/__main__.py --onefile --name onefuzz --additional-hooks-dir extra/pyinstaller --hidden-import='pkg_resources.py2_warn' --hidden-import='opentelemetry.baggage' --hidden-import='opentelemetry.baggage.propagation' --hidden-import='opentelemetry.context.contextvars_context' --copy-metadata opentelemetry-sdk --copy-metadata opentelemetry-api --exclude-module tkinter --exclude-module PySide2 --exclude-module PIL.ImageDraw --exclude-module Pillow --clean --add-data "onefuzz/data/privacy.txt;onefuzz/data" --add-data "onefuzz/data/licenses.json;onefuzz/data"
           ./dist/onefuzz.exe --version
           ./dist/onefuzz.exe privacy_statement
           mkdir -p ${GITHUB_WORKSPACE}/artifacts/windows-cli/
@@ -170,7 +172,7 @@ jobs:
           isort --profile black ./onefuzz ./examples/ ./tests/ --check
           pytest -v tests
           ../ci/disable-py-cache.sh
-          mypy --ignore-missing-imports ./onefuzz ./examples ./tests
+          mypy --ignore-missing-imports --implicit-reexport --namespace-packages ./onefuzz ./examples ./tests
 
           # set a minimum confidence to ignore known false positives
           vulture --min-confidence 61 onefuzz
@@ -290,7 +292,7 @@ jobs:
           # NB: ensure this stays in sync with get-version.sh
           if [[ "$version" =~ '+' ]]; then
             # if it has a suffix, split it into two parts
-            dotnet build -warnaserror --configuration Release /p:VersionPrefix=${version%+*} /p:VersionSuffix=${version#*+}
+            dotnet build -warnaserror --configuration Release /p:VersionPrefix=${version%+*} /p:SourceRevisionId=${version#*+}
           else
             dotnet build -warnaserror --configuration Release /p:VersionPrefix=${version}
           fi
@@ -474,7 +476,15 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
-      - run: |
+      - name: Cache integration tests
+        id: cache-integration-tests
+        uses: actions/cache@v3
+        with:
+          path: src/integration-tests/artifacts
+          key: integration-tests|linux|${{ hashFiles('src/integration-tests/**/*') }}
+      - name: Build integration tests
+        if: steps.cache-integration-tests.outputs.cache-hit != 'true'
+        run: |
           set -ex
           cd src/integration-tests
           mkdir -p artifacts
@@ -536,7 +546,15 @@ jobs:
     runs-on: windows-2019
     steps:
       - uses: actions/checkout@v3
-      - run: |
+      - name: Cache integration tests
+        id: cache-integration-tests
+        uses: actions/cache@v3
+        with:
+          path: src/integration-tests/artifacts
+          key: integration-tests|windows|${{ hashFiles('src/integration-tests/**/*') }}
+      - name: Build integration tests
+        if: steps.cache-integration-tests.outputs.cache-hit != 'true'
+        run: |
           Set-ExecutionPolicy Bypass -Scope Process -Force
           $ProgressPreference = 'SilentlyContinue'
           Invoke-Expression (Invoke-RestMethod 'https://chocolatey.org/install.ps1')

CURRENT_VERSION

@@ -1 +1 @@
-8.3.0
+8.4.0

docs/notifications/ado.md

@@ -127,6 +127,33 @@ To do nothing on duplicate reports, use the following `on_duplicate` entries:
 
 In the CLI, don't provide any of the --on*dup*\* arguments
 
+# To not reopen work items marked as Won't Fix
+
+This will not update/reopen work items that:
+
+* Have the state set to "Closed" **and** the reason set to "Wont Fix"
+
+**OR**
+
+* Have the state set to "Closed" **and** the reason set to "No Repro"
+
+**All fields _MUST_ match in order to skip**
+
+```jsonc
+"on_duplicate": {
+  "unless": [
+    {
+      "System.State": "Closed",
+      "System.Reason": "Wont Fix"
+    },
+    {
+      "System.State": "Closed",
+      "System.Reason": "No Repro"
+    }
+  ]
+}
+```
+
 # Example CLI usage:
 
 To create a similar configuration monitoring the container

docs/troubleshooting.md

@@ -0,0 +1,53 @@
+# Troubleshooting issues with binaries
+
+The agent exposes some validation tools to help troubleshoot issues with a binary.
+This allows the user to debug and fix errors that could prevent a job from running.
+
+## Using the validation tools
+Download a zip file containing the agent binaries:
+
+```
+onefuzz tools get <destination_folder>
+```
+
+Extract the zip file in a folder of your choice.
+Navigate to the folder that matches your os.
+Run the following command to see the tools available:
+`onefuzz-agent.exe validate --help`
+The current list of commands are:
+   - run_setup : Run the setup script
+   - validate_libfuzzer:  Validate the libfuzzer target
+   - execution_log: Get the execution logs to debug loading issues
+
+   More tools might be added in the future so please refer the help command to get the most up to date list.
+
+
+## In a docker container
+
+It could also be helpful to run the those command in an environment to closely match the vm where the agent is deployed.
+A docker container can help with that scenario.
+
+Make sure [docker](https://docs.docker.com/desktop/) is installed and runs properly.
+
+Navigate to the folder that matches your os in the tools folder created earlier and build the docker container:
+
+```cmd
+docker build --t <container_name> .
+```
+
+Use the container interactively to execute the validation command:
+
+windows
+
+```
+docker run --it --rm --entrypoint powershell <image_name>
+```
+
+linux
+
+```
+docker run --it --rm --entrypoint bash <image_name>
+```
+
+From there you can navigate to the onefuzz directory and execute the validation commands
+

docs/unmanaged-nodes.md

@@ -75,7 +75,7 @@ Under the `client_credential` section of the agent config file, update `client_i
 
 Save the config to the file.
 
-### Start the agent.
+### Start the agent
 
 Navigate to the folder corresponding to your OS.
 Set the necessary environment variable by running the script `set-env.ps1` (for Windows) or `set-env.sh` (for Linux).
@@ -111,7 +111,7 @@ From here you will be able to schedule jobs on that pool and they will run.
 
 ## Troubleshooting
 
-### increase the verbosity of the logs
+### Increase the verbosity of the logs
 
 It can help when investigating issues to increase the log verbosity. you will need to set the [RUST_LOG](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) environment variable when starting docker
 
@@ -127,7 +127,7 @@ log_level can be any of
 - debug
 - trace
 
-### use the container interactively
+### Use the container interactively
 
 you can use the container interactively by with the following command
 
@@ -143,7 +143,7 @@ linux
 docker run --it --rm --entrypoint bash <image_name>
 ```
 
-### mount a local folder in the container
+### Mount a local folder in the container
 
 docker allows you to [mount](https://docs.docker.com/storage/bind-mounts/#mount-into-a-non-empty-directory-on-the-container) a local folder when running a container
 

src/ApiService/ApiService/ApiService.csproj

@@ -34,7 +34,7 @@
     <PackageReference Include="Azure.Messaging.EventGrid" Version="4.15.0" />
     <PackageReference Include="Azure.ResourceManager" Version="1.6.0" />
     <PackageReference Include="Azure.ResourceManager.Network" Version="1.0.0" />
-    <PackageReference Include="Azure.ResourceManager.Resources" Version="1.3.0" />
+    <PackageReference Include="Azure.ResourceManager.Resources" Version="1.6.0" />
     <PackageReference Include="Azure.ResourceManager.Storage" Version="1.0.0-beta.11" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.11.0" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
@@ -51,6 +51,10 @@
     <PackageReference Include="Octokit" Version="2.0.1" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.219.0-preview" />
     <PackageReference Include="SmartAnalyzers.CSharpExtensions.Annotations" Version="4.2.7" />
+    <PackageReference Include="Polly" Version="8.0.0-alpha.2" />
+    <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
+    <PackageReference Include="OpenTelemetry.Api" Version="1.5.0-rc.1" />
+    <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.0-preview.4.23259.5" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

src/ApiService/ApiService/Functions/Download.cs

@@ -1,4 +1,5 @@
-using System.Web;
+using System.Net;
+using System.Web;
 using Azure.Storage.Sas;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
@@ -45,6 +46,16 @@ public async Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.An
             BlobSasPermissions.Read,
             TimeSpan.FromMinutes(5));
 
+        if (sasUri is null) {
+            // Note that we do not validate the existence of the file, only the container.
+            return await _context.RequestHandling.NotOk(req,
+                Error.Create(
+                    ErrorCode.INVALID_CONTAINER,
+                    "container not found"),
+                "generating download file SAS",
+                HttpStatusCode.NotFound);
+        }
+
         return RequestHandling.Redirect(req, sasUri);
     }
 }

src/ApiService/ApiService/Functions/Node.cs

@@ -174,6 +174,7 @@ private async Async.Task<HttpResponseData> Delete(HttpRequestData req) {
             }
         }
 
+
         return await RequestHandling.Ok(req, true);
     }
 }

src/ApiService/ApiService/Functions/Tasks.cs

@@ -61,7 +61,8 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
                 EndTime: task.EndTime,
                 UserInfo: task.UserInfo,
                 Nodes: nodes,
-                Events: events);
+                Events: events,
+                Timestamp: task.Timestamp);
 
             return await RequestHandling.Ok(req, result);
         }

src/ApiService/ApiService/HttpClient.cs

@@ -34,7 +34,7 @@ private async Task<HttpResponseMessage> Send(HttpMethod method, Uri url, HttpCon
             }
         }
 
-        return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+        return await _httpClient.SendAsync(request);
     }
 
     public async Task<HttpResponseMessage> Get(Uri url, string? json = null) {

src/ApiService/ApiService/Log.cs

@@ -26,6 +26,8 @@ public record TelemetryConfig(TelemetryClient TelemetryClient, ISet<Telemetry>?
 public class OneFuzzLogger : ILogger {
 
     public const string CorrelationId = "CorrelationId";
+    public const string TraceId = "TraceId";
+    public const string SpanId = "SpanId";
 
     private readonly string categoryName;
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -44,6 +44,9 @@ public enum ErrorCode {
     SCALE_IN_PROTECTION_INSTANCE_NO_LONGER_EXISTS = 487,
     SCALE_IN_PROTECTION_REACHED_MODEL_LIMIT = 488,
     SCALE_IN_PROTECTION_UNEXPECTED_ERROR = 489,
+    ADO_VALIDATION_UNEXPECTED_HTTP_EXCEPTION = 490,
+    ADO_VALIDATION_UNEXPECTED_ERROR = 491,
+    ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -577,30 +577,30 @@ public class NotificationTemplateConverter : JsonConverter<NotificationTemplate>
         try {
             return ValidateDeserialization(templateJson.Deserialize<AdoTemplate>(options));
         } catch (Exception ex) when (
-            ex is JsonException
-            || ex is ArgumentNullException
-            || ex is ArgumentOutOfRangeException
-        ) {
+              ex is JsonException
+              || ex is ArgumentNullException
+              || ex is ArgumentOutOfRangeException
+          ) {
 
         }
 
         try {
             return ValidateDeserialization(templateJson.Deserialize<TeamsTemplate>(options));
         } catch (Exception ex) when (
-            ex is JsonException
-            || ex is ArgumentNullException
-            || ex is ArgumentOutOfRangeException
-        ) {
+              ex is JsonException
+              || ex is ArgumentNullException
+              || ex is ArgumentOutOfRangeException
+          ) {
 
         }
 
         try {
             return ValidateDeserialization(templateJson.Deserialize<GithubIssuesTemplate>(options));
         } catch (Exception ex) when (
-            ex is JsonException
-            || ex is ArgumentNullException
-            || ex is ArgumentOutOfRangeException
-        ) {
+              ex is JsonException
+              || ex is ArgumentNullException
+              || ex is ArgumentOutOfRangeException
+          ) {
 
         }
 
@@ -657,7 +657,8 @@ public record ADODuplicateTemplate(
     List<string> Increment,
     Dictionary<string, string> SetState,
     Dictionary<string, string> AdoFields,
-    string? Comment = null
+    string? Comment = null,
+    List<Dictionary<string, string>>? Unless = null
 );
 
 public record AdoTemplate(

src/ApiService/ApiService/OneFuzzTypes/Responses.cs

@@ -48,7 +48,9 @@ public record TaskSearchResult(
     DateTimeOffset? EndTime,
     UserInfo? UserInfo,
     List<TaskEventSummary> Events,
-    List<NodeAssignment> Nodes
+    List<NodeAssignment> Nodes,
+    [property: JsonPropertyName("Timestamp")] // must retain capital T for backcompat
+    DateTimeOffset? Timestamp
 ) : BaseResponse();
 
 public record BoolResult(
@@ -93,7 +95,9 @@ public record JobResponse(
     JobConfig Config,
     string? Error,
     DateTimeOffset? EndTime,
-    List<JobTaskInfo>? TaskInfo
+    List<JobTaskInfo>? TaskInfo,
+    [property: JsonPropertyName("Timestamp")] // must retain capital T for backcompat
+    DateTimeOffset? Timestamp
 // not including UserInfo from Job model
 ) : BaseResponse() {
     public static JobResponse ForJob(Job j)
@@ -103,7 +107,8 @@ public static JobResponse ForJob(Job j)
             Config: j.Config,
             Error: j.Error,
             EndTime: j.EndTime,
-            TaskInfo: j.TaskInfo
+            TaskInfo: j.TaskInfo,
+            Timestamp: j.Timestamp
         );
 }