Revert removal of onefuzz repro get_files... command

.github/workflows/ci.yml

@@ -79,16 +79,24 @@ jobs:
           key: ${{env.ACTIONS_CACHE_KEY_DATE}} # additional key for cache-busting
           workspaces: src/agent
       - name: Linux Prereqs
-        if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
+        if: runner.os == 'Linux'
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config
+          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config clang
+      - name: Clone onefuzz-samples
+        run: git clone https://github.com/microsoft/onefuzz-samples
+      - name: Prepare for agent integration tests
+        shell: bash
+        working-directory: ./onefuzz-samples/examples/simple-libfuzzer
+        run: |
+          make
+          mkdir -p ../../../src/agent/onefuzz-task/tests/targets/simple
+          cp fuzz.exe ../../../src/agent/onefuzz-task/tests/targets/simple/fuzz.exe
+          cp *.pdb ../../../src/agent/onefuzz-task/tests/targets/simple/ 2>/dev/null || :
       - name: Install Rust Prereqs
-        if: steps.rust-build-cache.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
         run: src/ci/rust-prereqs.sh
       - run: src/ci/agent.sh
-        if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -115,7 +123,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: src/ci/check-check-pr.sh
@@ -129,7 +137,7 @@ jobs:
         shell: bash
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - uses: actions/download-artifact@v3
         with:
           name: artifact-onefuzztypes
@@ -182,7 +190,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: |
@@ -200,7 +208,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: lint
         shell: bash
         run: |
@@ -216,7 +224,7 @@ jobs:
       - run: src/ci/set-versions.sh
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - run: src/ci/onefuzztypes.sh
       - uses: actions/upload-artifact@v3
         with:
@@ -473,7 +481,7 @@ jobs:
           path: artifacts
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: "3.10"
       - name: Lint
         shell: bash
         run: |

CHANGELOG.md

@@ -7,6 +7,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.8.0
+
+### Added
+
+* Agent: Added Mariner Linux support for agent VMs [#3306](https://github.com/microsoft/onefuzz/pull/3306)
+* Service: Added support for custom ado fields that mark work items as duplicate [#3467](https://github.com/microsoft/onefuzz/pull/3467)
+* Service: Permanently store OneFuzz job result data - # crashing input, # regression crashing input, etc. - in Azure storage [#3380](https://github.com/microsoft/onefuzz/pull/3380), [#3439](https://github.com/microsoft/onefuzz/pull/3439)
+* Service: Added validation for Iteration/AreaPath on notifications when a job is submitted with a notification config and for `onefuzz debug notification test_template` [#3386](https://github.com/microsoft/onefuzz/pull/3386)
+
+### Changed
+
+* Agent: Updated libfuzzer-fuzz basic template to include required args and make it match cli [#3429](https://github.com/microsoft/onefuzz/pull/3429)
+* Agent: Downgraded some debug logs from warn to debug [#3450](https://github.com/microsoft/onefuzz/pull/3450)
+* CLI: Removed CLI commands from the local fuzzing tasks as they can now be described via yaml template [#3428](https://github.com/microsoft/onefuzz/pull/3428)
+* Service: AutoScale table entries are now deleted on VMSS shutdown [#3455](https://github.com/microsoft/onefuzz/pull/3455)
+
+### Fixed
+
+* Agent: Fixed local path generation [#3432](https://github.com/microsoft/onefuzz/pull/3432), [#3460](https://github.com/microsoft/onefuzz/pull/3460)
+
+## 8.7.1
+
+### Fixed
+
+* Service: Removed deprecated Azure retention policy setting that was causing scaleset deployment errors [#3452](https://github.com/microsoft/onefuzz/pull/3452)
+
 ## 8.7.0
 
 ### Added

CURRENT_VERSION

@@ -1 +1 @@
-8.7.0
+8.8.0

README.md

@@ -1,5 +1,26 @@
 # <img src="docs/onefuzz_text.svg" height="120" alt="OneFuzz" />
 
+# :exclamation: IMPORTANT NOTICE :exclamation:
+
+**_August 31, 2023_**.
+
+**_Since September 2020 when OneFuzz was first open sourced, we’ve been on a journey to create a best-in-class orchestrator for running fuzzers, driving security and quality into our products._**
+
+
+**_Initially launched by a small group in MSR, OneFuzz has now become a significant internal platform within Microsoft. As such, we are regretfully archiving the project to focus our attention on becoming a more deeply integrated service within the company. Unfortunately, we aren’t a large enough team to live in both the open-source world and the internal Microsoft world with its own unique set of requirements._**
+
+**_Our current plan is to archive the project in the next few months. That means we’ll still be making updates for a little while. Of course, even after it’s archived, you’ll still be able to fork it and make the changes you need. Once we’ve decided on a specific date for archiving, we’ll update this readme._**
+
+**_Thanks for taking the journey with us._**
+
+**_The OneFuzz team._**
+
+---
+**_Update: September 15 2023:_**
+**_Our current target to archive the project is September 30th, 2023._**
+
+---
+
 [![Onefuzz build status](https://github.com/microsoft/onefuzz/workflows/Build/badge.svg?branch=main)](https://github.com/microsoft/onefuzz/actions/workflows/ci.yml?query=branch%3Amain)
 
 ## A self-hosted Fuzzing-As-A-Service platform

contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json

@@ -13,6 +13,10 @@
       "System.AreaPath": "OneFuzz-Ado-Integration",
       "System.Title": "{{report.task_id}}"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+    },
     "on_duplicate": {
       "increment": [],
       "comment": "DUP {{report.input_sha256}} <br> Repro Command: <br> <pre> {{ repro_cmd }} </pre> ",

docs/notifications/ado.md

@@ -51,6 +51,13 @@ clickable, make it a link.
       "System.Title": "{{ report.crash_site }} - {{ report.executable }}",
       "Microsoft.VSTS.TCM.ReproSteps": "This is my call stack: <ul> {{ for item in report.call_stack }} <li> {{ item }} </li> {{ end }} </ul>"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+      // note: the fields and values below are checked by default and don't need to be specified
+      // "System.Reason": "Duplicate"
+      // "Microsoft.VSTS.Common.ResolvedReason": "Duplicate"
+    },
     "comment": "This is my comment. {{ report.input_sha256 }} {{ input_url }} <br> <pre>{{ repro_cmd }}</pre>",
     "unique_fields": ["System.Title", "System.AreaPath"],
     "on_duplicate": {

docs/webhook_events.md

@@ -2033,6 +2033,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                 },
                 "original_crash_test_result": {
                     "$ref": "#/definitions/CrashTestResult"
+                },
+                "report_url": {
+                    "title": "Report Url",
+                    "type": "string"
                 }
             },
             "required": [
@@ -6427,6 +6431,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                 },
                 "original_crash_test_result": {
                     "$ref": "#/definitions/CrashTestResult"
+                },
+                "report_url": {
+                    "title": "Report Url",
+                    "type": "string"
                 }
             },
             "required": [

src/ApiService/ApiService/FeatureFlags.cs

@@ -8,4 +8,5 @@ public static class FeatureFlagConstants {
     public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy";
     public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
     public const string EnableWorkItemCreation = "EnableWorkItemCreation";
+    public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies";
 }

src/ApiService/ApiService/Functions/Jobs.cs

@@ -83,7 +83,7 @@ private async Task<HttpResponseData> Post(HttpRequestData req, FunctionContext c
                 "job");
         }
 
-        await _context.Events.SendEvent(new EventJobCreated(job.JobId, job.Config, job.UserInfo));
+        await _context.Events.SendEvent(new EventJobCreated(job.JobId, job.Config, job.UserInfo, _context.ServiceConfiguration.OneFuzzVersion));
         return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo: null));
     }
 

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -1,5 +1,6 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using System.Threading.Tasks;
 using Azure.Core;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
@@ -54,14 +55,16 @@ public async Async.Task Run(
             return;
         }
 
+        var storageAccount = new ResourceIdentifier(topicElement.GetString()!);
+
         try {
             // Setting isLastRetryAttempt to false will rethrow any exceptions
             // With the intention that the azure functions runtime will handle requeing
             // the message for us. The difference is for the poison queue, we're handling the
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(storageAccount, fileChangeEvent, isLastRetryAttempt: false);
             if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
                 await RequeueMessage(msg, TimeSpan.FromDays(1));
             }
@@ -71,16 +74,47 @@ public async Async.Task Run(
         }
     }
 
-    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storageAccount, JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
 
-        var container = parts[0];
+        var container = Container.Parse(parts[0]);
         var path = string.Join('/', parts.Skip(1));
 
-        _log.LogInformation("file added : {Container} - {Path}", container, path);
-        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+        _log.LogInformation("file added : {Container} - {Path}", container.String, path);
+
+        var (_, result) = await (
+            ApplyRetentionPolicy(storageAccount, container, path),
+            _notificationOperations.NewFiles(container, path, isLastRetryAttempt));
+
+        return result;
+    }
+
+    private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
+        if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) {
+            // default retention period can be applied to the container
+            // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
+            var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
+            var containerClient = account.GetBlobContainerClient(container.String);
+            var containerProps = await containerClient.GetPropertiesAsync();
+            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata);
+            if (!retentionPeriod.IsOk) {
+                _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV);
+            } else if (retentionPeriod.OkV is TimeSpan period) {
+                var blobClient = containerClient.GetBlobClient(path);
+                var tags = (await blobClient.GetTagsAsync()).Value.Tags;
+                var expiryDate = DateTime.UtcNow + period;
+                var tag = RetentionPolicyUtils.CreateExpiryDateTag(DateOnly.FromDateTime(expiryDate));
+                if (tags.TryAdd(tag.Key, tag.Value)) {
+                    _ = await blobClient.SetTagsAsync(tags);
+                    _log.LogInformation("applied container retention policy ({Policy}) to {Path}", period, path);
+                    return true;
+                }
+            }
+        }
+
+        return false;
     }
 
     private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
@@ -94,20 +128,22 @@ private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout
             newCustomDequeueCount = json["data"]!["customDequeueCount"]!.GetValue<int>();
         }
 
-        var queueName = QueueFileChangesQueueName;
         if (newCustomDequeueCount > MAX_DEQUEUE_COUNT) {
             _log.LogWarning("Message retried more than {MAX_DEQUEUE_COUNT} times with no success: {msg}", MAX_DEQUEUE_COUNT, msg);
-            queueName = QueueFileChangesPoisonQueueName;
+            await _context.Queue.QueueObject(
+                QueueFileChangesPoisonQueueName,
+                json,
+                StorageType.Config)
+                .IgnoreResult();
+        } else {
+            json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
+            await _context.Queue.QueueObject(
+                QueueFileChangesQueueName,
+                json,
+                StorageType.Config,
+                visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount))
+                .IgnoreResult();
         }
-
-        json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
-
-        await _context.Queue.QueueObject(
-            queueName,
-            json,
-            StorageType.Config,
-            visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount))
-            .IgnoreResult();
     }
 
     // Possible return values:

src/ApiService/ApiService/Functions/QueueNodeHeartbeat.cs

@@ -41,9 +41,8 @@ public async Async.Task Run([QueueTrigger("node-heartbeat", Connection = "AzureW
         var nodeHeartbeatEvent = new EventNodeHeartbeat(node.MachineId, node.ScalesetId, node.PoolName, node.State);
         // TODO: do we still send event if we fail do update the table ?
         await events.SendEvent(nodeHeartbeatEvent);
-        if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableCustomMetricTelemetry)) {
-            metrics.SendMetric(1, nodeHeartbeatEvent);
-        }
+        metrics.SendMetric(1, nodeHeartbeatEvent);
+
 
     }
 }

src/ApiService/ApiService/Functions/QueueTaskHeartbeat.cs

@@ -45,8 +45,7 @@ public async Async.Task Run([QueueTrigger("task-heartbeat", Connection = "AzureW
 
         var taskHeartBeatEvent = new EventTaskHeartbeat(newTask.JobId, newTask.TaskId, job.Config.Project, job.Config.Name, newTask.State, newTask.Config);
         await _events.SendEvent(taskHeartBeatEvent);
-        if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableCustomMetricTelemetry)) {
-            _metrics.SendMetric(1, taskHeartBeatEvent);
-        }
+        _metrics.SendMetric(1, taskHeartBeatEvent);
+
     }
 }

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -49,6 +49,8 @@ public enum ErrorCode {
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
     ADO_WORKITEM_PROCESSING_DISABLED = 494,
     ADO_VALIDATION_INVALID_PATH = 495,
+    ADO_VALIDATION_INVALID_PROJECT = 496,
+    INVALID_RETENTION_PERIOD = 497,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -124,7 +124,8 @@ TaskConfig Config
 public record EventJobCreated(
    Guid JobId,
    JobConfig Config,
-   StoredUserInfo? UserInfo
+   StoredUserInfo? UserInfo,
+   string OneFuzzVersion
    ) : BaseEvent();
 
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -678,7 +678,8 @@ public record ADODuplicateTemplate(
     Dictionary<string, string> SetState,
     Dictionary<string, string> AdoFields,
     string? Comment = null,
-    List<Dictionary<string, string>>? Unless = null
+    List<Dictionary<string, string>>? Unless = null,
+    List<string>? RegressionIgnoreStates = null
 );
 
 public record AdoTemplate(
@@ -689,6 +690,7 @@ public record AdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
     ) : NotificationTemplate {
     public async Task<OneFuzzResultVoid> Validate() {
@@ -704,8 +706,9 @@ public record RenderedAdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
-    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);
+    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, AdoDuplicateFields, Comment) { }
 
 public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
     public Task<OneFuzzResultVoid> Validate() {