From 2331ae4017bb8a93a75eaf6b4edf306ea94e1604 Mon Sep 17 00:00:00 2001 From: George Pollard Date: Wed, 27 Sep 2023 09:13:06 +1300 Subject: [PATCH] Support for retention policies on containers (#3501) - [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class. --- .github/workflows/ci.yml | 12 +- src/ApiService/ApiService/FeatureFlags.cs | 1 + .../ApiService/Functions/QueueFileChanges.cs | 44 ++++- .../ApiService/OneFuzzTypes/Enums.cs | 1 + .../onefuzzlib/NotificationOperations.cs | 5 +- .../ApiService/onefuzzlib/RententionPolicy.cs | 24 --- .../ApiService/onefuzzlib/RetentionPolicy.cs | 43 +++++ src/cli/examples/domato.py | 21 ++- src/cli/examples/honggfuzz.py | 19 +- .../source-coverage-libfuzzer.py | 10 +- .../llvm-source-coverage/source-coverage.py | 10 +- src/cli/onefuzz/templates/__init__.py | 93 ++++++++-- src/cli/onefuzz/templates/afl.py | 22 +-- src/cli/onefuzz/templates/libfuzzer.py | 162 ++++++++++++------ src/cli/onefuzz/templates/ossfuzz.py | 8 +- src/cli/onefuzz/templates/radamsa.py | 51 ++++-- src/cli/onefuzz/templates/regression.py | 37 ++-- .../bicep-templates/feature-flags.bicep | 13 ++ src/pytypes/onefuzztypes/enums.py | 1 + 19 files changed, 393 insertions(+), 184 deletions(-) delete mode 100644 src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs create mode 100644 src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99e9bddd32..5f07124dd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.10" - name: lint shell: bash run: src/ci/check-check-pr.sh @@ -137,7 +137,7 @@ jobs: shell: bash - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.10" - uses: actions/download-artifact@v3 with: name: artifact-onefuzztypes @@ -190,7 +190,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - name: lint shell: bash run: | @@ -208,7 +208,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - name: lint shell: bash run: | @@ -224,7 +224,7 @@ jobs: - run: src/ci/set-versions.sh - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.10" - run: src/ci/onefuzztypes.sh - uses: actions/upload-artifact@v3 with: @@ -481,7 +481,7 @@ jobs: path: artifacts - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.10" - name: Lint shell: bash run: | diff --git a/src/ApiService/ApiService/FeatureFlags.cs b/src/ApiService/ApiService/FeatureFlags.cs index aa4bc87079..e74396e882 100644 --- a/src/ApiService/ApiService/FeatureFlags.cs +++ b/src/ApiService/ApiService/FeatureFlags.cs @@ -8,4 +8,5 @@ public static class FeatureFlagConstants { public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy"; public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention"; public const string EnableWorkItemCreation = "EnableWorkItemCreation"; + public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies"; } diff --git a/src/ApiService/ApiService/Functions/QueueFileChanges.cs b/src/ApiService/ApiService/Functions/QueueFileChanges.cs index acdd3e328d..f1c4711f9d 100644 --- a/src/ApiService/ApiService/Functions/QueueFileChanges.cs +++ b/src/ApiService/ApiService/Functions/QueueFileChanges.cs @@ -1,5 +1,6 @@ using System.Text.Json; using System.Text.Json.Nodes; +using System.Threading.Tasks; using Azure.Core; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; @@ -54,6 +55,8 @@ public async Async.Task Run( return; } + var storageAccount = new ResourceIdentifier(topicElement.GetString()!); + try { // Setting isLastRetryAttempt to false will rethrow any exceptions // With the intention that the azure functions runtime will handle requeing @@ -61,7 +64,7 @@ public async Async.Task Run( // requeuing ourselves because azure functions doesn't support retry policies // for queue based functions. - var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false); + var result = await FileAdded(storageAccount, fileChangeEvent, isLastRetryAttempt: false); if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) { await RequeueMessage(msg, TimeSpan.FromDays(1)); } @@ -71,16 +74,47 @@ public async Async.Task Run( } } - private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) { + private async Async.Task FileAdded(ResourceIdentifier storageAccount, JsonDocument fileChangeEvent, bool isLastRetryAttempt) { var data = fileChangeEvent.RootElement.GetProperty("data"); var url = data.GetProperty("url").GetString()!; var parts = url.Split("/").Skip(3).ToList(); - var container = parts[0]; + var container = Container.Parse(parts[0]); var path = string.Join('/', parts.Skip(1)); - _log.LogInformation("file added : {Container} - {Path}", container, path); - return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt); + _log.LogInformation("file added : {Container} - {Path}", container.String, path); + + var (_, result) = await ( + ApplyRetentionPolicy(storageAccount, container, path), + _notificationOperations.NewFiles(container, path, isLastRetryAttempt)); + + return result; + } + + private async Async.Task ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) { + if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) { + // default retention period can be applied to the container + // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one + var account = await _storage.GetBlobServiceClientForAccount(storageAccount); + var containerClient = account.GetBlobContainerClient(container.String); + var containerProps = await containerClient.GetPropertiesAsync(); + var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata); + if (!retentionPeriod.IsOk) { + _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV); + } else if (retentionPeriod.OkV is TimeSpan period) { + var blobClient = containerClient.GetBlobClient(path); + var tags = (await blobClient.GetTagsAsync()).Value.Tags; + var expiryDate = DateTime.UtcNow + period; + var tag = RetentionPolicyUtils.CreateExpiryDateTag(DateOnly.FromDateTime(expiryDate)); + if (tags.TryAdd(tag.Key, tag.Value)) { + _ = await blobClient.SetTagsAsync(tags); + _log.LogInformation("applied container retention policy ({Policy}) to {Path}", period, path); + return true; + } + } + } + + return false; } private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) { diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 4739987e6b..4692debfe8 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -50,6 +50,7 @@ public enum ErrorCode { ADO_WORKITEM_PROCESSING_DISABLED = 494, ADO_VALIDATION_INVALID_PATH = 495, ADO_VALIDATION_INVALID_PROJECT = 496, + INVALID_RETENTION_PERIOD = 497, // NB: if you update this enum, also update enums.py } diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs index dc133e1fba..0eca5b1e00 100644 --- a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs @@ -22,13 +22,12 @@ public NotificationOperations(ILogger log, IOnefuzzConte } public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) { - var result = OneFuzzResultVoid.Ok; - // We don't want to store file added events for the events container because that causes an infinite loop if (container == WellKnownContainers.Events) { - return result; + return Result.Ok(); } + var result = OneFuzzResultVoid.Ok; var notifications = GetNotifications(container); var hasNotifications = await notifications.AnyAsync(); var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications); diff --git a/src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs b/src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs deleted file mode 100644 index 4052db93e1..0000000000 --- a/src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace Microsoft.OneFuzz.Service; - - -public interface IRetentionPolicy { - DateOnly GetExpiryDate(); -} - -public class RetentionPolicyUtils { - public const string EXPIRY_TAG = "Expiry"; - public static KeyValuePair CreateExpiryDateTag(DateOnly expiryDate) => - new(EXPIRY_TAG, expiryDate.ToString()); - - public static DateOnly? GetExpiryDateTagFromTags(IDictionary? blobTags) { - if (blobTags != null && - blobTags.TryGetValue(EXPIRY_TAG, out var expiryTag) && - !string.IsNullOrWhiteSpace(expiryTag) && - DateOnly.TryParse(expiryTag, out var expiryDate)) { - return expiryDate; - } - return null; - } - - public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'"; -} diff --git a/src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs b/src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs new file mode 100644 index 0000000000..48d81df5c7 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs @@ -0,0 +1,43 @@ +using System.Xml; + +namespace Microsoft.OneFuzz.Service; + + +public interface IRetentionPolicy { + DateOnly GetExpiryDate(); +} + +public class RetentionPolicyUtils { + public const string EXPIRY_TAG = "Expiry"; + public static KeyValuePair CreateExpiryDateTag(DateOnly expiryDate) => + new(EXPIRY_TAG, expiryDate.ToString()); + + public static DateOnly? GetExpiryDateTagFromTags(IDictionary? blobTags) { + if (blobTags != null && + blobTags.TryGetValue(EXPIRY_TAG, out var expiryTag) && + !string.IsNullOrWhiteSpace(expiryTag) && + DateOnly.TryParse(expiryTag, out var expiryDate)) { + return expiryDate; + } + return null; + } + + public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'"; + + // NB: this must match the value used on the CLI side + public const string CONTAINER_RETENTION_KEY = "onefuzz_retentionperiod"; + + public static OneFuzzResult GetContainerRetentionPeriodFromMetadata(IDictionary? containerMetadata) { + if (containerMetadata is not null && + containerMetadata.TryGetValue(CONTAINER_RETENTION_KEY, out var retentionString) && + !string.IsNullOrWhiteSpace(retentionString)) { + try { + return Result.Ok(XmlConvert.ToTimeSpan(retentionString)); + } catch (Exception ex) { + return Error.Create(ErrorCode.INVALID_RETENTION_PERIOD, ex.Message); + } + } + + return Result.Ok(null); + } +} diff --git a/src/cli/examples/domato.py b/src/cli/examples/domato.py index 7c2abc6301..4bdf2a297c 100755 --- a/src/cli/examples/domato.py +++ b/src/cli/examples/domato.py @@ -67,7 +67,7 @@ def upload_to_fuzzer_container(of: Onefuzz, fuzzer_name: str, fuzzer_url: str) - def upload_to_setup_container(of: Onefuzz, helper: JobHelper, setup_dir: str) -> None: - setup_sas = of.containers.get(helper.containers[ContainerType.setup]).sas_url + setup_sas = of.containers.get(helper.container_name(ContainerType.setup)).sas_url if AZCOPY_PATH is None: raise Exception("missing azcopy") command = [AZCOPY_PATH, "sync", setup_dir, setup_sas] @@ -143,13 +143,16 @@ def main() -> None: helper.create_containers() helper.setup_notifications(notification_config) upload_to_setup_container(of, helper, args.setup_dir) - add_setup_script(of, helper.containers[ContainerType.setup]) + add_setup_script(of, helper.container_name(ContainerType.setup)) containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), - (ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), + ( + ContainerType.unique_reports, + helper.container_name(ContainerType.unique_reports), + ), ] of.logger.info("Creating generic_crash_report task") @@ -164,11 +167,11 @@ def main() -> None: containers = [ (ContainerType.tools, Container(FUZZER_NAME)), - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), ( ContainerType.readonly_inputs, - helper.containers[ContainerType.readonly_inputs], + helper.container_name(ContainerType.readonly_inputs), ), ] diff --git a/src/cli/examples/honggfuzz.py b/src/cli/examples/honggfuzz.py index 225b7f7510..9466716d98 100644 --- a/src/cli/examples/honggfuzz.py +++ b/src/cli/examples/honggfuzz.py @@ -88,13 +88,16 @@ def main() -> None: if args.inputs: helper.upload_inputs(args.inputs) - add_setup_script(of, helper.containers[ContainerType.setup]) + add_setup_script(of, helper.container_name(ContainerType.setup)) containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), - (ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), + ( + ContainerType.unique_reports, + helper.container_name(ContainerType.unique_reports), + ), ] of.logger.info("Creating generic_crash_report task") @@ -109,11 +112,11 @@ def main() -> None: containers = [ (ContainerType.tools, Container("honggfuzz")), - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), ( ContainerType.inputs, - helper.containers[ContainerType.inputs], + helper.container_name(ContainerType.inputs), ), ] diff --git a/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py b/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py index a8a6a91ac9..b8ab5f347a 100755 --- a/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py +++ b/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py @@ -74,15 +74,15 @@ def main() -> None: helper.create_containers() of.containers.files.upload_file( - helper.containers[ContainerType.tools], f"{args.tools}/source-coverage.sh" + helper.container_name(ContainerType.tools), f"{args.tools}/source-coverage.sh" ) containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.analysis, helper.containers[ContainerType.analysis]), - (ContainerType.tools, helper.containers[ContainerType.tools]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.analysis, helper.container_name(ContainerType.analysis)), + (ContainerType.tools, helper.container_name(ContainerType.tools)), # note, analysis is typically for crashes, but this is analyzing inputs - (ContainerType.crashes, helper.containers[ContainerType.inputs]), + (ContainerType.crashes, helper.container_name(ContainerType.inputs)), ] of.logger.info("Creating generic_analysis task") diff --git a/src/cli/examples/llvm-source-coverage/source-coverage.py b/src/cli/examples/llvm-source-coverage/source-coverage.py index 749662caba..ae903cd3b5 100755 --- a/src/cli/examples/llvm-source-coverage/source-coverage.py +++ b/src/cli/examples/llvm-source-coverage/source-coverage.py @@ -61,15 +61,15 @@ def main() -> None: helper.upload_inputs(args.inputs) of.containers.files.upload_file( - helper.containers[ContainerType.tools], f"{args.tools}/source-coverage.sh" + helper.container_name(ContainerType.tools), f"{args.tools}/source-coverage.sh" ) containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.analysis, helper.containers[ContainerType.analysis]), - (ContainerType.tools, helper.containers[ContainerType.tools]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.analysis, helper.container_name(ContainerType.analysis)), + (ContainerType.tools, helper.container_name(ContainerType.tools)), # note, analysis is typically for crashes, but this is analyzing inputs - (ContainerType.crashes, helper.containers[ContainerType.inputs]), + (ContainerType.crashes, helper.container_name(ContainerType.inputs)), ] of.logger.info("Creating generic_analysis task") diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index c7bb21d41a..a88db46303 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -6,6 +6,7 @@ import os import tempfile import zipfile +from datetime import timedelta from typing import Any, Dict, List, Optional, Tuple from uuid import uuid4 @@ -22,6 +23,30 @@ class StoppedEarly(Exception): pass +class ContainerTemplate: + def __init__( + self, + name: Container, + exists: bool, + *, + retention_period: Optional[timedelta] = None, + ): + self.name = name + self.retention_period = retention_period + # TODO: exists is not yet used/checked + self.exists = exists + + @staticmethod + def existing(name: Container) -> "ContainerTemplate": + return ContainerTemplate(name, True) + + @staticmethod + def fresh( + name: Container, *, retention_period: Optional[timedelta] = None + ) -> "ContainerTemplate": + return ContainerTemplate(name, False, retention_period=retention_period) + + class JobHelper: def __init__( self, @@ -59,7 +84,7 @@ def __init__( self.wait_for_running: bool = False self.wait_for_stopped: bool = False - self.containers: Dict[ContainerType, Container] = {} + self.containers: Dict[ContainerType, ContainerTemplate] = {} self.tags: Dict[str, str] = {"project": project, "name": name, "build": build} if job is None: self.onefuzz.versions.check() @@ -71,6 +96,20 @@ def __init__( else: self.job = job + def add_existing_container( + self, container_type: ContainerType, container: Container + ) -> None: + self.containers[container_type] = ContainerTemplate.existing(container) + + def container_name(self, container_type: ContainerType) -> Container: + return self.containers[container_type].name + + def container_names(self) -> Dict[ContainerType, Container]: + return { + container_type: container.name + for (container_type, container) in self.containers.items() + } + def define_containers(self, *types: ContainerType) -> None: """ Define default container set based on provided types @@ -79,13 +118,23 @@ def define_containers(self, *types: ContainerType) -> None: """ for container_type in types: - self.containers[container_type] = self.onefuzz.utils.build_container_name( + container_name = self.onefuzz.utils.build_container_name( container_type=container_type, project=self.project, name=self.name, build=self.build, platform=self.platform, ) + self.containers[container_type] = ContainerTemplate.fresh( + container_name, + retention_period=JobHelper._default_retention_period(container_type), + ) + + @staticmethod + def _default_retention_period(container_type: ContainerType) -> Optional[timedelta]: + if container_type == ContainerType.crashdumps: + return timedelta(days=90) + return None def get_unique_container_name(self, container_type: ContainerType) -> Container: return Container( @@ -97,11 +146,17 @@ def get_unique_container_name(self, container_type: ContainerType) -> Container: ) def create_containers(self) -> None: - for container_type, container_name in self.containers.items(): - self.logger.info("using container: %s", container_name) - self.onefuzz.containers.create( - container_name, metadata={"container_type": container_type.name} - ) + for container_type, container in self.containers.items(): + self.logger.info("using container: %s", container.name) + metadata = {"container_type": container_type.name} + if container.retention_period is not None: + # format as ISO8601 period + # NB: this must match the value used on the server side + metadata[ + "onefuzz_retentionperiod" + ] = f"P{container.retention_period.days}D" + + self.onefuzz.containers.create(container.name, metadata=metadata) def delete_container(self, container_name: Container) -> None: self.onefuzz.containers.delete(container_name) @@ -112,12 +167,12 @@ def setup_notifications(self, config: Optional[NotificationConfig]) -> None: containers: List[Container] = [] if ContainerType.unique_reports in self.containers: - containers.append(self.containers[ContainerType.unique_reports]) + containers.append(self.container_name(ContainerType.unique_reports)) else: - containers.append(self.containers[ContainerType.reports]) + containers.append(self.container_name(ContainerType.reports)) if ContainerType.regression_reports in self.containers: - containers.append(self.containers[ContainerType.regression_reports]) + containers.append(self.container_name(ContainerType.regression_reports)) for container in containers: self.logger.info("creating notification config for %s", container) @@ -141,25 +196,25 @@ def upload_setup( self.logger.info("uploading setup dir `%s`" % setup_dir) self.onefuzz.containers.files.upload_dir( - self.containers[ContainerType.setup], setup_dir + self.container_name(ContainerType.setup), setup_dir ) else: self.logger.info("uploading target exe `%s`" % target_exe) self.onefuzz.containers.files.upload_file( - self.containers[ContainerType.setup], target_exe + self.container_name(ContainerType.setup), target_exe ) pdb_path = os.path.splitext(target_exe)[0] + ".pdb" if os.path.exists(pdb_path): pdb_name = os.path.basename(pdb_path) self.onefuzz.containers.files.upload_file( - self.containers[ContainerType.setup], pdb_path, pdb_name + self.container_name(ContainerType.setup), pdb_path, pdb_name ) if setup_files: for filename in setup_files: self.logger.info("uploading %s", filename) self.onefuzz.containers.files.upload_file( - self.containers[ContainerType.setup], filename + self.container_name(ContainerType.setup), filename ) def upload_inputs(self, path: Directory, read_only: bool = False) -> None: @@ -167,7 +222,9 @@ def upload_inputs(self, path: Directory, read_only: bool = False) -> None: container_type = ContainerType.inputs if read_only: container_type = ContainerType.readonly_inputs - self.onefuzz.containers.files.upload_dir(self.containers[container_type], path) + self.onefuzz.containers.files.upload_dir( + self.container_name(container_type), path + ) def upload_inputs_zip(self, path: File) -> None: with tempfile.TemporaryDirectory() as tmp_dir: @@ -176,7 +233,7 @@ def upload_inputs_zip(self, path: File) -> None: self.logger.info("uploading inputs from zip: `%s`" % path) self.onefuzz.containers.files.upload_dir( - self.containers[ContainerType.inputs], Directory(tmp_dir) + self.container_name(ContainerType.inputs), Directory(tmp_dir) ) @classmethod @@ -195,8 +252,8 @@ def wait_on( wait_for_files = [] self.to_monitor = { - self.containers[x]: len( - self.onefuzz.containers.files.list(self.containers[x]).files + self.container_name(x): len( + self.onefuzz.containers.files.list(self.container_name(x)).files ) for x in wait_for_files } diff --git a/src/cli/onefuzz/templates/afl.py b/src/cli/onefuzz/templates/afl.py index d3019a19cf..e4d49233dc 100644 --- a/src/cli/onefuzz/templates/afl.py +++ b/src/cli/onefuzz/templates/afl.py @@ -98,7 +98,7 @@ def basic( if existing_inputs: self.onefuzz.containers.get(existing_inputs) - helper.containers[ContainerType.inputs] = existing_inputs + helper.add_existing_container(ContainerType.inputs, existing_inputs) else: helper.define_containers(ContainerType.inputs) @@ -112,7 +112,7 @@ def basic( if ( len( self.onefuzz.containers.files.list( - helper.containers[ContainerType.inputs] + helper.containers[ContainerType.inputs].name ).files ) == 0 @@ -131,16 +131,16 @@ def basic( containers = [ (ContainerType.tools, afl_container), - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.inputs, helper.containers[ContainerType.inputs]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.inputs, helper.container_name(ContainerType.inputs)), ] if extra_setup_container is not None: containers.append( ( ContainerType.extra_setup, - helper.containers[ContainerType.extra_setup], + extra_setup_container, ) ) @@ -169,12 +169,12 @@ def basic( ) report_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), ( ContainerType.unique_reports, - helper.containers[ContainerType.unique_reports], + helper.container_name(ContainerType.unique_reports), ), ] @@ -182,7 +182,7 @@ def basic( report_containers.append( ( ContainerType.extra_setup, - helper.containers[ContainerType.extra_setup], + helper.container_name(ContainerType.extra_setup), ) ) diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 7716cfefed..f487372121 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -85,7 +85,7 @@ def _create_tasks( task_env: Optional[Dict[str, str]] = None, ) -> None: target_options = target_options or [] - regression_containers = [ + regression_containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), (ContainerType.unique_reports, containers[ContainerType.unique_reports]), @@ -129,7 +129,7 @@ def _create_tasks( task_env=task_env, ) - fuzzer_containers = [ + fuzzer_containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), (ContainerType.crashdumps, containers[ContainerType.crashdumps]), @@ -184,7 +184,7 @@ def _create_tasks( prereq_tasks = [fuzzer_task.task_id, regression_task.task_id] - coverage_containers = [ + coverage_containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.coverage, containers[ContainerType.coverage]), (ContainerType.readonly_inputs, containers[ContainerType.inputs]), @@ -245,7 +245,7 @@ def _create_tasks( task_env=task_env, ) - report_containers = [ + report_containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), (ContainerType.reports, containers[ContainerType.reports]), @@ -285,7 +285,7 @@ def _create_tasks( if analyzer_exe is not None: self.logger.info("creating custom analysis") - analysis_containers = [ + analysis_containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.analysis, containers[ContainerType.analysis]), (ContainerType.crashes, containers[ContainerType.crashes]), @@ -428,15 +428,17 @@ def basic( ) if existing_inputs: - helper.containers[ContainerType.inputs] = existing_inputs + helper.add_existing_container(ContainerType.inputs, existing_inputs) else: helper.define_containers(ContainerType.inputs) if readonly_inputs: - helper.containers[ContainerType.readonly_inputs] = readonly_inputs + helper.add_existing_container( + ContainerType.readonly_inputs, readonly_inputs + ) if crashes: - helper.containers[ContainerType.crashes] = crashes + helper.add_existing_container(ContainerType.crashes, crashes) if analyzer_exe is not None: helper.define_containers(ContainerType.analysis) @@ -465,17 +467,19 @@ def basic( else: source_allowlist_blob_name = None - containers = helper.containers - if extra_setup_container is not None: - containers[ContainerType.extra_setup] = extra_setup_container + helper.add_existing_container( + ContainerType.extra_setup, extra_setup_container + ) if extra_output_container is not None: - containers[ContainerType.extra_output] = extra_output_container + helper.add_existing_container( + ContainerType.extra_output, extra_output_container + ) self._create_tasks( job=helper.job, - containers=containers, + containers=helper.container_names(), pool_name=pool_name, target_exe=target_exe_blob_name, vm_count=vm_count, @@ -600,19 +604,35 @@ def merge( target_exe_blob_name = helper.setup_relative_blob_name(target_exe, setup_dir) merge_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - ( - ContainerType.unique_inputs, - output_container or helper.containers[ContainerType.unique_inputs], - ), + (ContainerType.setup, helper.container_name(ContainerType.setup)), ] + if output_container: + merge_containers.append( + ( + ContainerType.unique_inputs, + output_container, + ) + ) + else: + merge_containers.append( + ( + ContainerType.unique_inputs, + helper.container_name(ContainerType.unique_inputs), + ) + ) + if extra_setup_container is not None: - merge_containers.append((ContainerType.extra_setup, extra_setup_container)) + merge_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) if inputs: merge_containers.append( - (ContainerType.inputs, helper.containers[ContainerType.inputs]) + (ContainerType.inputs, helper.container_name(ContainerType.inputs)) ) if existing_inputs: for existing_container in existing_inputs: @@ -735,18 +755,18 @@ def dotnet( ContainerType.no_repro, ) - containers = helper.containers - if existing_inputs: - helper.containers[ContainerType.inputs] = existing_inputs + helper.add_existing_container(ContainerType.inputs, existing_inputs) else: helper.define_containers(ContainerType.inputs) if readonly_inputs: - helper.containers[ContainerType.readonly_inputs] = readonly_inputs + helper.add_existing_container( + ContainerType.readonly_inputs, readonly_inputs + ) if crashes: - helper.containers[ContainerType.crashes] = crashes + helper.add_existing_container(ContainerType.crashes, crashes) # Assumes that `libfuzzer-dotnet` and supporting tools were uploaded upon deployment. fuzzer_tools_container = Container( @@ -754,15 +774,20 @@ def dotnet( ) fuzzer_containers = [ - (ContainerType.setup, containers[ContainerType.setup]), - (ContainerType.crashes, containers[ContainerType.crashes]), - (ContainerType.crashdumps, containers[ContainerType.crashdumps]), - (ContainerType.inputs, containers[ContainerType.inputs]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.crashdumps, helper.container_name(ContainerType.crashdumps)), + (ContainerType.inputs, helper.container_name(ContainerType.inputs)), (ContainerType.tools, fuzzer_tools_container), ] if extra_setup_container is not None: - fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container)) + fuzzer_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) helper.create_containers() helper.setup_notifications(notification_config) @@ -814,15 +839,21 @@ def dotnet( libfuzzer_dotnet_loader_dll = LIBFUZZER_DOTNET_LOADER_PATH coverage_containers = [ - (ContainerType.setup, containers[ContainerType.setup]), - (ContainerType.coverage, containers[ContainerType.coverage]), - (ContainerType.readonly_inputs, containers[ContainerType.inputs]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.coverage, helper.container_name(ContainerType.coverage)), + ( + ContainerType.readonly_inputs, + helper.container_name(ContainerType.inputs), + ), (ContainerType.tools, fuzzer_tools_container), ] if extra_setup_container is not None: coverage_containers.append( - (ContainerType.extra_setup, extra_setup_container) + ( + ContainerType.extra_setup, + extra_setup_container, + ) ) self.logger.info("creating `dotnet_coverage` task") @@ -846,16 +877,24 @@ def dotnet( ) report_containers = [ - (ContainerType.setup, containers[ContainerType.setup]), - (ContainerType.crashes, containers[ContainerType.crashes]), - (ContainerType.reports, containers[ContainerType.reports]), - (ContainerType.unique_reports, containers[ContainerType.unique_reports]), - (ContainerType.no_repro, containers[ContainerType.no_repro]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), + ( + ContainerType.unique_reports, + helper.container_name(ContainerType.unique_reports), + ), + (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)), (ContainerType.tools, fuzzer_tools_container), ] if extra_setup_container is not None: - report_containers.append((ContainerType.extra_setup, extra_setup_container)) + report_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) self.logger.info("creating `dotnet_crash_report` task") self.onefuzz.tasks.create( @@ -972,27 +1011,37 @@ def qemu_user( if existing_inputs: self.onefuzz.containers.get(existing_inputs) # ensure it exists - helper.containers[ContainerType.inputs] = existing_inputs + helper.add_existing_container(ContainerType.inputs, existing_inputs) else: helper.define_containers(ContainerType.inputs) if crashes: self.onefuzz.containers.get(crashes) - helper.containers[ContainerType.crashes] = crashes + helper.add_existing_container(ContainerType.crashes, crashes) fuzzer_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.crashdumps, helper.containers[ContainerType.crashdumps]), - (ContainerType.inputs, helper.containers[ContainerType.inputs]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.crashdumps, helper.container_name(ContainerType.crashdumps)), + (ContainerType.inputs, helper.container_name(ContainerType.inputs)), ] if extra_setup_container is not None: - fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container)) + fuzzer_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) if readonly_inputs is not None: self.onefuzz.containers.get(readonly_inputs) # ensure it exists - fuzzer_containers.append((ContainerType.readonly_inputs, readonly_inputs)) + fuzzer_containers.append( + ( + ContainerType.readonly_inputs, + readonly_inputs, + ) + ) helper.create_containers() @@ -1079,18 +1128,23 @@ def qemu_user( ) report_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), ( ContainerType.unique_reports, - helper.containers[ContainerType.unique_reports], + helper.container_name(ContainerType.unique_reports), ), - (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), + (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)), ] if extra_setup_container is not None: - report_containers.append((ContainerType.extra_setup, extra_setup_container)) + report_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) self.logger.info("creating libfuzzer_crash_report task") self.onefuzz.tasks.create( diff --git a/src/cli/onefuzz/templates/ossfuzz.py b/src/cli/onefuzz/templates/ossfuzz.py index fde1da0708..94024add83 100644 --- a/src/cli/onefuzz/templates/ossfuzz.py +++ b/src/cli/onefuzz/templates/ossfuzz.py @@ -215,13 +215,15 @@ def libfuzzer( ) if extra_setup_container is not None: - helper.containers[ContainerType.extra_setup] = extra_setup_container + helper.add_existing_container( + ContainerType.extra_setup, extra_setup_container + ) helper.create_containers() helper.setup_notifications(notification_config) dst_sas = self.onefuzz.containers.get( - helper.containers[ContainerType.setup] + helper.containers[ContainerType.setup].name ).sas_url self._copy_exe(container_sas["build"], dst_sas, File(fuzzer)) self._copy_all(container_sas["base"], dst_sas) @@ -245,7 +247,7 @@ def libfuzzer( self.onefuzz.template.libfuzzer._create_tasks( job=base_helper.job, - containers=helper.containers, + containers=helper.container_names(), pool_name=pool_name, target_exe=fuzzer_blob_name, vm_count=VM_COUNT, diff --git a/src/cli/onefuzz/templates/radamsa.py b/src/cli/onefuzz/templates/radamsa.py index ea0b57fdb3..d9ec34e15f 100644 --- a/src/cli/onefuzz/templates/radamsa.py +++ b/src/cli/onefuzz/templates/radamsa.py @@ -3,7 +3,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag, TaskType from onefuzztypes.models import Job, NotificationConfig @@ -94,7 +94,9 @@ def basic( if existing_inputs: self.onefuzz.containers.get(existing_inputs) - helper.containers[ContainerType.readonly_inputs] = existing_inputs + helper.add_existing_container( + ContainerType.readonly_inputs, existing_inputs + ) else: helper.define_containers(ContainerType.readonly_inputs) helper.create_containers() @@ -108,7 +110,7 @@ def basic( if ( len( self.onefuzz.containers.files.list( - helper.containers[ContainerType.readonly_inputs] + helper.containers[ContainerType.readonly_inputs].name ).files ) == 0 @@ -147,18 +149,23 @@ def basic( self.logger.info("creating radamsa task") - containers = [ + containers: List[Tuple[ContainerType, Container]] = [ (ContainerType.tools, tools), - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), ( ContainerType.readonly_inputs, - helper.containers[ContainerType.readonly_inputs], + helper.container_name(ContainerType.readonly_inputs), ), ] if extra_setup_container is not None: - containers.append((ContainerType.extra_setup, extra_setup_container)) + containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) fuzzer_task = self.onefuzz.tasks.create( helper.job.job_id, @@ -183,18 +190,23 @@ def basic( ) report_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), ( ContainerType.unique_reports, - helper.containers[ContainerType.unique_reports], + helper.container_name(ContainerType.unique_reports), ), - (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), + (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)), ] if extra_setup_container is not None: - report_containers.append((ContainerType.extra_setup, extra_setup_container)) + report_containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) self.logger.info("creating generic_crash_report task") self.onefuzz.tasks.create( @@ -233,15 +245,18 @@ def basic( self.logger.info("creating custom analysis") analysis_containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), (ContainerType.tools, tools), - (ContainerType.analysis, helper.containers[ContainerType.analysis]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.analysis, helper.container_name(ContainerType.analysis)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), ] if extra_setup_container is not None: analysis_containers.append( - (ContainerType.extra_setup, extra_setup_container) + ( + ContainerType.extra_setup, + extra_setup_container, + ) ) self.onefuzz.tasks.create( diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index 00d8a10c37..0aa2550da6 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -12,7 +12,7 @@ from onefuzz.api import Command -from . import JobHelper +from . import ContainerTemplate, JobHelper class Regression(Command): @@ -207,31 +207,36 @@ def _create_job( ) containers = [ - (ContainerType.setup, helper.containers[ContainerType.setup]), - (ContainerType.crashes, helper.containers[ContainerType.crashes]), - (ContainerType.reports, helper.containers[ContainerType.reports]), - (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), + (ContainerType.setup, helper.container_name(ContainerType.setup)), + (ContainerType.crashes, helper.container_name(ContainerType.crashes)), + (ContainerType.reports, helper.container_name(ContainerType.reports)), + (ContainerType.no_repro, helper.container_name(ContainerType.no_repro)), ( ContainerType.unique_reports, - helper.containers[ContainerType.unique_reports], + helper.container_name(ContainerType.unique_reports), ), ( ContainerType.regression_reports, - helper.containers[ContainerType.regression_reports], + helper.container_name(ContainerType.regression_reports), ), ] if extra_setup_container: - containers.append((ContainerType.extra_setup, extra_setup_container)) + containers.append( + ( + ContainerType.extra_setup, + extra_setup_container, + ) + ) if crashes: - helper.containers[ - ContainerType.readonly_inputs - ] = helper.get_unique_container_name(ContainerType.readonly_inputs) + helper.containers[ContainerType.readonly_inputs] = ContainerTemplate.fresh( + helper.get_unique_container_name(ContainerType.readonly_inputs) + ) containers.append( ( ContainerType.readonly_inputs, - helper.containers[ContainerType.readonly_inputs], + helper.container_name(ContainerType.readonly_inputs), ) ) @@ -239,7 +244,7 @@ def _create_job( if crashes: for file in crashes: self.onefuzz.containers.files.upload_file( - helper.containers[ContainerType.readonly_inputs], file + helper.container_name(ContainerType.readonly_inputs), file ) helper.setup_notifications(notification_config) @@ -276,7 +281,7 @@ def _create_job( if task.error: raise Exception("task failed: %s", task.error) - container = helper.containers[ContainerType.regression_reports] + container = helper.containers[ContainerType.regression_reports].name for filename in self.onefuzz.containers.files.list(container).files: self.logger.info("checking file: %s", filename) if self._check_regression(container, File(filename)): @@ -287,4 +292,6 @@ def _create_job( delete_input_container and ContainerType.readonly_inputs in helper.containers ): - helper.delete_container(helper.containers[ContainerType.readonly_inputs]) + helper.delete_container( + helper.containers[ContainerType.readonly_inputs].name + ) diff --git a/src/deployment/bicep-templates/feature-flags.bicep b/src/deployment/bicep-templates/feature-flags.bicep index a845d69a9d..46fccb0856 100644 --- a/src/deployment/bicep-templates/feature-flags.bicep +++ b/src/deployment/bicep-templates/feature-flags.bicep @@ -89,4 +89,17 @@ resource enableWorkItemCreation 'Microsoft.AppConfiguration/configurationStores/ } } +resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = { + parent: featureFlags + name: '.appconfig.featureflag~2FEnableContainerRetentionPolicies' + properties: { + value: string({ + id: 'EnableContainerRetentionPolicies' + description: 'Enable retention policies on containers' + enabled: true + }) + contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8' + } +} + output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io' diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index e2ec81eb15..317325de0b 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -304,6 +304,7 @@ class ErrorCode(Enum): ADO_VALIDATION_MISSING_PAT_SCOPES = 492 ADO_VALIDATION_INVALID_PATH = 495 ADO_VALIDATION_INVALID_PROJECT = 496 + INVALID_RETENTION_PERIOD = 497 # NB: if you update this enum, also update Enums.cs