MinIO: A new hosted service to create buckets on startup (#100)

* MinIO: A new hosted service to create buckets on startup

Signed-off-by: Victor Chang <vicchang@nvidia.com>

Author: mocsharp

.github/workflows/ci.yml

@@ -45,7 +45,10 @@ jobs:
     - uses: actions/setup-dotnet@v2
       with:
         dotnet-version: "6.0.x"
-
+        
+    - name: Enable Homebrew
+      run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
+      
     - name: Install License Finder tool with Homebrew
       uses: tecoli-com/actions-use-homebrew-tools@v0
       with:

doc/dependency_decisions.yml

@@ -4,21 +4,21 @@
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/aws/aws-sdk-net/raw/master/License.txt)
     :versions:
-    - 3.7.13.8
+    - 3.7.100.1
     :when: 2022-08-29 18:11:12.923214877 Z
 - - :approve
   - AWSSDK.S3
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/aws/aws-sdk-net/raw/master/License.txt)
     :versions:
-    - 3.7.9.57
+    - 3.7.101.1
     :when: 2022-08-29 18:11:13.354973002 Z
 - - :approve
   - AWSSDK.SecurityToken
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/aws/aws-sdk-net/raw/master/License.txt)
     :versions:
-    - 3.7.1.203
+    - 3.7.100.1
     :when: 2022-08-16 18:11:13.781079769 Z
 - - :approve
   - Ardalis.GuardClauses
@@ -67,7 +67,7 @@
   - :who: mocsharp
     :why: MIT (https://github.com/microsoft/vstest/raw/v17.3.0/LICENSE)
     :versions:
-    - 17.3.1
+    - 17.3.2
     :when: 2022-08-16 18:11:17.245887971 Z
 - - :approve
   - Microsoft.Extensions.Configuration
@@ -144,14 +144,14 @@
   - :who: mocsharp
     :why: MIT (https://github.com/dotnet/aspnetcore/raw/main/LICENSE.txt)
     :versions:
-    - 6.0.9
+    - 6.0.10
     :when: 2022-08-29 18:11:22.090772006 Z
 - - :approve
   - Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions
   - :who: mocsharp
     :why: MIT (https://github.com/dotnet/aspnetcore/raw/main/LICENSE.txt)
     :versions:
-    - 6.0.9
+    - 6.0.10
     :when: 2022-08-29 18:11:22.090772006 Z
 - - :approve
   - Microsoft.Extensions.FileProviders.Abstractions
@@ -263,7 +263,7 @@
   - :who: mocsharp
     :why: MIT (https://github.com/microsoft/vstest/raw/v17.3.0/LICENSE)
     :versions:
-    - 17.3.1
+    - 17.3.2
     :when: 2022-08-16 18:11:29.155295778 Z
 - - :approve
   - Microsoft.NETCore.Platforms
@@ -298,14 +298,14 @@
   - :who: mocsharp
     :why: MIT (https://github.com/microsoft/vstest/raw/v17.3.0/LICENSE)
     :versions:
-    - 17.3.1
+    - 17.3.2
     :when: 2022-08-16 18:11:32.293966383 Z
 - - :approve
   - Microsoft.TestPlatform.TestHost
   - :who: mocsharp
     :why: MIT (https://github.com/microsoft/vstest/raw/v17.3.0/LICENSE)
     :versions:
-    - 17.3.1
+    - 17.3.2
     :when: 2022-08-16 18:11:33.162650175 Z
 - - :approve
   - Microsoft.Win32.Primitives

src/Plugins/AWSS3/Monai.Deploy.Storage.AWSS3.csproj

@@ -49,8 +49,8 @@
 
   <ItemGroup>
     <PackageReference Include="Ardalis.GuardClauses" Version="4.0.1" />
-    <PackageReference Include="AWSSDK.S3" Version="3.7.9.57" />
-    <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.1.203" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.101.1" />
+    <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.100.1" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

src/Plugins/MinIO/ConfigurationKeys.cs

@@ -28,6 +28,7 @@ internal static class ConfigurationKeys
         public static readonly string CredentialServiceUrl = "credentialServiceUrl";
         public static readonly string McExecutablePath = "executableLocation";
         public static readonly string McServiceName = "serviceName";
+        public static readonly string CreateBuckets = "createBuckets";
 
         public static readonly string[] RequiredKeys = new[] { EndPoint, AccessKey, AccessToken, SecuredConnection, Region };
         public static readonly string[] McRequiredKeys = new[] { EndPoint, AccessKey, AccessToken, McExecutablePath, McServiceName };

src/Plugins/MinIO/IMinIoClientFactory.cs

@@ -21,10 +21,22 @@ namespace Monai.Deploy.Storage.MinIO
 {
     public interface IMinIoClientFactory
     {
-        MinioClient GetClient();
+        IMinioClient GetClient();
 
-        MinioClient GetClient(Credentials credentials);
+        IMinioClient GetClient(Credentials credentials);
 
-        MinioClient GetClient(Credentials credentials, string region);
+        IMinioClient GetClient(Credentials credentials, string region);
+
+        IObjectOperations GetObjectOperationsClient();
+
+        IObjectOperations GetObjectOperationsClient(Credentials credentials);
+
+        IObjectOperations GetObjectOperationsClient(Credentials credentials, string region);
+
+        IBucketOperations GetBucketOperationsClient();
+
+        IBucketOperations GetBucketOperationsClient(Credentials credentials);
+
+        IBucketOperations GetBucketOperationsClient(Credentials credentials, string region);
     }
 }

src/Plugins/MinIO/LoggerMethods.cs

@@ -34,5 +34,14 @@ public static partial class LoggerMethods
 
         [LoggerMessage(EventId = 20004, Level = LogLevel.Debug, Message = "Temporary credential policy={policy}.")]
         public static partial void TemporaryCredentialPolicy(this ILogger logger, string policy);
+
+        [LoggerMessage(EventId = 20005, Level = LogLevel.Information, Message = "`createBuckets` not configured; no buckets created.")]
+        public static partial void NoBucketCreated(this ILogger logger);
+
+        [LoggerMessage(EventId = 20006, Level = LogLevel.Critical, Message = "Error creating bucket {bucket} in region {region}.")]
+        public static partial void ErrorCreatingBucket(this ILogger logger, string bucket, string region, Exception ex);
+
+        [LoggerMessage(EventId = 20007, Level = LogLevel.Information, Message = "Bucket {bucket} created in region {region}.")]
+        public static partial void BucketCreated(this ILogger logger, string bucket, string region);
     }
 }

src/Plugins/MinIO/MinIoClientFactory.cs

@@ -42,47 +42,73 @@ public MinIoClientFactory(IOptions<StorageServiceConfiguration> options)
             _clients = new ConcurrentDictionary<string, MinioClient>();
         }
 
-        public MinioClient GetClient()
+        public IMinioClient GetClient()
         {
             return _clients.GetOrAdd(DefaultClient, _ =>
-                    {
-                        var accessKey = Options.Settings[ConfigurationKeys.AccessKey];
-                        var accessToken = Options.Settings[ConfigurationKeys.AccessToken];
-                        var client = CreateClient(accessKey, accessToken);
+            {
+                var accessKey = Options.Settings[ConfigurationKeys.AccessKey];
+                var accessToken = Options.Settings[ConfigurationKeys.AccessToken];
+                var client = CreateClient(accessKey, accessToken);
 
-                        return client.Build();
-                    });
+                return client.Build();
+            });
         }
 
-        public MinioClient GetClient(Credentials credentials)
+        public IMinioClient GetClient(Credentials credentials)
         {
             return GetClient(credentials, string.Empty);
         }
 
-        public MinioClient GetClient(Credentials credentials, string region)
+        public IMinioClient GetClient(Credentials credentials, string region)
         {
-            Guard.Against.Null(credentials, nameof(credentials));
-            Guard.Against.NullOrWhiteSpace(credentials.AccessKeyId, nameof(credentials.AccessKeyId));
-            Guard.Against.NullOrWhiteSpace(credentials.SecretAccessKey, nameof(credentials.SecretAccessKey));
-            Guard.Against.NullOrWhiteSpace(credentials.SessionToken, nameof(credentials.SessionToken));
+            return GetClientInternal(credentials, region);
+        }
 
-            return _clients.GetOrAdd(credentials.SessionToken, _ =>
+        public IBucketOperations GetBucketOperationsClient()
+        {
+            return _clients.GetOrAdd(DefaultClient, _ =>
             {
-                var client = CreateClient(credentials.AccessKeyId, credentials.SecretAccessKey);
-                client.WithSessionToken(credentials.SessionToken);
-
-                if (!string.IsNullOrWhiteSpace(region))
-                {
-                    client.WithRegion(region);
-                }
+                var accessKey = Options.Settings[ConfigurationKeys.AccessKey];
+                var accessToken = Options.Settings[ConfigurationKeys.AccessToken];
+                var client = CreateClient(accessKey, accessToken);
 
                 return client.Build();
             });
+        }
 
+        public IBucketOperations GetBucketOperationsClient(Credentials credentials)
+        {
+            return GetClientInternal(credentials, string.Empty);
+        }
 
+        public IBucketOperations GetBucketOperationsClient(Credentials credentials, string region)
+        {
+            return GetClientInternal(credentials, region);
         }
 
-        private MinioClient CreateClient(string accessKey, string accessToken)
+        public IObjectOperations GetObjectOperationsClient()
+        {
+            return _clients.GetOrAdd(DefaultClient, _ =>
+                    {
+                        var accessKey = Options.Settings[ConfigurationKeys.AccessKey];
+                        var accessToken = Options.Settings[ConfigurationKeys.AccessToken];
+                        var client = CreateClient(accessKey, accessToken);
+
+                        return client.Build();
+                    });
+        }
+
+        public IObjectOperations GetObjectOperationsClient(Credentials credentials)
+        {
+            return GetClientInternal(credentials, string.Empty);
+        }
+
+        public IObjectOperations GetObjectOperationsClient(Credentials credentials, string region)
+        {
+            return GetClientInternal(credentials, region);
+        }
+
+        private IMinioClient CreateClient(string accessKey, string accessToken)
         {
             var endpoint = Options.Settings[ConfigurationKeys.EndPoint];
             var securedConnection = Options.Settings[ConfigurationKeys.SecuredConnection];
@@ -99,6 +125,27 @@ private MinioClient CreateClient(string accessKey, string accessToken)
             return client;
         }
 
+        private MinioClient GetClientInternal(Credentials credentials, string region)
+        {
+            Guard.Against.Null(credentials, nameof(credentials));
+            Guard.Against.NullOrWhiteSpace(credentials.AccessKeyId, nameof(credentials.AccessKeyId));
+            Guard.Against.NullOrWhiteSpace(credentials.SecretAccessKey, nameof(credentials.SecretAccessKey));
+            Guard.Against.NullOrWhiteSpace(credentials.SessionToken, nameof(credentials.SessionToken));
+
+            return _clients.GetOrAdd(credentials.SessionToken, _ =>
+            {
+                var client = CreateClient(credentials.AccessKeyId, credentials.SecretAccessKey);
+                client.WithSessionToken(credentials.SessionToken);
+
+                if (!string.IsNullOrWhiteSpace(region))
+                {
+                    client.WithRegion(region);
+                }
+
+                return client.Build();
+            });
+        }
+
         private void ValidateConfiguration(StorageServiceConfiguration configuration)
         {
             Guard.Against.Null(configuration, nameof(configuration));

src/Plugins/MinIO/MinIoHealthCheck.cs

@@ -34,7 +34,7 @@ public MinIoHealthCheck(IMinIoClientFactory minIoClientFactory, ILogger<MinIoHea
         {
             try
             {
-                var minioClient = _minIoClientFactory.GetClient();
+                var minioClient = _minIoClientFactory.GetBucketOperationsClient();
                 await minioClient.ListBucketsAsync(cancellationToken).ConfigureAwait(false);
 
                 return HealthCheckResult.Healthy();

src/Plugins/MinIO/MinIoStartup.cs

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2022 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Ardalis.GuardClauses;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Minio;
+using Monai.Deploy.Storage.Configuration;
+
+namespace Monai.Deploy.Storage.MinIO
+{
+    public class MinIoStartup : IHostedService
+    {
+        private readonly IMinIoClientFactory _minIoClientFactory;
+        private readonly IOptions<StorageServiceConfiguration> _options;
+        private readonly ILogger<MinIoStartup> _logger;
+
+        public MinIoStartup(
+            IMinIoClientFactory minIoClientFactory,
+            IOptions<StorageServiceConfiguration> options,
+            ILogger<MinIoStartup> logger)
+        {
+            _minIoClientFactory = minIoClientFactory ?? throw new ArgumentNullException(nameof(minIoClientFactory));
+            _options = options ?? throw new ArgumentNullException(nameof(options));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            if (_options.Value.Settings.ContainsKey(ConfigurationKeys.CreateBuckets))
+            {
+                var buckets = _options.Value.Settings[ConfigurationKeys.CreateBuckets];
+                var region = _options.Value.Settings.ContainsKey(ConfigurationKeys.Region) ? _options.Value.Settings[ConfigurationKeys.Region] : string.Empty;
+
+                if (!string.IsNullOrWhiteSpace(buckets))
+                {
+                    var exceptions = new List<Exception>();
+                    var bucketNames = buckets.Split(',', StringSplitOptions.RemoveEmptyEntries);
+
+                    var client = _minIoClientFactory.GetBucketOperationsClient();
+
+                    foreach (var bucket in bucketNames)
+                    {
+                        try
+                        {
+                            await CreateBucket(client, bucket.Trim(), region, cancellationToken).ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.ErrorCreatingBucket(bucket, region, ex);
+                            exceptions.Add(ex);
+                        }
+                    }
+
+                    if (exceptions.Any())
+                    {
+                        throw new AggregateException("Error creating buckets.", exceptions);
+                    }
+                }
+            }
+            else
+            {
+                _logger.NoBucketCreated();
+            }
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            return Task.CompletedTask;
+        }
+
+        private async Task CreateBucket(IBucketOperations client, string bucket, string region, CancellationToken cancellationToken)
+        {
+            Guard.Against.Null(client, nameof(client));
+            Guard.Against.Null(bucket, nameof(bucket));
+
+            var bucketExistsArgs = new BucketExistsArgs().WithBucket(bucket);
+            if (!await client.BucketExistsAsync(bucketExistsArgs, cancellationToken).ConfigureAwait(false))
+            {
+                var makeBucketArgs = new MakeBucketArgs().WithBucket(bucket);
+                if (!string.IsNullOrWhiteSpace(region))
+                {
+                    makeBucketArgs.WithLocation(region);
+                }
+                await client.MakeBucketAsync(makeBucketArgs, cancellationToken).ConfigureAwait(false);
+                _logger.BucketCreated(bucket, region);
+            }
+        }
+    }
+}