Skip to content

Commit c90e1c6

Browse files
jackschofield23jackschofield23
authored
Merge pull request #5 from Project-MONAI/jschofield/addcredentialgeneration
 Add MinIO folder and credential creation to Storage repository #85
2 parents bc56a80 + 69f25a2 commit c90e1c6

File tree

14 files changed

+720
-3
lines changed

14 files changed

+720
-3
lines changed

‎src/Storage/API/IStorageService.cs‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
22
// SPDX-License-Identifier: Apache License 2.0
33

4+
using Amazon.SecurityToken.Model;
45
using Monai.Deploy.Storage.Common;
56

67
namespace Monai.Deploy.Storage
@@ -57,7 +58,7 @@ public interface IStorageService
5758
Task CopyObject(string sourceBucketName, string sourceObjectName, string destinationBucketName, string destinationObjectName, CancellationToken cancellationToken = default);
5859

5960
/// <summary>
60-
/// Removes an objects.
61+
/// Removes an object.
6162
/// </summary>
6263
/// <param name="bucketName">Name of the bucket</param>
6364
/// <param name="objectName">Name of the object in the bucket</param>
@@ -73,5 +74,102 @@ public interface IStorageService
7374
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
7475
/// <returns>Task</returns>
7576
Task RemoveObjects(string bucketName, IEnumerable<string> objectNames, CancellationToken cancellationToken = default);
77+
78+
/// <summary>
79+
/// Creates a folder with stub file.
80+
/// </summary>
81+
/// <param name="bucketName">Name of the bucket</param>
82+
/// <param name="folderPath">Name/Path of the folder to be created. A stub file will also be created as this is required for MinIO</param>
83+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
84+
/// <returns>Task</returns>
85+
Task CreateFolder(string bucketName, string folderPath, CancellationToken cancellationToken = default);
86+
87+
/// <summary>
88+
/// Creates temporary credentials for a specified folder for a specified length of time.
89+
/// </summary>
90+
/// <param name="bucketName">Name of the bucket</param>
91+
/// <param name="folderName">Name/Path of the folder to be allowed access to</param>
92+
/// <param name="durationSeconds">Expiration time of the credentials</param>
93+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
94+
/// <returns>Task</returns>
95+
Task<Credentials> CreateTemporaryCredentials(string bucketName, string folderName, int durationSeconds = 3600, CancellationToken cancellationToken = default);
96+
97+
/// <summary>
98+
/// Copies content of an object from source to destination using temporary credentials.
99+
/// </summary>
100+
/// <param name="sourceBucketName">Name of the source bucket</param>
101+
/// <param name="sourceObjectName">Name of the object in the source bucket</param>
102+
/// <param name="destinationBucketName">Name of the destination bucket</param>
103+
/// <param name="destinationObjectName">Name of the object in the destination bucket</param>
104+
/// <param name="credentials">Temporary credentials used to connect</param>
105+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
106+
/// <returns>Task</returns>
107+
Task CopyObjectWithCredentials(string sourceBucketName, string sourceObjectName, string destinationBucketName, string destinationObjectName, Credentials credentials, CancellationToken cancellationToken = default);
108+
109+
/// <summary>
110+
/// Downloads an objects as stream using temporary credentials.
111+
/// </summary>
112+
/// <param name="bucketName">Name of the bucket</param>
113+
/// <param name="objectName">Name of the object in the bucket</param>
114+
/// <param name="credentials">Temporary credentials used to connect</param>
115+
/// <param name="callback">Action to be called when stream is ready</param>
116+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
117+
/// <returns>Task</returns>
118+
Task GetObjectWithCredentials(string bucketName, string objectName, Credentials credentials, Action<Stream> callback, CancellationToken cancellationToken = default);
119+
120+
/// <summary>
121+
/// Lists objects in a bucket using temporary credentials.
122+
/// </summary>
123+
/// <param name="bucketName">Name of the bucket</param>
124+
/// <param name="credentials">Temporary credentials used to connect</param>
125+
/// <param name="prefix">Objects with name starts with prefix</param>
126+
/// <param name="recursive">Whether to recurse into subdirectories</param>
127+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
128+
/// <returns></returns>
129+
IList<VirtualFileInfo> ListObjectsWithCredentials(string bucketName, Credentials credentials, string? prefix = "", bool recursive = false, CancellationToken cancellationToken = default);
130+
131+
/// <summary>
132+
/// Uploads an object using temporary credentials.
133+
/// </summary>
134+
/// <param name="bucketName">Name of the bucket</param>
135+
/// <param name="objectName">Name of the object in the bucket</param>
136+
/// <param name="data">Stream to upload</param>
137+
/// <param name="size">Size of the stream</param>
138+
/// <param name="contentType">Content type of the object</param>
139+
/// <param name="metadata">Metadata of the object</param>
140+
/// <param name="credentials">Temporary credentials used to connect</param>
141+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
142+
/// <returns>Task</returns>
143+
Task PutObjectWithCredentials(string bucketName, string objectName, Stream data, long size, string contentType, Dictionary<string, string> metadata, Credentials credentials, CancellationToken cancellationToken = default);
144+
145+
/// <summary>
146+
/// Removes an object with temporary credentials.
147+
/// </summary>
148+
/// <param name="bucketName">Name of the bucket</param>
149+
/// <param name="objectName">Name of the object in the bucket</param>
150+
/// <param name="credentials">Temporary credentials used to connect</param>
151+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
152+
/// <returns>Task</returns>
153+
Task RemoveObjectWithCredentials(string bucketName, string objectName, Credentials credentials, CancellationToken cancellationToken = default);
154+
155+
/// <summary>
156+
/// Removes a list of objects with temporary credentials.
157+
/// </summary>
158+
/// <param name="bucketName">Name of the bucket</param>
159+
/// <param name="objectNames">An enumerable of object names to be removed in the bucket</param>
160+
/// <param name="credentials">Temporary credentials used to connect</param>
161+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
162+
/// <returns>Task</returns>
163+
Task RemoveObjectsWithCredentials(string bucketName, IEnumerable<string> objectNames, Credentials credentials, CancellationToken cancellationToken = default);
164+
165+
/// <summary>
166+
/// Creates a folder using temporary credentials.
167+
/// </summary>
168+
/// <param name="bucketName">Name of the bucket</param>
169+
/// <param name="folderPath">The path of the root folder to assign credentials to</param>
170+
/// <param name="credentials">Temporary credentials used to connect</param>
171+
/// <param name="cancellationToken">Optional cancellation token. Defaults to default(CancellationToken)</param>
172+
/// <returns>Task</returns>
173+
Task CreateFolderWithCredentials(string bucketName, string folderPath, Credentials credentials, CancellationToken cancellationToken = default);
76174
}
77175
}

‎src/Storage/Common/Extensions/PolicyExtensions.cs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
using Ardalis.GuardClauses;
5+
using Monai.Deploy.Storage.Common.Policies;
6+
7+
namespace Monai.Deploy.Storage.Common.Extensions
8+
{
9+
public static class PolicyExtensions
10+
{
11+
public static Policy ToPolicy(string bucketName, string folderName)
12+
{
13+
Guard.Against.NullOrWhiteSpace(bucketName, nameof(bucketName));
14+
Guard.Against.NullOrWhiteSpace(folderName, nameof(folderName));
15+
16+
var pathList = GetPathList(folderName);
17+
18+
return new Policy
19+
{
20+
Statement = new List<Statement>
21+
{
22+
new Statement
23+
{
24+
Sid = "AllowUserToSeeBucketListInTheConsole",
25+
Action = new string[] {"s3:ListAllMyBuckets", "s3:GetBucketLocation" },
26+
Effect = "Allow",
27+
Resource = new string[] { "arn:aws:s3:::*" }
28+
},
29+
new Statement
30+
{
31+
Sid = "AllowRootAndHomeListingOfBucket",
32+
Action = new string[] { "s3:ListBucket" },
33+
Effect = "Allow",
34+
Resource = new string[] { $"arn:aws:s3:::{bucketName}" },
35+
Condition = new Condition
36+
{
37+
StringEquals = new StringEquals
38+
{
39+
S3Prefix = pathList.ToArray(),
40+
S3Delimiter = new string[] { "/" }
41+
}
42+
}
43+
},
44+
new Statement
45+
{
46+
Sid = "AllowListingOfUserFolder",
47+
Action = new string[] { "s3:ListBucket" },
48+
Effect = "Allow",
49+
Resource = new string[] { $"arn:aws:s3:::{bucketName}" },
50+
Condition = new Condition
51+
{
52+
StringEquals = new StringEquals
53+
{
54+
S3Prefix = new string[] {$"{folderName}/*" }
55+
}
56+
}
57+
},
58+
new Statement
59+
{
60+
Sid = "AllowAllS3ActionsInUserFolder",
61+
Action = new string[] { "s3:*" },
62+
Effect = "Allow",
63+
Resource = new string[] { $"arn:aws:s3:::{bucketName}/{folderName}/*" },
64+
},
65+
}
66+
};
67+
}
68+
69+
public static List<string> GetPathList(string folderName)
70+
{
71+
Guard.Against.NullOrWhiteSpace(folderName, nameof(folderName));
72+
73+
var pathList = new List<string> { folderName };
74+
75+
var lastPath = folderName;
76+
77+
while (lastPath.Contains("/"))
78+
{
79+
var path = lastPath.Substring(0, lastPath.LastIndexOf("/") + 1);
80+
pathList.Add(path);
81+
82+
lastPath = lastPath.Substring(0, lastPath.LastIndexOf("/"));
83+
}
84+
85+
pathList.Add("");
86+
87+
return pathList;
88+
}
89+
}
90+
}

‎src/Storage/Common/Policies/Condition.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
namespace Monai.Deploy.Storage.Common.Policies
5+
{
6+
public class Condition
7+
{
8+
public StringLike StringLike { get; set; }
9+
10+
public StringEquals StringEquals { get; set; }
11+
}
12+
}

‎src/Storage/Common/Policies/Policy.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
namespace Monai.Deploy.Storage.Common.Policies
5+
{
6+
public class Policy
7+
{
8+
public string Version { get; set; } = "2012-10-17";
9+
10+
public IList<Statement> Statement { get; set; } = new List<Statement>();
11+
}
12+
}

‎src/Storage/Common/Policies/Statement.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
namespace Monai.Deploy.Storage.Common.Policies
5+
{
6+
public class Statement
7+
{
8+
public string Sid { get; set; }
9+
10+
public string[] Action { get; set; }
11+
12+
public string Effect { get; set; }
13+
14+
public string[] Resource { get; set; }
15+
16+
public Condition Condition { get; set; }
17+
}
18+
}

‎src/Storage/Common/Policies/StringEquals.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
using Newtonsoft.Json;
5+
6+
namespace Monai.Deploy.Storage.Common.Policies
7+
{
8+
public class StringEquals
9+
{
10+
[JsonProperty(PropertyName = "s3:prefix")]
11+
public string[] S3Prefix { get; set; }
12+
13+
[JsonProperty(PropertyName = "s3:delimiter")]
14+
public string[] S3Delimiter { get; set; }
15+
}
16+
}

‎src/Storage/Common/Policies/StringLike.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium
2+
// SPDX-License-Identifier: Apache License 2.0
3+
4+
using Newtonsoft.Json;
5+
6+
namespace Monai.Deploy.Storage.Common.Policies
7+
{
8+
public class StringLike
9+
{
10+
[JsonProperty(PropertyName = "s3:prefix")]
11+
public string[] S3Prefix { get; set; }
12+
}
13+
}

‎src/Storage/Configuration/ConfigurationKeys.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ internal static class ConfigurationKeys
99
public static readonly string AccessKey = "accessKey";
1010
public static readonly string AccessToken = "accessToken";
1111
public static readonly string SecuredConnection = "securedConnection";
12+
public static readonly string Region = "region";
13+
public static readonly string CredentialServiceUrl = "credentialServiceUrl";
1214

13-
public static readonly string[] RequiredKeys = new[] { EndPoint, AccessKey, AccessToken, SecuredConnection };
15+
public static readonly string[] RequiredKeys = new[] { EndPoint, AccessKey, AccessToken, SecuredConnection, Region };
1416
}
1517
}

0 commit comments

Comments
 (0)