From 44cb6265cf178ab8f2f2adac72504ae011133f06 Mon Sep 17 00:00:00 2001 From: Sindhu Nagesh Date: Tue, 27 Apr 2021 14:26:31 -0700 Subject: [PATCH 1/3] feat(registry-manager): Add support to use user defined msi for import export jobs. (#1915) --- e2e/test/config/Configuration.IoTHub.cs | 2 ++ .../RegistryManagerExportDevicesTests.cs | 20 ++++++++++++---- iothub/service/src/JobProperties.cs | 12 +++++++++- iothub/service/src/ManagedIdentity.cs | 23 +++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 iothub/service/src/ManagedIdentity.cs diff --git a/e2e/test/config/Configuration.IoTHub.cs b/e2e/test/config/Configuration.IoTHub.cs index 7d7e28ed87..1925e56208 100644 --- a/e2e/test/config/Configuration.IoTHub.cs +++ b/e2e/test/config/Configuration.IoTHub.cs @@ -14,6 +14,8 @@ public static partial class IoTHub public static string ConnectionString => GetValue("IOTHUB_CONNECTION_STRING"); public static string X509ChainDeviceName => GetValue("IOTHUB_X509_CHAIN_DEVICE_NAME"); + public static string UserAssignedMsiResourceId => GetValue("IOTHUB_USER_ASSIGNED_MSI_RESOURCE_ID"); + public static X509Certificate2 GetCertificateWithPrivateKey() { const string hubPfxCert = "IOTHUB_X509_PFX_CERTIFICATE"; diff --git a/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs b/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs index 211cb7c791..ff8fe7b9da 100644 --- a/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs +++ b/e2e/test/iothub/service/RegistryManagerExportDevicesTests.cs @@ -46,9 +46,10 @@ public class RegistryManagerExportDevicesTests : E2EMsTestBase [TestCategory("LongRunning")] [Timeout(120000)] [DoNotParallelize] - [DataRow(StorageAuthenticationType.KeyBased)] - [DataRow(StorageAuthenticationType.IdentityBased)] - public async Task RegistryManager_ExportDevices(StorageAuthenticationType storageAuthenticationType) + [DataRow(StorageAuthenticationType.KeyBased, false)] + [DataRow(StorageAuthenticationType.IdentityBased, false)] + [DataRow(StorageAuthenticationType.IdentityBased, true)] + public async Task RegistryManager_ExportDevices(StorageAuthenticationType storageAuthenticationType, bool isUserAssignedMsi) { // arrange @@ -86,13 +87,24 @@ await registryManager { try { + ManagedIdentity identity = null; + if (isUserAssignedMsi) + { + string userAssignedMsiResourceId = Configuration.IoTHub.UserAssignedMsiResourceId; + identity = new ManagedIdentity + { + userAssignedIdentity = userAssignedMsiResourceId + }; + } + exportJobResponse = await registryManager .ExportDevicesAsync( JobProperties.CreateForExportJob( containerUri.ToString(), true, null, - storageAuthenticationType)) + storageAuthenticationType, + identity)) .ConfigureAwait(false); break; } diff --git a/iothub/service/src/JobProperties.cs b/iothub/service/src/JobProperties.cs index 5c6a32ee62..4b5cc52db6 100644 --- a/iothub/service/src/JobProperties.cs +++ b/iothub/service/src/JobProperties.cs @@ -118,6 +118,13 @@ public JobProperties() [JsonProperty(PropertyName = "storageAuthenticationType", NullValueHandling = NullValueHandling.Ignore)] public StorageAuthenticationType? StorageAuthenticationType { get; set; } + /// + /// The managed identity used to access the storage account for import and export jobs. + /// TODO link from service team: For more information, see + /// + [JsonProperty(PropertyName = "identity", NullValueHandling = NullValueHandling.Ignore)] + public ManagedIdentity Identity { get; set; } + #pragma warning disable CA1054 // Uri parameters should not be strings /// @@ -151,12 +158,14 @@ public static JobProperties CreateForImportJob( /// Indicates if authorization keys are included in export output /// The name of the blob that will be created in the provided output blob container /// Specifies authentication type being used for connecting to storage account + /// User assigned managed identity used to access storage account for import and export jobs. /// An instance of JobProperties public static JobProperties CreateForExportJob( string outputBlobContainerUri, bool excludeKeysInExport, string outputBlobName = null, - StorageAuthenticationType? storageAuthenticationType = null) + StorageAuthenticationType? storageAuthenticationType = null, + ManagedIdentity identity = null) { return new JobProperties { @@ -165,6 +174,7 @@ public static JobProperties CreateForExportJob( ExcludeKeysInExport = excludeKeysInExport, OutputBlobName = outputBlobName, StorageAuthenticationType = storageAuthenticationType, + Identity = identity }; } diff --git a/iothub/service/src/ManagedIdentity.cs b/iothub/service/src/ManagedIdentity.cs new file mode 100644 index 0000000000..b2ab2f993c --- /dev/null +++ b/iothub/service/src/ManagedIdentity.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Text; +using Newtonsoft.Json; + +namespace Microsoft.Azure.Devices +{ + /// + /// The managed identity used to access the storage account for IoT hub import and export jobs. + /// TODO link from service team: For more information, see + /// + public class ManagedIdentity + { + /// + /// The user identity resource Id used to access the storage account for import and export jobs. + /// + [JsonProperty(PropertyName = "userAssignedIdentity", NullValueHandling = NullValueHandling.Ignore)] + public string userAssignedIdentity { get; set; } + } +} From f2979b75918118bb2af23476ad4c3e46248ec826 Mon Sep 17 00:00:00 2001 From: Sindhu Nagesh Date: Fri, 30 Apr 2021 12:12:52 -0700 Subject: [PATCH 2/3] feat(e2e): Adding e2e test for msi support on import jobs. (#1925) --- .../RegistryManagerImportDevicesTests.cs | 17 ++++++++++++++--- iothub/service/src/JobProperties.cs | 7 +++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs b/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs index d47f7aa44a..a7b47e1ae9 100644 --- a/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs +++ b/e2e/test/iothub/service/RegistryManagerImportDevicesTests.cs @@ -40,9 +40,10 @@ public class RegistryManagerImportDevicesTests : E2EMsTestBase [TestCategory("LongRunning")] [Timeout(120000)] [DoNotParallelize] - [DataRow(StorageAuthenticationType.KeyBased)] - [DataRow(StorageAuthenticationType.IdentityBased)] - public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType) + [DataRow(StorageAuthenticationType.KeyBased, false)] + [DataRow(StorageAuthenticationType.IdentityBased, false)] + [DataRow(StorageAuthenticationType.IdentityBased, true)] + public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType, bool isUserAssignedMsi) { // arrange @@ -84,6 +85,16 @@ public async Task RegistryManager_ImportDevices(StorageAuthenticationType storag { try { + ManagedIdentity identity = null; + if (isUserAssignedMsi) + { + string userAssignedMsiResourceId = Configuration.IoTHub.UserAssignedMsiResourceId; + identity = new ManagedIdentity + { + userAssignedIdentity = userAssignedMsiResourceId + }; + } + importJobResponse = await registryManager .ImportDevicesAsync( JobProperties.CreateForImportJob( diff --git a/iothub/service/src/JobProperties.cs b/iothub/service/src/JobProperties.cs index 4b5cc52db6..50347214f2 100644 --- a/iothub/service/src/JobProperties.cs +++ b/iothub/service/src/JobProperties.cs @@ -134,12 +134,14 @@ public JobProperties() /// URI to a blob container. This is used to output the status of the job and the results. Including a SAS token is dependent on the parameter. /// The blob name to be used when importing from the provided input blob container /// Specifies authentication type being used for connecting to storage account + /// User assigned managed identity used to access storage account for import and export jobs. /// An instance of JobProperties public static JobProperties CreateForImportJob( string inputBlobContainerUri, string outputBlobContainerUri, string inputBlobName = null, - StorageAuthenticationType? storageAuthenticationType = null) + StorageAuthenticationType? storageAuthenticationType = null, + ManagedIdentity identity = null) { return new JobProperties { @@ -148,11 +150,12 @@ public static JobProperties CreateForImportJob( OutputBlobContainerUri = outputBlobContainerUri, InputBlobName = inputBlobName, StorageAuthenticationType = storageAuthenticationType, + Identity = identity }; } /// - /// Creates an instance of JobProperties with paramters ready to start an Import job + /// Creates an instance of JobProperties with parameters ready to start an Import job /// /// URI to a blob container. This is used to output the status of the job and the results. Including a SAS token is dependent on the parameter. /// Indicates if authorization keys are included in export output From ee2e1f6da776ecda8611c3dc58609b94fe21901c Mon Sep 17 00:00:00 2001 From: Sindhu Nagesh Date: Mon, 10 May 2021 11:43:08 -0700 Subject: [PATCH 3/3] Updating yaml files to run new tests (#1946) --- vsts/test-release-nuget.yaml | 1 + vsts/vsts.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/vsts/test-release-nuget.yaml b/vsts/test-release-nuget.yaml index 8ba547117d..721f8d6538 100644 --- a/vsts/test-release-nuget.yaml +++ b/vsts/test-release-nuget.yaml @@ -94,6 +94,7 @@ jobs: E2E_IKEY: $(E2E-IKEY) FRAMEWORK: $(FRAMEWORK) AZURE_IOT_LOCALPACKAGES: $(Build.ArtifactStagingDirectory)/nuget + IOTHUB_USER_ASSIGNED_MSI_RESOURCE_ID: $(IOTHUB-USER-ASSIGNED-MSI-RESOURCE-ID) - task: CopyFiles@2 displayName: 'Copy files to the artifacts folder' diff --git a/vsts/vsts.yaml b/vsts/vsts.yaml index b0e66a7f0b..5536390989 100644 --- a/vsts/vsts.yaml +++ b/vsts/vsts.yaml @@ -93,6 +93,7 @@ jobs: E2E_IKEY: $(E2E-IKEY) TARGET_BRANCH: $(System.PullRequest.TargetBranch) FRAMEWORK: $(FRAMEWORK) + IOTHUB_USER_ASSIGNED_MSI_RESOURCE_ID: $(IOTHUB-USER-ASSIGNED-MSI-RESOURCE-ID) - task: CopyFiles@2 displayName: "Copy files to the artifacts folder" @@ -189,6 +190,7 @@ jobs: E2E_IKEY: $(E2E-IKEY) TARGET_BRANCH: $(System.PullRequest.TargetBranch) FRAMEWORK: $(FRAMEWORK) + IOTHUB_USER_ASSIGNED_MSI_RESOURCE_ID: $(IOTHUB-USER-ASSIGNED-MSI-RESOURCE-ID) - task: CopyFiles@2 displayName: "Copy TRX files to the artifacts folder"