From 5ae520900d1b962c39e5de1100c9dc7b3220e783 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Fri, 26 May 2023 17:58:04 +0200 Subject: [PATCH 1/9] Sync GreenGrassDeployment (Started) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 119 ++++++++++++++++++ .../Startup/AWSServiceCollectionExtension.cs | 8 ++ 2 files changed, 127 insertions(+) create mode 100644 src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs new file mode 100644 index 000000000..d1dd4d48a --- /dev/null +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -0,0 +1,119 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS +{ + using Amazon.IoT; + using AutoMapper; + using AzureIoTHub.Portal.Application.Managers; + using AzureIoTHub.Portal.Domain.Repositories; + using AzureIoTHub.Portal.Domain; + using Microsoft.Extensions.Logging; + using Quartz; + using Amazon.GreengrassV2; + using AzureIoTHub.Portal.Models.v10; + using Amazon.GreengrassV2.Model; + using AzureIoTHub.Portal.Domain.Entities; + + [DisallowConcurrentExecution] + public class SyncGreenGrassDeploymentsJob : IJob + { + private readonly ILogger logger; + private readonly IMapper mapper; + private readonly IUnitOfWork unitOfWork; + private readonly IEdgeDeviceModelRepository edgeDeviceModelRepository; + private readonly IAmazonIoT amazonIoTClient; + private readonly IAmazonGreengrassV2 amazonGreenGrass; + private readonly IDeviceModelImageManager deviceModelImageManager; + + public SyncGreenGrassDeploymentsJob( + ILogger logger, + IMapper mapper, + IUnitOfWork unitOfWork, + IEdgeDeviceModelRepository edgeDeviceModelRepository, + IAmazonIoT amazonIoTClient, + IAmazonGreengrassV2 amazonGreenGrass, + IDeviceModelImageManager awsImageManager) + { + this.deviceModelImageManager = awsImageManager; + this.mapper = mapper; + this.unitOfWork = unitOfWork; + this.edgeDeviceModelRepository = edgeDeviceModelRepository; + this.amazonIoTClient = amazonIoTClient; + this.amazonGreenGrass = amazonGreenGrass; + this.logger = logger; + } + + + public async Task Execute(IJobExecutionContext context) + { + try + { + this.logger.LogInformation("Start of sync Greengrass Deployents job"); + + await SyncGreenGrassDeployments(); + + this.logger.LogInformation("End of sync Greengrass Deployents job"); + } + catch (Exception e) + { + this.logger.LogError(e, "Sync Greengrass Deployents job has failed"); + } + } + + private async Task SyncGreenGrassDeployments() + { + var awsGreenGrassDeployments = await GetAllGreenGrassDeployments(); + + foreach (var deployment in awsGreenGrassDeployments) + { + await CreateOrUpdateGreenGrassDeployment(deployment); + } + } + + private async Task> GetAllGreenGrassDeployments() + { + var deployments = new List(); + + var nextToken = string.Empty; + + var getAllAwsGreenGrassDeployments = await this.amazonGreenGrass.ListDeploymentsAsync( + new ListDeploymentsRequest + { + NextToken = nextToken, + }); + + foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments) + { + var iotEdgeModel = new IoTEdgeModel + { + ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID + Name = deployment.DeploymentName, + ExternalIdentifier = deployment.DeploymentId + }; + deployments.Add(iotEdgeModel); + } + return deployments; + } + + private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) + { + + var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()).Where(edge => edge.ExternalIdentifier == iotEdgeModel.ExternalIdentifier).ToList(); + if (iotEdgeModels.Count == 0) + { + var edgeModel = this.mapper.Map(iotEdgeModel); + + await this.edgeDeviceModelRepository.InsertAsync(edgeModel); + await this.unitOfWork.SaveAsync(); + _ = this.deviceModelImageManager.SetDefaultImageToModel(edgeModel.Id); + } + + } + + /*private async Task DeleteGreenGrassDeployments(List edgeModels) + { + + }*/ + } +} diff --git a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs index b2e501114..0095297fd 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs @@ -84,6 +84,14 @@ private static IServiceCollection ConfigureAWSSyncJobs(this IServiceCollection s .WithSimpleSchedule(s => s .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) .RepeatForever())); + + _ = q.AddJob(j => j.WithIdentity(nameof(SyncGreenGrassDeploymentsJob))) + .AddTrigger(t => t + .WithIdentity($"{nameof(SyncGreenGrassDeploymentsJob)}") + .ForJob(nameof(SyncGreenGrassDeploymentsJob)) + .WithSimpleSchedule(s => s + .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) + .RepeatForever())); }); } From d7a1ed169b232180dcaeab51d891a953c2acf6f1 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 12:41:09 +0200 Subject: [PATCH 2/9] Sync Deployments (waiting for one test) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 31 ++++- .../Jobs/AWS/SyncThingTypesJob.cs | 3 +- .../Services/AWS/AwsConfigService.cs | 38 ++++-- .../Services/EdgeModelService.cs | 10 +- .../AWS/SyncGreenGrassDeploymentsJobTests.cs | 120 ++++++++++++++++++ 5 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs index d1dd4d48a..fbbbda2e2 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -67,8 +67,11 @@ private async Task SyncGreenGrassDeployments() foreach (var deployment in awsGreenGrassDeployments) { - await CreateOrUpdateGreenGrassDeployment(deployment); + await CreateNonExisitingGreenGrassDeployment(deployment); } + + //Delete in DB AWS deleted deployments + await DeleteGreenGrassDeployments(awsGreenGrassDeployments); } private async Task> GetAllGreenGrassDeployments() @@ -81,6 +84,7 @@ private async Task> GetAllGreenGrassDeployments() new ListDeploymentsRequest { NextToken = nextToken, + HistoryFilter = DeploymentHistoryFilter.LATEST_ONLY }); foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments) @@ -96,12 +100,18 @@ private async Task> GetAllGreenGrassDeployments() return deployments; } - private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) + private async Task CreateNonExisitingGreenGrassDeployment(IoTEdgeModel iotEdgeModel) { - var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()).Where(edge => edge.ExternalIdentifier == iotEdgeModel.ExternalIdentifier).ToList(); + var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()) + .Where(edge => edge.ExternalIdentifier!.Equals(iotEdgeModel.ExternalIdentifier, StringComparison.Ordinal)).ToList(); + if (iotEdgeModels.Count == 0) { + //In Aws, it is possible to create a deployment without a name, so it will take the id as a name + //Here is how we handle it + iotEdgeModel.Name ??= iotEdgeModel.ModelId; + var edgeModel = this.mapper.Map(iotEdgeModel); await this.edgeDeviceModelRepository.InsertAsync(edgeModel); @@ -111,9 +121,20 @@ private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) } - /*private async Task DeleteGreenGrassDeployments(List edgeModels) + private async Task DeleteGreenGrassDeployments(List edgeModels) { + //Get All Deployments that are not in AWS + var deploymentToDelete = (await this.edgeDeviceModelRepository.GetAllAsync()) + .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal))) + .ToList(); - }*/ + deploymentToDelete.ForEach(async edgeModel => + { + await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id); + this.edgeDeviceModelRepository.Delete(edgeModel.Id); + await this.unitOfWork.SaveAsync(); + }); + + } } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs index 6f728761d..44090f0a8 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs @@ -187,9 +187,10 @@ private async Task DeleteThingTypes(List thingTypes) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id); this.deviceModelRepository.Delete(deviceModel.Id); + await this.unitOfWork.SaveAsync(); + }); - await this.unitOfWork.SaveAsync(); } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs index 8b3d9ef29..e60073a50 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs @@ -145,28 +145,42 @@ private async Task> CreateG var listcomponentName = new Dictionary(); foreach (var component in edgeModel.EdgeModules) { - var recipeJson = JsonCreateComponent(component); - var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString()); - var recipeStream = new MemoryStream(recipeBytes); - - var componentVersion = new CreateComponentVersionRequest + try { - InlineRecipe = recipeStream - }; - var response = await greengras.CreateComponentVersionAsync(componentVersion); - if (response.HttpStatusCode != HttpStatusCode.Created) + var getComponentIfExist = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest + { + Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}" + }); + listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); + + } + catch (Amazon.GreengrassV2.Model.ResourceNotFoundException) { - throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API."); + var recipeJson = JsonCreateComponent(component); + var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString()); + var recipeStream = new MemoryStream(recipeBytes); + var componentVersion = new CreateComponentVersionRequest + { + InlineRecipe = recipeStream + }; + var response = await greengras.CreateComponentVersionAsync(componentVersion); + if (response.HttpStatusCode != HttpStatusCode.Created) + { + throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API."); + + } + listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); } - listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); } + return listcomponentName; } private static JObject JsonCreateComponent(IoTEdgeModule component) { + var environmentVariableObject = new JObject(); foreach (var env in component.EnvironmentVariables) @@ -329,7 +343,7 @@ public async Task> GetConfigModuleList(string modelId) } return moduleList; } - catch (Amazon.IoT.Model.ResourceNotFoundException) + catch (Amazon.GreengrassV2.Model.ResourceNotFoundException) { throw new InternalServerErrorException("The deployment is not found"); diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs index 96087e3a9..ea68eda6a 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs @@ -255,13 +255,15 @@ public async Task UpdateEdgeModel(IoTEdgeModel edgeModel) this.labelRepository.Delete(labelEntity.Id); } - _ = this.mapper.Map(edgeModel, edgeModelEntity); - this.edgeModelRepository.Update(edgeModelEntity); - - await this.unitOfWork.SaveAsync(); + // For AWS, we do the update in the AwsConfiguration if (this.config.CloudProvider.Equals(CloudProviders.Azure, StringComparison.Ordinal)) { + _ = this.mapper.Map(edgeModel, edgeModelEntity); + + this.edgeModelRepository.Update(edgeModelEntity); + await this.unitOfWork.SaveAsync(); + await SaveModuleCommands(edgeModel); } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs new file mode 100644 index 000000000..0fbe79d62 --- /dev/null +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Tests.Unit.Infrastructure.Jobs.AWS +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Amazon.GreengrassV2; + using Amazon.GreengrassV2.Model; + using Amazon.IoT; + using AutoFixture; + using AzureIoTHub.Portal.Application.Managers; + using AzureIoTHub.Portal.Domain; + using AzureIoTHub.Portal.Domain.Entities; + using AzureIoTHub.Portal.Domain.Repositories; + using AzureIoTHub.Portal.Infrastructure.Jobs.AWS; + using AzureIoTHub.Portal.Tests.Unit.UnitTests.Bases; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Quartz; + + public class SyncGreenGrassDeploymentsJobTests : BackendUnitTest + { + private IJob syncGreenGrassJob; + + private Mock mockUnitOfWork; + private Mock mockEdgeDeviceModelRepository; + private Mock mockAmazonIoTClient; + private Mock mockAmazonGreenGrass; + private Mock mockDeviceModelImageManager; + + public override void Setup() + { + base.Setup(); + + this.mockDeviceModelImageManager = MockRepository.Create(); + this.mockUnitOfWork = MockRepository.Create(); + this.mockEdgeDeviceModelRepository = MockRepository.Create(); + this.mockAmazonIoTClient = MockRepository.Create(); + this.mockAmazonGreenGrass = MockRepository.Create(); + + _ = ServiceCollection.AddSingleton(this.mockDeviceModelImageManager.Object); + _ = ServiceCollection.AddSingleton(this.mockUnitOfWork.Object); + _ = ServiceCollection.AddSingleton(this.mockEdgeDeviceModelRepository.Object); + _ = ServiceCollection.AddSingleton(this.mockAmazonIoTClient.Object); + _ = ServiceCollection.AddSingleton(this.mockAmazonGreenGrass.Object); + _ = ServiceCollection.AddSingleton(); + + + Services = ServiceCollection.BuildServiceProvider(); + + this.syncGreenGrassJob = Services.GetRequiredService(); + } + + [Test] + public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymentInDB() + { + + //Arrange + var mockJobExecutionContext = MockRepository.Create(); + + var deploymentId = Fixture.Create(); + + var listDeploymentsInAws = new ListDeploymentsResponse + { + Deployments = new List() + { + new Deployment + { + DeploymentId = deploymentId, + }, + new Deployment + { + DeploymentId = Fixture.Create(), + }, + new Deployment + { + DeploymentId = Fixture.Create(), + } + } + }; + var existingDeployments = new List + { + new EdgeDeviceModel + { + Id = Fixture.Create(), + ExternalIdentifier = deploymentId, + }, + new EdgeDeviceModel + { + Id = Fixture.Create(), + ExternalIdentifier = Fixture.Create(), + } + }; + + _ = this.mockAmazonGreenGrass.Setup(greengrass => greengrass.ListDeploymentsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(listDeploymentsInAws); + + _ = this.mockEdgeDeviceModelRepository.Setup(u => u.GetAllAsync(null, It.IsAny())) + .ReturnsAsync(existingDeployments); + + _ = this.mockEdgeDeviceModelRepository.Setup(u => u.InsertAsync(It.Is(s => !s.ExternalIdentifier.Equals(deploymentId, StringComparison.Ordinal)))) + .Returns(Task.CompletedTask); + _ = this.mockDeviceModelImageManager.Setup(c => c.SetDefaultImageToModel(It.Is(s => !s.Equals(deploymentId, StringComparison.Ordinal)))) + .ReturnsAsync(Fixture.Create()); + _ = this.mockUnitOfWork.Setup(c => c.SaveAsync()) + .Returns(Task.CompletedTask); + + // Act + await this.syncGreenGrassJob.Execute(mockJobExecutionContext.Object); + + // Assert + MockRepository.VerifyAll(); + + } + } +} From d3e9c8002b98a99e09f4cf782241b4f273985614 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 14:52:51 +0200 Subject: [PATCH 3/9] Sync Deployment (DONE) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 5 +- .../Jobs/AWS/SyncThingTypesJob.cs | 5 +- .../AWS/SyncGreenGrassDeploymentsJobTests.cs | 13 +++- .../Services/AWS_Tests/AwsConfigTests.cs | 60 ++++++++++++++++++- .../Server/Services/EdgeModelServiceTest.cs | 5 -- 5 files changed, 74 insertions(+), 14 deletions(-) diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs index fbbbda2e2..465ce4b77 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -128,13 +128,12 @@ private async Task DeleteGreenGrassDeployments(List edgeModels) .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal))) .ToList(); - deploymentToDelete.ForEach(async edgeModel => + foreach (var edgeModel in deploymentToDelete) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id); this.edgeDeviceModelRepository.Delete(edgeModel.Id); await this.unitOfWork.SaveAsync(); - }); - + } } } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs index 44090f0a8..1533f80df 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs @@ -183,13 +183,12 @@ private async Task DeleteThingTypes(List thingTypes) thingTypes.Any(thingType => deviceModel.Id.Equals(thingType.ThingTypeId, StringComparison.Ordinal) && thingType.ThingTypeMetadata.Deprecated)) .ToList(); - deviceModelsToDelete.ForEach(async deviceModel => + foreach (var deviceModel in deviceModelsToDelete) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id); this.deviceModelRepository.Delete(deviceModel.Id); await this.unitOfWork.SaveAsync(); - - }); + } } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs index 0fbe79d62..3a7c9d916 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs @@ -63,6 +63,7 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen var mockJobExecutionContext = MockRepository.Create(); var deploymentId = Fixture.Create(); + var nonExistingDeploymentId = Fixture.Create(); var listDeploymentsInAws = new ListDeploymentsResponse { @@ -91,8 +92,8 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen }, new EdgeDeviceModel { - Id = Fixture.Create(), - ExternalIdentifier = Fixture.Create(), + Id = nonExistingDeploymentId, + ExternalIdentifier = nonExistingDeploymentId, } }; @@ -106,6 +107,13 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen .Returns(Task.CompletedTask); _ = this.mockDeviceModelImageManager.Setup(c => c.SetDefaultImageToModel(It.Is(s => !s.Equals(deploymentId, StringComparison.Ordinal)))) .ReturnsAsync(Fixture.Create()); + + this.mockEdgeDeviceModelRepository.Setup(u => u.Delete(It.Is(s => s.Equals(nonExistingDeploymentId, StringComparison.Ordinal)))) + .Verifiable(); + + _ = this.mockDeviceModelImageManager.Setup(c => c.DeleteDeviceModelImageAsync(It.Is(s => s.Equals(nonExistingDeploymentId, StringComparison.Ordinal)))) + .Returns(Task.CompletedTask); + _ = this.mockUnitOfWork.Setup(c => c.SaveAsync()) .Returns(Task.CompletedTask); @@ -116,5 +124,6 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen MockRepository.VerifyAll(); } + } } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs index ce13873fa..2fa4c0b8a 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs @@ -72,6 +72,8 @@ public void SetUp() public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTypeShouldCreateTheDeployment() { //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); var edge = Fixture.Create(); var edgeDeviceModelEntity = Mapper.Map(edge); @@ -90,6 +92,10 @@ public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTyp { HttpStatusCode = HttpStatusCode.Created }); + + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + + .ThrowsAsync(new Amazon.GreengrassV2.Model.ResourceNotFoundException("Resource Not found")); _ = this.mockGreengrasClient.Setup(s3 => s3.CreateComponentVersionAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new CreateComponentVersionResponse { @@ -112,9 +118,58 @@ public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTyp } [Test] - public async Task CreateDeploymentWithComponentsAndNonExistingThingGroupAndThingTypeShouldCreateThingGroupAndThingTypeAndTheDeployment() + public async Task CreateDeploymentWithExistingComponentsAndExistingThingGroupAndThingTypeShouldCreateTheDeployment() { //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); + + var edge = Fixture.Create(); + var edgeDeviceModelEntity = Mapper.Map(edge); + + _ = this.mockIotClient.Setup(s3 => s3.DescribeThingGroupAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingGroupResponse + { + HttpStatusCode = HttpStatusCode.OK + }); + + _ = this.mockIotClient.Setup(s3 => s3.DescribeThingTypeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingTypeResponse()); + + _ = this.mockGreengrasClient.Setup(s3 => s3.CreateDeploymentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CreateDeploymentResponse + { + HttpStatusCode = HttpStatusCode.Created + }); + + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeComponentResponse + { + HttpStatusCode = HttpStatusCode.OK + }); + + _ = this.mockEdgeModelRepository.Setup(x => x.GetByIdAsync(It.IsAny())) + .ReturnsAsync(edgeDeviceModelEntity); + + _ = this.mockEdgeModelRepository.Setup(repository => repository.Update(It.IsAny())); + _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) + .Returns(Task.CompletedTask); + + //Arrange + await this.awsConfigService.RollOutEdgeModelConfiguration(edge); + + //Assert + MockRepository.VerifyAll(); + + } + + [Test] + public async Task CreateDeploymentWithNonExistingComponentsAndNonExistingThingGroupAndThingTypeShouldCreateThingGroupAndThingTypeAndTheDeployment() + { + //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); + var edge = Fixture.Create(); var edgeDeviceModelEntity = Mapper.Map(edge); @@ -135,6 +190,9 @@ public async Task CreateDeploymentWithComponentsAndNonExistingThingGroupAndThing { HttpStatusCode = HttpStatusCode.Created }); + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new Amazon.GreengrassV2.Model.ResourceNotFoundException("Resource Not found")); + _ = this.mockGreengrasClient.Setup(s3 => s3.CreateComponentVersionAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new CreateComponentVersionResponse { diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs b/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs index 6857ad796..a02deb186 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs @@ -404,11 +404,6 @@ public async Task UpdateEdgeModelForAWSShouldUpdateEdgeModel() this.mockLabelRepository.Setup(repository => repository.Delete(It.IsAny())) .Verifiable(); - _ = this.mockEdgeDeviceModelRepository.Setup(repository => repository.Update(It.IsAny())); - _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) - .Returns(Task.CompletedTask); - - _ = this.mockConfigService.Setup(x => x.RollOutEdgeModelConfiguration(It.IsAny())) .Returns(Task.CompletedTask); From a965d8b88334ff3c9568e146b5fc5e6b0658b1ae Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 14:54:40 +0200 Subject: [PATCH 4/9] remove getComponentIfExist --- .../Services/AWS/AwsConfigService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs index e60073a50..598200617 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs @@ -147,7 +147,7 @@ private async Task> CreateG { try { - var getComponentIfExist = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest + _ = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest { Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}" }); From 4607e65828544c9cc5cfac2bc0f5757eb2dc7b09 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Fri, 26 May 2023 17:58:04 +0200 Subject: [PATCH 5/9] Sync GreenGrassDeployment (Started) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 119 ++++++++++++++++++ .../Startup/AWSServiceCollectionExtension.cs | 8 ++ 2 files changed, 127 insertions(+) create mode 100644 src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs new file mode 100644 index 000000000..d1dd4d48a --- /dev/null +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -0,0 +1,119 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS +{ + using Amazon.IoT; + using AutoMapper; + using AzureIoTHub.Portal.Application.Managers; + using AzureIoTHub.Portal.Domain.Repositories; + using AzureIoTHub.Portal.Domain; + using Microsoft.Extensions.Logging; + using Quartz; + using Amazon.GreengrassV2; + using AzureIoTHub.Portal.Models.v10; + using Amazon.GreengrassV2.Model; + using AzureIoTHub.Portal.Domain.Entities; + + [DisallowConcurrentExecution] + public class SyncGreenGrassDeploymentsJob : IJob + { + private readonly ILogger logger; + private readonly IMapper mapper; + private readonly IUnitOfWork unitOfWork; + private readonly IEdgeDeviceModelRepository edgeDeviceModelRepository; + private readonly IAmazonIoT amazonIoTClient; + private readonly IAmazonGreengrassV2 amazonGreenGrass; + private readonly IDeviceModelImageManager deviceModelImageManager; + + public SyncGreenGrassDeploymentsJob( + ILogger logger, + IMapper mapper, + IUnitOfWork unitOfWork, + IEdgeDeviceModelRepository edgeDeviceModelRepository, + IAmazonIoT amazonIoTClient, + IAmazonGreengrassV2 amazonGreenGrass, + IDeviceModelImageManager awsImageManager) + { + this.deviceModelImageManager = awsImageManager; + this.mapper = mapper; + this.unitOfWork = unitOfWork; + this.edgeDeviceModelRepository = edgeDeviceModelRepository; + this.amazonIoTClient = amazonIoTClient; + this.amazonGreenGrass = amazonGreenGrass; + this.logger = logger; + } + + + public async Task Execute(IJobExecutionContext context) + { + try + { + this.logger.LogInformation("Start of sync Greengrass Deployents job"); + + await SyncGreenGrassDeployments(); + + this.logger.LogInformation("End of sync Greengrass Deployents job"); + } + catch (Exception e) + { + this.logger.LogError(e, "Sync Greengrass Deployents job has failed"); + } + } + + private async Task SyncGreenGrassDeployments() + { + var awsGreenGrassDeployments = await GetAllGreenGrassDeployments(); + + foreach (var deployment in awsGreenGrassDeployments) + { + await CreateOrUpdateGreenGrassDeployment(deployment); + } + } + + private async Task> GetAllGreenGrassDeployments() + { + var deployments = new List(); + + var nextToken = string.Empty; + + var getAllAwsGreenGrassDeployments = await this.amazonGreenGrass.ListDeploymentsAsync( + new ListDeploymentsRequest + { + NextToken = nextToken, + }); + + foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments) + { + var iotEdgeModel = new IoTEdgeModel + { + ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID + Name = deployment.DeploymentName, + ExternalIdentifier = deployment.DeploymentId + }; + deployments.Add(iotEdgeModel); + } + return deployments; + } + + private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) + { + + var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()).Where(edge => edge.ExternalIdentifier == iotEdgeModel.ExternalIdentifier).ToList(); + if (iotEdgeModels.Count == 0) + { + var edgeModel = this.mapper.Map(iotEdgeModel); + + await this.edgeDeviceModelRepository.InsertAsync(edgeModel); + await this.unitOfWork.SaveAsync(); + _ = this.deviceModelImageManager.SetDefaultImageToModel(edgeModel.Id); + } + + } + + /*private async Task DeleteGreenGrassDeployments(List edgeModels) + { + + }*/ + } +} diff --git a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs index b2e501114..0095297fd 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs @@ -84,6 +84,14 @@ private static IServiceCollection ConfigureAWSSyncJobs(this IServiceCollection s .WithSimpleSchedule(s => s .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) .RepeatForever())); + + _ = q.AddJob(j => j.WithIdentity(nameof(SyncGreenGrassDeploymentsJob))) + .AddTrigger(t => t + .WithIdentity($"{nameof(SyncGreenGrassDeploymentsJob)}") + .ForJob(nameof(SyncGreenGrassDeploymentsJob)) + .WithSimpleSchedule(s => s + .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) + .RepeatForever())); }); } From 89f7fe90b13071adff0673a87e73b12376e08f7d Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 12:41:09 +0200 Subject: [PATCH 6/9] Sync Deployments (waiting for one test) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 31 ++++- .../Jobs/AWS/SyncThingTypesJob.cs | 3 +- .../Services/AWS/AwsConfigService.cs | 38 ++++-- .../Services/EdgeModelService.cs | 10 +- .../AWS/SyncGreenGrassDeploymentsJobTests.cs | 120 ++++++++++++++++++ 5 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs index d1dd4d48a..fbbbda2e2 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -67,8 +67,11 @@ private async Task SyncGreenGrassDeployments() foreach (var deployment in awsGreenGrassDeployments) { - await CreateOrUpdateGreenGrassDeployment(deployment); + await CreateNonExisitingGreenGrassDeployment(deployment); } + + //Delete in DB AWS deleted deployments + await DeleteGreenGrassDeployments(awsGreenGrassDeployments); } private async Task> GetAllGreenGrassDeployments() @@ -81,6 +84,7 @@ private async Task> GetAllGreenGrassDeployments() new ListDeploymentsRequest { NextToken = nextToken, + HistoryFilter = DeploymentHistoryFilter.LATEST_ONLY }); foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments) @@ -96,12 +100,18 @@ private async Task> GetAllGreenGrassDeployments() return deployments; } - private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) + private async Task CreateNonExisitingGreenGrassDeployment(IoTEdgeModel iotEdgeModel) { - var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()).Where(edge => edge.ExternalIdentifier == iotEdgeModel.ExternalIdentifier).ToList(); + var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync()) + .Where(edge => edge.ExternalIdentifier!.Equals(iotEdgeModel.ExternalIdentifier, StringComparison.Ordinal)).ToList(); + if (iotEdgeModels.Count == 0) { + //In Aws, it is possible to create a deployment without a name, so it will take the id as a name + //Here is how we handle it + iotEdgeModel.Name ??= iotEdgeModel.ModelId; + var edgeModel = this.mapper.Map(iotEdgeModel); await this.edgeDeviceModelRepository.InsertAsync(edgeModel); @@ -111,9 +121,20 @@ private async Task CreateOrUpdateGreenGrassDeployment(IoTEdgeModel iotEdgeModel) } - /*private async Task DeleteGreenGrassDeployments(List edgeModels) + private async Task DeleteGreenGrassDeployments(List edgeModels) { + //Get All Deployments that are not in AWS + var deploymentToDelete = (await this.edgeDeviceModelRepository.GetAllAsync()) + .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal))) + .ToList(); - }*/ + deploymentToDelete.ForEach(async edgeModel => + { + await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id); + this.edgeDeviceModelRepository.Delete(edgeModel.Id); + await this.unitOfWork.SaveAsync(); + }); + + } } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs index 6f728761d..44090f0a8 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs @@ -187,9 +187,10 @@ private async Task DeleteThingTypes(List thingTypes) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id); this.deviceModelRepository.Delete(deviceModel.Id); + await this.unitOfWork.SaveAsync(); + }); - await this.unitOfWork.SaveAsync(); } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs index 8b3d9ef29..e60073a50 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs @@ -145,28 +145,42 @@ private async Task> CreateG var listcomponentName = new Dictionary(); foreach (var component in edgeModel.EdgeModules) { - var recipeJson = JsonCreateComponent(component); - var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString()); - var recipeStream = new MemoryStream(recipeBytes); - - var componentVersion = new CreateComponentVersionRequest + try { - InlineRecipe = recipeStream - }; - var response = await greengras.CreateComponentVersionAsync(componentVersion); - if (response.HttpStatusCode != HttpStatusCode.Created) + var getComponentIfExist = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest + { + Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}" + }); + listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); + + } + catch (Amazon.GreengrassV2.Model.ResourceNotFoundException) { - throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API."); + var recipeJson = JsonCreateComponent(component); + var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString()); + var recipeStream = new MemoryStream(recipeBytes); + var componentVersion = new CreateComponentVersionRequest + { + InlineRecipe = recipeStream + }; + var response = await greengras.CreateComponentVersionAsync(componentVersion); + if (response.HttpStatusCode != HttpStatusCode.Created) + { + throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API."); + + } + listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); } - listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version }); } + return listcomponentName; } private static JObject JsonCreateComponent(IoTEdgeModule component) { + var environmentVariableObject = new JObject(); foreach (var env in component.EnvironmentVariables) @@ -329,7 +343,7 @@ public async Task> GetConfigModuleList(string modelId) } return moduleList; } - catch (Amazon.IoT.Model.ResourceNotFoundException) + catch (Amazon.GreengrassV2.Model.ResourceNotFoundException) { throw new InternalServerErrorException("The deployment is not found"); diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs index 96087e3a9..ea68eda6a 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs @@ -255,13 +255,15 @@ public async Task UpdateEdgeModel(IoTEdgeModel edgeModel) this.labelRepository.Delete(labelEntity.Id); } - _ = this.mapper.Map(edgeModel, edgeModelEntity); - this.edgeModelRepository.Update(edgeModelEntity); - - await this.unitOfWork.SaveAsync(); + // For AWS, we do the update in the AwsConfiguration if (this.config.CloudProvider.Equals(CloudProviders.Azure, StringComparison.Ordinal)) { + _ = this.mapper.Map(edgeModel, edgeModelEntity); + + this.edgeModelRepository.Update(edgeModelEntity); + await this.unitOfWork.SaveAsync(); + await SaveModuleCommands(edgeModel); } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs new file mode 100644 index 000000000..0fbe79d62 --- /dev/null +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Tests.Unit.Infrastructure.Jobs.AWS +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Amazon.GreengrassV2; + using Amazon.GreengrassV2.Model; + using Amazon.IoT; + using AutoFixture; + using AzureIoTHub.Portal.Application.Managers; + using AzureIoTHub.Portal.Domain; + using AzureIoTHub.Portal.Domain.Entities; + using AzureIoTHub.Portal.Domain.Repositories; + using AzureIoTHub.Portal.Infrastructure.Jobs.AWS; + using AzureIoTHub.Portal.Tests.Unit.UnitTests.Bases; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Quartz; + + public class SyncGreenGrassDeploymentsJobTests : BackendUnitTest + { + private IJob syncGreenGrassJob; + + private Mock mockUnitOfWork; + private Mock mockEdgeDeviceModelRepository; + private Mock mockAmazonIoTClient; + private Mock mockAmazonGreenGrass; + private Mock mockDeviceModelImageManager; + + public override void Setup() + { + base.Setup(); + + this.mockDeviceModelImageManager = MockRepository.Create(); + this.mockUnitOfWork = MockRepository.Create(); + this.mockEdgeDeviceModelRepository = MockRepository.Create(); + this.mockAmazonIoTClient = MockRepository.Create(); + this.mockAmazonGreenGrass = MockRepository.Create(); + + _ = ServiceCollection.AddSingleton(this.mockDeviceModelImageManager.Object); + _ = ServiceCollection.AddSingleton(this.mockUnitOfWork.Object); + _ = ServiceCollection.AddSingleton(this.mockEdgeDeviceModelRepository.Object); + _ = ServiceCollection.AddSingleton(this.mockAmazonIoTClient.Object); + _ = ServiceCollection.AddSingleton(this.mockAmazonGreenGrass.Object); + _ = ServiceCollection.AddSingleton(); + + + Services = ServiceCollection.BuildServiceProvider(); + + this.syncGreenGrassJob = Services.GetRequiredService(); + } + + [Test] + public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymentInDB() + { + + //Arrange + var mockJobExecutionContext = MockRepository.Create(); + + var deploymentId = Fixture.Create(); + + var listDeploymentsInAws = new ListDeploymentsResponse + { + Deployments = new List() + { + new Deployment + { + DeploymentId = deploymentId, + }, + new Deployment + { + DeploymentId = Fixture.Create(), + }, + new Deployment + { + DeploymentId = Fixture.Create(), + } + } + }; + var existingDeployments = new List + { + new EdgeDeviceModel + { + Id = Fixture.Create(), + ExternalIdentifier = deploymentId, + }, + new EdgeDeviceModel + { + Id = Fixture.Create(), + ExternalIdentifier = Fixture.Create(), + } + }; + + _ = this.mockAmazonGreenGrass.Setup(greengrass => greengrass.ListDeploymentsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(listDeploymentsInAws); + + _ = this.mockEdgeDeviceModelRepository.Setup(u => u.GetAllAsync(null, It.IsAny())) + .ReturnsAsync(existingDeployments); + + _ = this.mockEdgeDeviceModelRepository.Setup(u => u.InsertAsync(It.Is(s => !s.ExternalIdentifier.Equals(deploymentId, StringComparison.Ordinal)))) + .Returns(Task.CompletedTask); + _ = this.mockDeviceModelImageManager.Setup(c => c.SetDefaultImageToModel(It.Is(s => !s.Equals(deploymentId, StringComparison.Ordinal)))) + .ReturnsAsync(Fixture.Create()); + _ = this.mockUnitOfWork.Setup(c => c.SaveAsync()) + .Returns(Task.CompletedTask); + + // Act + await this.syncGreenGrassJob.Execute(mockJobExecutionContext.Object); + + // Assert + MockRepository.VerifyAll(); + + } + } +} From e3148d5d7e7c2c422e68cb90362095110eb3d806 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 14:52:51 +0200 Subject: [PATCH 7/9] Sync Deployment (DONE) --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 5 +- .../Jobs/AWS/SyncThingTypesJob.cs | 5 +- .../AWS/SyncGreenGrassDeploymentsJobTests.cs | 13 +++- .../Services/AWS_Tests/AwsConfigTests.cs | 60 ++++++++++++++++++- .../Server/Services/EdgeModelServiceTest.cs | 5 -- 5 files changed, 74 insertions(+), 14 deletions(-) diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs index fbbbda2e2..465ce4b77 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -128,13 +128,12 @@ private async Task DeleteGreenGrassDeployments(List edgeModels) .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal))) .ToList(); - deploymentToDelete.ForEach(async edgeModel => + foreach (var edgeModel in deploymentToDelete) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id); this.edgeDeviceModelRepository.Delete(edgeModel.Id); await this.unitOfWork.SaveAsync(); - }); - + } } } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs index 44090f0a8..1533f80df 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs @@ -183,13 +183,12 @@ private async Task DeleteThingTypes(List thingTypes) thingTypes.Any(thingType => deviceModel.Id.Equals(thingType.ThingTypeId, StringComparison.Ordinal) && thingType.ThingTypeMetadata.Deprecated)) .ToList(); - deviceModelsToDelete.ForEach(async deviceModel => + foreach (var deviceModel in deviceModelsToDelete) { await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id); this.deviceModelRepository.Delete(deviceModel.Id); await this.unitOfWork.SaveAsync(); - - }); + } } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs index 0fbe79d62..3a7c9d916 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs @@ -63,6 +63,7 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen var mockJobExecutionContext = MockRepository.Create(); var deploymentId = Fixture.Create(); + var nonExistingDeploymentId = Fixture.Create(); var listDeploymentsInAws = new ListDeploymentsResponse { @@ -91,8 +92,8 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen }, new EdgeDeviceModel { - Id = Fixture.Create(), - ExternalIdentifier = Fixture.Create(), + Id = nonExistingDeploymentId, + ExternalIdentifier = nonExistingDeploymentId, } }; @@ -106,6 +107,13 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen .Returns(Task.CompletedTask); _ = this.mockDeviceModelImageManager.Setup(c => c.SetDefaultImageToModel(It.Is(s => !s.Equals(deploymentId, StringComparison.Ordinal)))) .ReturnsAsync(Fixture.Create()); + + this.mockEdgeDeviceModelRepository.Setup(u => u.Delete(It.Is(s => s.Equals(nonExistingDeploymentId, StringComparison.Ordinal)))) + .Verifiable(); + + _ = this.mockDeviceModelImageManager.Setup(c => c.DeleteDeviceModelImageAsync(It.Is(s => s.Equals(nonExistingDeploymentId, StringComparison.Ordinal)))) + .Returns(Task.CompletedTask); + _ = this.mockUnitOfWork.Setup(c => c.SaveAsync()) .Returns(Task.CompletedTask); @@ -116,5 +124,6 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen MockRepository.VerifyAll(); } + } } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs index ce13873fa..2fa4c0b8a 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Services/AWS_Tests/AwsConfigTests.cs @@ -72,6 +72,8 @@ public void SetUp() public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTypeShouldCreateTheDeployment() { //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); var edge = Fixture.Create(); var edgeDeviceModelEntity = Mapper.Map(edge); @@ -90,6 +92,10 @@ public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTyp { HttpStatusCode = HttpStatusCode.Created }); + + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + + .ThrowsAsync(new Amazon.GreengrassV2.Model.ResourceNotFoundException("Resource Not found")); _ = this.mockGreengrasClient.Setup(s3 => s3.CreateComponentVersionAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new CreateComponentVersionResponse { @@ -112,9 +118,58 @@ public async Task CreateDeploymentWithComponentsAndExistingThingGroupAndThingTyp } [Test] - public async Task CreateDeploymentWithComponentsAndNonExistingThingGroupAndThingTypeShouldCreateThingGroupAndThingTypeAndTheDeployment() + public async Task CreateDeploymentWithExistingComponentsAndExistingThingGroupAndThingTypeShouldCreateTheDeployment() { //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); + + var edge = Fixture.Create(); + var edgeDeviceModelEntity = Mapper.Map(edge); + + _ = this.mockIotClient.Setup(s3 => s3.DescribeThingGroupAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingGroupResponse + { + HttpStatusCode = HttpStatusCode.OK + }); + + _ = this.mockIotClient.Setup(s3 => s3.DescribeThingTypeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingTypeResponse()); + + _ = this.mockGreengrasClient.Setup(s3 => s3.CreateDeploymentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CreateDeploymentResponse + { + HttpStatusCode = HttpStatusCode.Created + }); + + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeComponentResponse + { + HttpStatusCode = HttpStatusCode.OK + }); + + _ = this.mockEdgeModelRepository.Setup(x => x.GetByIdAsync(It.IsAny())) + .ReturnsAsync(edgeDeviceModelEntity); + + _ = this.mockEdgeModelRepository.Setup(repository => repository.Update(It.IsAny())); + _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) + .Returns(Task.CompletedTask); + + //Arrange + await this.awsConfigService.RollOutEdgeModelConfiguration(edge); + + //Assert + MockRepository.VerifyAll(); + + } + + [Test] + public async Task CreateDeploymentWithNonExistingComponentsAndNonExistingThingGroupAndThingTypeShouldCreateThingGroupAndThingTypeAndTheDeployment() + { + //Act + _ = this.mockConfigHandler.Setup(handler => handler.AWSRegion).Returns("eu-west-1"); + _ = this.mockConfigHandler.Setup(handler => handler.AWSAccountId).Returns("00000000"); + var edge = Fixture.Create(); var edgeDeviceModelEntity = Mapper.Map(edge); @@ -135,6 +190,9 @@ public async Task CreateDeploymentWithComponentsAndNonExistingThingGroupAndThing { HttpStatusCode = HttpStatusCode.Created }); + _ = this.mockGreengrasClient.Setup(s3 => s3.DescribeComponentAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new Amazon.GreengrassV2.Model.ResourceNotFoundException("Resource Not found")); + _ = this.mockGreengrasClient.Setup(s3 => s3.CreateComponentVersionAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new CreateComponentVersionResponse { diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs b/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs index 6857ad796..a02deb186 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Server/Services/EdgeModelServiceTest.cs @@ -404,11 +404,6 @@ public async Task UpdateEdgeModelForAWSShouldUpdateEdgeModel() this.mockLabelRepository.Setup(repository => repository.Delete(It.IsAny())) .Verifiable(); - _ = this.mockEdgeDeviceModelRepository.Setup(repository => repository.Update(It.IsAny())); - _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) - .Returns(Task.CompletedTask); - - _ = this.mockConfigService.Setup(x => x.RollOutEdgeModelConfiguration(It.IsAny())) .Returns(Task.CompletedTask); From 9c67f6ee28d390ea104f19758218892333bc38d6 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 14:54:40 +0200 Subject: [PATCH 8/9] remove getComponentIfExist --- .../Services/AWS/AwsConfigService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs index e60073a50..598200617 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs @@ -147,7 +147,7 @@ private async Task> CreateG { try { - var getComponentIfExist = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest + _ = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest { Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}" }); From e7327241f09f37a2d0024f2c79edcd34b6c25fb3 Mon Sep 17 00:00:00 2001 From: ssgueye2 Date: Tue, 30 May 2023 16:16:33 +0200 Subject: [PATCH 9/9] update Sync Deployment With Dynamic Thing group --- .../Jobs/AWS/SyncGreenGrassDeploymentsJob.cs | 25 ++++++++++++++----- .../AWS/SyncGreenGrassDeploymentsJobTests.cs | 9 +++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs index 465ce4b77..bb27e5bea 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs @@ -14,6 +14,7 @@ namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS using AzureIoTHub.Portal.Models.v10; using Amazon.GreengrassV2.Model; using AzureIoTHub.Portal.Domain.Entities; + using System.Text.RegularExpressions; [DisallowConcurrentExecution] public class SyncGreenGrassDeploymentsJob : IJob @@ -89,13 +90,25 @@ private async Task> GetAllGreenGrassDeployments() foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments) { - var iotEdgeModel = new IoTEdgeModel + var awsThingGroupRegex = new Regex(@"/([^/]+)$"); + var matches = awsThingGroupRegex.Match(deployment.TargetArn); + + if (matches.Success && matches.Groups.Count > 1) { - ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID - Name = deployment.DeploymentName, - ExternalIdentifier = deployment.DeploymentId - }; - deployments.Add(iotEdgeModel); + var thinggroupName = matches.Groups[1].Value; + var s = await this.amazonIoTClient.DescribeThingGroupAsync(new Amazon.IoT.Model.DescribeThingGroupRequest { ThingGroupName = thinggroupName }); + if (s.QueryString != null) + { + var iotEdgeModel = new IoTEdgeModel + { + ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID + Name = deployment.DeploymentName, + ExternalIdentifier = deployment.DeploymentId + }; + deployments.Add(iotEdgeModel); + } + } + } return deployments; } diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs index 3a7c9d916..14018e142 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs @@ -10,6 +10,7 @@ namespace AzureIoTHub.Portal.Tests.Unit.Infrastructure.Jobs.AWS using Amazon.GreengrassV2; using Amazon.GreengrassV2.Model; using Amazon.IoT; + using Amazon.IoT.Model; using AutoFixture; using AzureIoTHub.Portal.Application.Managers; using AzureIoTHub.Portal.Domain; @@ -72,14 +73,19 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen new Deployment { DeploymentId = deploymentId, + TargetArn = "arn:aws:iot:eu-west-1:0000000000:thinggroup/DemoEdgeModel" }, new Deployment { DeploymentId = Fixture.Create(), + TargetArn = "arn:aws:iot:eu-west-1:0000000000:thinggroup/toto" + }, new Deployment { DeploymentId = Fixture.Create(), + TargetArn = "arn:aws:iot:eu-west-1:0000000000:thinggroup/titi" + } } }; @@ -100,6 +106,9 @@ public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymen _ = this.mockAmazonGreenGrass.Setup(greengrass => greengrass.ListDeploymentsAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(listDeploymentsInAws); + _ = this.mockAmazonIoTClient.Setup(iot => iot.DescribeThingGroupAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Fixture.Create); + _ = this.mockEdgeDeviceModelRepository.Setup(u => u.GetAllAsync(null, It.IsAny())) .ReturnsAsync(existingDeployments);