Sync GreenGrassDeployment

src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs

@@ -0,0 +1,140 @@
+// Copyright (c) CGI France. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS
+{
+    using Amazon.IoT;
+    using AutoMapper;
+    using AzureIoTHub.Portal.Application.Managers;
+    using AzureIoTHub.Portal.Domain.Repositories;
+    using AzureIoTHub.Portal.Domain;
+    using Microsoft.Extensions.Logging;
+    using Quartz;
+    using Amazon.GreengrassV2;
+    using AzureIoTHub.Portal.Models.v10;
+    using Amazon.GreengrassV2.Model;
+    using AzureIoTHub.Portal.Domain.Entities;
+
+    [DisallowConcurrentExecution]
+    public class SyncGreenGrassDeploymentsJob : IJob
+    {
+        private readonly ILogger<SyncThingTypesJob> logger;
+        private readonly IMapper mapper;
+        private readonly IUnitOfWork unitOfWork;
+        private readonly IEdgeDeviceModelRepository edgeDeviceModelRepository;
+        private readonly IAmazonIoT amazonIoTClient;
+        private readonly IAmazonGreengrassV2 amazonGreenGrass;
+        private readonly IDeviceModelImageManager deviceModelImageManager;
+
+        public SyncGreenGrassDeploymentsJob(
+            ILogger<SyncThingTypesJob> logger,
+            IMapper mapper,
+            IUnitOfWork unitOfWork,
+            IEdgeDeviceModelRepository edgeDeviceModelRepository,
+            IAmazonIoT amazonIoTClient,
+            IAmazonGreengrassV2 amazonGreenGrass,
+            IDeviceModelImageManager awsImageManager)
+        {
+            this.deviceModelImageManager = awsImageManager;
+            this.mapper = mapper;
+            this.unitOfWork = unitOfWork;
+            this.edgeDeviceModelRepository = edgeDeviceModelRepository;
+            this.amazonIoTClient = amazonIoTClient;
+            this.amazonGreenGrass = amazonGreenGrass;
+            this.logger = logger;
+        }
+
+
+        public async Task Execute(IJobExecutionContext context)
+        {
+            try
+            {
+                this.logger.LogInformation("Start of sync Greengrass Deployents job");
+
+                await SyncGreenGrassDeployments();
+
+                this.logger.LogInformation("End of sync Greengrass Deployents job");
+            }
+            catch (Exception e)
+            {
+                this.logger.LogError(e, "Sync Greengrass Deployents job has failed");
+            }
+        }
+
+        private async Task SyncGreenGrassDeployments()
+        {
+            var awsGreenGrassDeployments = await GetAllGreenGrassDeployments();
+
+            foreach (var deployment in awsGreenGrassDeployments)
+            {
+                await CreateNonExisitingGreenGrassDeployment(deployment);
+            }
+
+            //Delete in DB AWS deleted deployments
+            await DeleteGreenGrassDeployments(awsGreenGrassDeployments);
+        }
+
+        private async Task<List<IoTEdgeModel>> GetAllGreenGrassDeployments()
+        {
+            var deployments = new List<IoTEdgeModel>();
+
+            var nextToken = string.Empty;
+
+            var getAllAwsGreenGrassDeployments = await this.amazonGreenGrass.ListDeploymentsAsync(
+                new ListDeploymentsRequest
+                {
+                    NextToken = nextToken,
+                    HistoryFilter = DeploymentHistoryFilter.LATEST_ONLY
+                });
+
+            foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments)
+            {
+                var iotEdgeModel = new IoTEdgeModel
+                {
+                    ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID
+                    Name = deployment.DeploymentName,
+                    ExternalIdentifier = deployment.DeploymentId
+                };
+                deployments.Add(iotEdgeModel);
+            }
+            return deployments;
+        }
+
+        private async Task CreateNonExisitingGreenGrassDeployment(IoTEdgeModel iotEdgeModel)
+        {
+
+            var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync())
+                .Where(edge => edge.ExternalIdentifier!.Equals(iotEdgeModel.ExternalIdentifier, StringComparison.Ordinal)).ToList();
+
+            if (iotEdgeModels.Count == 0)
+            {
+                //In Aws, it is possible to create a deployment without a name, so it will take the id as a name
+                //Here is how we handle it
+                iotEdgeModel.Name ??= iotEdgeModel.ModelId;
+
+                var edgeModel = this.mapper.Map<EdgeDeviceModel>(iotEdgeModel);
+
+                await this.edgeDeviceModelRepository.InsertAsync(edgeModel);
+                await this.unitOfWork.SaveAsync();
+                _ = this.deviceModelImageManager.SetDefaultImageToModel(edgeModel.Id);
+            }
+
+        }
+
+        private async Task DeleteGreenGrassDeployments(List<IoTEdgeModel> edgeModels)
+        {
+            //Get All Deployments that are not in AWS
+            var deploymentToDelete = (await this.edgeDeviceModelRepository.GetAllAsync())
+                .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal)))
+                .ToList();
+
+            deploymentToDelete.ForEach(async edgeModel =>
+            {
+                await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id);
+                this.edgeDeviceModelRepository.Delete(edgeModel.Id);
+                await this.unitOfWork.SaveAsync();
+            });
+
+        }
+    }
+}

src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs

@@ -187,9 +187,10 @@ private async Task DeleteThingTypes(List<DescribeThingTypeResponse> thingTypes)
             {
                 await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id);
                 this.deviceModelRepository.Delete(deviceModel.Id);
+                await this.unitOfWork.SaveAsync();
+
             });
 
-            await this.unitOfWork.SaveAsync();
         }
 
     }

src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs

@@ -145,28 +145,42 @@ private async Task<Dictionary<string, ComponentDeploymentSpecification>> CreateG
             var listcomponentName = new Dictionary<string, ComponentDeploymentSpecification>();
             foreach (var component in edgeModel.EdgeModules)
             {
-                var recipeJson = JsonCreateComponent(component);
-                var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString());
-                var recipeStream = new MemoryStream(recipeBytes);
-
-                var componentVersion = new CreateComponentVersionRequest
+                try
                 {
-                    InlineRecipe = recipeStream
-                };
-                var response = await greengras.CreateComponentVersionAsync(componentVersion);
-                if (response.HttpStatusCode != HttpStatusCode.Created)
+                    var getComponentIfExist = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest
+                    {
+                        Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}"
+                    });
+                    listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
+
+                }
+                catch (Amazon.GreengrassV2.Model.ResourceNotFoundException)
                 {
-                    throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API.");
+                    var recipeJson = JsonCreateComponent(component);
+                    var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString());
+                    var recipeStream = new MemoryStream(recipeBytes);
 
+                    var componentVersion = new CreateComponentVersionRequest
+                    {
+                        InlineRecipe = recipeStream
+                    };
+                    var response = await greengras.CreateComponentVersionAsync(componentVersion);
+                    if (response.HttpStatusCode != HttpStatusCode.Created)
+                    {
+                        throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API.");
+
+                    }
+                    listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
                 }
-                listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
             }
 
+
             return listcomponentName;
         }
 
         private static JObject JsonCreateComponent(IoTEdgeModule component)
         {
+
             var environmentVariableObject = new JObject();
 
             foreach (var env in component.EnvironmentVariables)
@@ -329,7 +343,7 @@ public async Task<List<IoTEdgeModule>> GetConfigModuleList(string modelId)
                 }
                 return moduleList;
             }
-            catch (Amazon.IoT.Model.ResourceNotFoundException)
+            catch (Amazon.GreengrassV2.Model.ResourceNotFoundException)
             {
                 throw new InternalServerErrorException("The deployment is not found");
 

src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs

@@ -255,13 +255,15 @@ public async Task UpdateEdgeModel(IoTEdgeModel edgeModel)
                 this.labelRepository.Delete(labelEntity.Id);
             }
 
-            _ = this.mapper.Map(edgeModel, edgeModelEntity);
-            this.edgeModelRepository.Update(edgeModelEntity);
-
-            await this.unitOfWork.SaveAsync();
 
+            // For AWS, we do the update in the AwsConfiguration
             if (this.config.CloudProvider.Equals(CloudProviders.Azure, StringComparison.Ordinal))
             {
+                _ = this.mapper.Map(edgeModel, edgeModelEntity);
+
+                this.edgeModelRepository.Update(edgeModelEntity);
+                await this.unitOfWork.SaveAsync();
+
                 await SaveModuleCommands(edgeModel);
             }
 

src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs

@@ -84,6 +84,14 @@ private static IServiceCollection ConfigureAWSSyncJobs(this IServiceCollection s
                     .WithSimpleSchedule(s => s
                             .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes)
                             .RepeatForever()));
+
+                _ = q.AddJob<SyncGreenGrassDeploymentsJob>(j => j.WithIdentity(nameof(SyncGreenGrassDeploymentsJob)))
+                    .AddTrigger(t => t
+                        .WithIdentity($"{nameof(SyncGreenGrassDeploymentsJob)}")
+                        .ForJob(nameof(SyncGreenGrassDeploymentsJob))
+                    .WithSimpleSchedule(s => s
+                            .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes)
+                            .RepeatForever()));
             });
         }
 

src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJobTests.cs

@@ -0,0 +1,120 @@
+// Copyright (c) CGI France. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace AzureIoTHub.Portal.Tests.Unit.Infrastructure.Jobs.AWS
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Amazon.GreengrassV2;
+    using Amazon.GreengrassV2.Model;
+    using Amazon.IoT;
+    using AutoFixture;
+    using AzureIoTHub.Portal.Application.Managers;
+    using AzureIoTHub.Portal.Domain;
+    using AzureIoTHub.Portal.Domain.Entities;
+    using AzureIoTHub.Portal.Domain.Repositories;
+    using AzureIoTHub.Portal.Infrastructure.Jobs.AWS;
+    using AzureIoTHub.Portal.Tests.Unit.UnitTests.Bases;
+    using Microsoft.Extensions.DependencyInjection;
+    using Moq;
+    using NUnit.Framework;
+    using Quartz;
+
+    public class SyncGreenGrassDeploymentsJobTests : BackendUnitTest
+    {
+        private IJob syncGreenGrassJob;
+
+        private Mock<IUnitOfWork> mockUnitOfWork;
+        private Mock<IEdgeDeviceModelRepository> mockEdgeDeviceModelRepository;
+        private Mock<IAmazonIoT> mockAmazonIoTClient;
+        private Mock<IAmazonGreengrassV2> mockAmazonGreenGrass;
+        private Mock<IDeviceModelImageManager> mockDeviceModelImageManager;
+
+        public override void Setup()
+        {
+            base.Setup();
+
+            this.mockDeviceModelImageManager = MockRepository.Create<IDeviceModelImageManager>();
+            this.mockUnitOfWork = MockRepository.Create<IUnitOfWork>();
+            this.mockEdgeDeviceModelRepository = MockRepository.Create<IEdgeDeviceModelRepository>();
+            this.mockAmazonIoTClient = MockRepository.Create<IAmazonIoT>();
+            this.mockAmazonGreenGrass = MockRepository.Create<IAmazonGreengrassV2>();
+
+            _ = ServiceCollection.AddSingleton(this.mockDeviceModelImageManager.Object);
+            _ = ServiceCollection.AddSingleton(this.mockUnitOfWork.Object);
+            _ = ServiceCollection.AddSingleton(this.mockEdgeDeviceModelRepository.Object);
+            _ = ServiceCollection.AddSingleton(this.mockAmazonIoTClient.Object);
+            _ = ServiceCollection.AddSingleton(this.mockAmazonGreenGrass.Object);
+            _ = ServiceCollection.AddSingleton<IJob, SyncGreenGrassDeploymentsJob>();
+
+
+            Services = ServiceCollection.BuildServiceProvider();
+
+            this.syncGreenGrassJob = Services.GetRequiredService<IJob>();
+        }
+
+        [Test]
+        public async Task ExecuteSyncExistingAWSDeploymentsAndCreateNinExistingDeploymentInDB()
+        {
+
+            //Arrange
+            var mockJobExecutionContext = MockRepository.Create<IJobExecutionContext>();
+
+            var deploymentId = Fixture.Create<string>();
+
+            var listDeploymentsInAws = new ListDeploymentsResponse
+            {
+                Deployments = new List<Deployment>()
+                {
+                    new Deployment
+                    {
+                        DeploymentId = deploymentId,
+                    },
+                    new Deployment
+                    {
+                        DeploymentId = Fixture.Create<string>(),
+                    },
+                    new Deployment
+                    {
+                        DeploymentId = Fixture.Create<string>(),
+                    }
+                }
+            };
+            var existingDeployments = new List<EdgeDeviceModel>
+            {
+                new EdgeDeviceModel
+                {
+                    Id = Fixture.Create<string>(),
+                    ExternalIdentifier = deploymentId,
+                },
+                new EdgeDeviceModel
+                {
+                    Id = Fixture.Create<string>(),
+                    ExternalIdentifier = Fixture.Create<string>(),
+                }
+            };
+
+            _ = this.mockAmazonGreenGrass.Setup(greengrass => greengrass.ListDeploymentsAsync(It.IsAny<ListDeploymentsRequest>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync(listDeploymentsInAws);
+
+            _ = this.mockEdgeDeviceModelRepository.Setup(u => u.GetAllAsync(null, It.IsAny<CancellationToken>()))
+              .ReturnsAsync(existingDeployments);
+
+            _ = this.mockEdgeDeviceModelRepository.Setup(u => u.InsertAsync(It.Is<EdgeDeviceModel>(s => !s.ExternalIdentifier.Equals(deploymentId, StringComparison.Ordinal))))
+                .Returns(Task.CompletedTask);
+            _ = this.mockDeviceModelImageManager.Setup(c => c.SetDefaultImageToModel(It.Is<string>(s => !s.Equals(deploymentId, StringComparison.Ordinal))))
+                .ReturnsAsync(Fixture.Create<string>());
+            _ = this.mockUnitOfWork.Setup(c => c.SaveAsync())
+                .Returns(Task.CompletedTask);
+
+            // Act
+            await this.syncGreenGrassJob.Execute(mockJobExecutionContext.Object);
+
+            // Assert
+            MockRepository.VerifyAll();
+
+        }
+    }
+}