Sync GreenGrassDeployment (#2120)

* Sync GreenGrassDeployment (Started)

* Sync Deployments (waiting for one test)

* Sync Deployment (DONE)

* remove getComponentIfExist

* Sync GreenGrassDeployment (Started)

* Sync Deployments (waiting for one test)

* Sync Deployment (DONE)

* remove getComponentIfExist

* update Sync Deployment With Dynamic Thing group

src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncGreenGrassDeploymentsJob.cs

@@ -0,0 +1,152 @@
+// Copyright (c) CGI France. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS
+{
+    using Amazon.IoT;
+    using AutoMapper;
+    using AzureIoTHub.Portal.Application.Managers;
+    using AzureIoTHub.Portal.Domain.Repositories;
+    using AzureIoTHub.Portal.Domain;
+    using Microsoft.Extensions.Logging;
+    using Quartz;
+    using Amazon.GreengrassV2;
+    using AzureIoTHub.Portal.Models.v10;
+    using Amazon.GreengrassV2.Model;
+    using AzureIoTHub.Portal.Domain.Entities;
+    using System.Text.RegularExpressions;
+
+    [DisallowConcurrentExecution]
+    public class SyncGreenGrassDeploymentsJob : IJob
+    {
+        private readonly ILogger<SyncThingTypesJob> logger;
+        private readonly IMapper mapper;
+        private readonly IUnitOfWork unitOfWork;
+        private readonly IEdgeDeviceModelRepository edgeDeviceModelRepository;
+        private readonly IAmazonIoT amazonIoTClient;
+        private readonly IAmazonGreengrassV2 amazonGreenGrass;
+        private readonly IDeviceModelImageManager deviceModelImageManager;
+
+        public SyncGreenGrassDeploymentsJob(
+            ILogger<SyncThingTypesJob> logger,
+            IMapper mapper,
+            IUnitOfWork unitOfWork,
+            IEdgeDeviceModelRepository edgeDeviceModelRepository,
+            IAmazonIoT amazonIoTClient,
+            IAmazonGreengrassV2 amazonGreenGrass,
+            IDeviceModelImageManager awsImageManager)
+        {
+            this.deviceModelImageManager = awsImageManager;
+            this.mapper = mapper;
+            this.unitOfWork = unitOfWork;
+            this.edgeDeviceModelRepository = edgeDeviceModelRepository;
+            this.amazonIoTClient = amazonIoTClient;
+            this.amazonGreenGrass = amazonGreenGrass;
+            this.logger = logger;
+        }
+
+
+        public async Task Execute(IJobExecutionContext context)
+        {
+            try
+            {
+                this.logger.LogInformation("Start of sync Greengrass Deployents job");
+
+                await SyncGreenGrassDeployments();
+
+                this.logger.LogInformation("End of sync Greengrass Deployents job");
+            }
+            catch (Exception e)
+            {
+                this.logger.LogError(e, "Sync Greengrass Deployents job has failed");
+            }
+        }
+
+        private async Task SyncGreenGrassDeployments()
+        {
+            var awsGreenGrassDeployments = await GetAllGreenGrassDeployments();
+
+            foreach (var deployment in awsGreenGrassDeployments)
+            {
+                await CreateNonExisitingGreenGrassDeployment(deployment);
+            }
+
+            //Delete in DB AWS deleted deployments
+            await DeleteGreenGrassDeployments(awsGreenGrassDeployments);
+        }
+
+        private async Task<List<IoTEdgeModel>> GetAllGreenGrassDeployments()
+        {
+            var deployments = new List<IoTEdgeModel>();
+
+            var nextToken = string.Empty;
+
+            var getAllAwsGreenGrassDeployments = await this.amazonGreenGrass.ListDeploymentsAsync(
+                new ListDeploymentsRequest
+                {
+                    NextToken = nextToken,
+                    HistoryFilter = DeploymentHistoryFilter.LATEST_ONLY
+                });
+
+            foreach (var deployment in getAllAwsGreenGrassDeployments.Deployments)
+            {
+                var awsThingGroupRegex = new Regex(@"/([^/]+)$");
+                var matches = awsThingGroupRegex.Match(deployment.TargetArn);
+
+                if (matches.Success && matches.Groups.Count > 1)
+                {
+                    var thinggroupName = matches.Groups[1].Value;
+                    var s = await this.amazonIoTClient.DescribeThingGroupAsync(new Amazon.IoT.Model.DescribeThingGroupRequest { ThingGroupName = thinggroupName });
+                    if (s.QueryString != null)
+                    {
+                        var iotEdgeModel = new IoTEdgeModel
+                        {
+                            ModelId = deployment.DeploymentId, //Instead of giving a random Id here, we can give the deploymentID
+                            Name = deployment.DeploymentName,
+                            ExternalIdentifier = deployment.DeploymentId
+                        };
+                        deployments.Add(iotEdgeModel);
+                    }
+                }
+
+            }
+            return deployments;
+        }
+
+        private async Task CreateNonExisitingGreenGrassDeployment(IoTEdgeModel iotEdgeModel)
+        {
+
+            var iotEdgeModels = (await this.edgeDeviceModelRepository.GetAllAsync())
+                .Where(edge => edge.ExternalIdentifier!.Equals(iotEdgeModel.ExternalIdentifier, StringComparison.Ordinal)).ToList();
+
+            if (iotEdgeModels.Count == 0)
+            {
+                //In Aws, it is possible to create a deployment without a name, so it will take the id as a name
+                //Here is how we handle it
+                iotEdgeModel.Name ??= iotEdgeModel.ModelId;
+
+                var edgeModel = this.mapper.Map<EdgeDeviceModel>(iotEdgeModel);
+
+                await this.edgeDeviceModelRepository.InsertAsync(edgeModel);
+                await this.unitOfWork.SaveAsync();
+                _ = this.deviceModelImageManager.SetDefaultImageToModel(edgeModel.Id);
+            }
+
+        }
+
+        private async Task DeleteGreenGrassDeployments(List<IoTEdgeModel> edgeModels)
+        {
+            //Get All Deployments that are not in AWS
+            var deploymentToDelete = (await this.edgeDeviceModelRepository.GetAllAsync())
+                .Where(edge => !edgeModels.Any(edgeModel => edge.ExternalIdentifier!.Equals(edgeModel.ExternalIdentifier, StringComparison.Ordinal)))
+                .ToList();
+
+            foreach (var edgeModel in deploymentToDelete)
+            {
+                await this.deviceModelImageManager.DeleteDeviceModelImageAsync(edgeModel.Id);
+                this.edgeDeviceModelRepository.Delete(edgeModel.Id);
+                await this.unitOfWork.SaveAsync();
+            }
+        }
+    }
+}

src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingTypesJob.cs

@@ -183,13 +183,13 @@ private async Task DeleteThingTypes(List<DescribeThingTypeResponse> thingTypes)
                     thingTypes.Any(thingType => deviceModel.Id.Equals(thingType.ThingTypeId, StringComparison.Ordinal) && thingType.ThingTypeMetadata.Deprecated))
                 .ToList();
 
-            deviceModelsToDelete.ForEach(async deviceModel =>
+            foreach (var deviceModel in deviceModelsToDelete)
             {
                 await this.deviceModelImageManager.DeleteDeviceModelImageAsync(deviceModel.Id);
                 this.deviceModelRepository.Delete(deviceModel.Id);
-            });
+                await this.unitOfWork.SaveAsync();
+            }
 
-            await this.unitOfWork.SaveAsync();
         }
 
     }

src/AzureIoTHub.Portal.Infrastructure/Services/AWS/AwsConfigService.cs

@@ -145,28 +145,42 @@ private async Task<Dictionary<string, ComponentDeploymentSpecification>> CreateG
             var listcomponentName = new Dictionary<string, ComponentDeploymentSpecification>();
             foreach (var component in edgeModel.EdgeModules)
             {
-                var recipeJson = JsonCreateComponent(component);
-                var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString());
-                var recipeStream = new MemoryStream(recipeBytes);
-
-                var componentVersion = new CreateComponentVersionRequest
+                try
                 {
-                    InlineRecipe = recipeStream
-                };
-                var response = await greengras.CreateComponentVersionAsync(componentVersion);
-                if (response.HttpStatusCode != HttpStatusCode.Created)
+                    _ = await this.greengras.DescribeComponentAsync(new DescribeComponentRequest
+                    {
+                        Arn = $"arn:aws:greengrass:{config.AWSRegion}:{config.AWSAccountId}:components:{component.ModuleName}:versions:{component.Version}"
+                    });
+                    listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
+
+                }
+                catch (Amazon.GreengrassV2.Model.ResourceNotFoundException)
                 {
-                    throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API.");
+                    var recipeJson = JsonCreateComponent(component);
+                    var recipeBytes = Encoding.UTF8.GetBytes(recipeJson.ToString());
+                    var recipeStream = new MemoryStream(recipeBytes);
 
+                    var componentVersion = new CreateComponentVersionRequest
+                    {
+                        InlineRecipe = recipeStream
+                    };
+                    var response = await greengras.CreateComponentVersionAsync(componentVersion);
+                    if (response.HttpStatusCode != HttpStatusCode.Created)
+                    {
+                        throw new InternalServerErrorException("The component creation failed due to an error in the Amazon IoT API.");
+
+                    }
+                    listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
                 }
-                listcomponentName.Add(component.ModuleName, new ComponentDeploymentSpecification { ComponentVersion = component.Version });
             }
 
+
             return listcomponentName;
         }
 
         private static JObject JsonCreateComponent(IoTEdgeModule component)
         {
+
             var environmentVariableObject = new JObject();
 
             foreach (var env in component.EnvironmentVariables)
@@ -329,7 +343,7 @@ public async Task<List<IoTEdgeModule>> GetConfigModuleList(string modelId)
                 }
                 return moduleList;
             }
-            catch (Amazon.IoT.Model.ResourceNotFoundException)
+            catch (Amazon.GreengrassV2.Model.ResourceNotFoundException)
             {
                 throw new InternalServerErrorException("The deployment is not found");
 

src/AzureIoTHub.Portal.Infrastructure/Services/EdgeModelService.cs

@@ -255,13 +255,15 @@ public async Task UpdateEdgeModel(IoTEdgeModel edgeModel)
                 this.labelRepository.Delete(labelEntity.Id);
             }
 
-            _ = this.mapper.Map(edgeModel, edgeModelEntity);
-            this.edgeModelRepository.Update(edgeModelEntity);
-
-            await this.unitOfWork.SaveAsync();
 
+            // For AWS, we do the update in the AwsConfiguration
             if (this.config.CloudProvider.Equals(CloudProviders.Azure, StringComparison.Ordinal))
             {
+                _ = this.mapper.Map(edgeModel, edgeModelEntity);
+
+                this.edgeModelRepository.Update(edgeModelEntity);
+                await this.unitOfWork.SaveAsync();
+
                 await SaveModuleCommands(edgeModel);
             }
 

src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs

@@ -84,6 +84,14 @@ private static IServiceCollection ConfigureAWSSyncJobs(this IServiceCollection s
                     .WithSimpleSchedule(s => s
                             .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes)
                             .RepeatForever()));
+
+                _ = q.AddJob<SyncGreenGrassDeploymentsJob>(j => j.WithIdentity(nameof(SyncGreenGrassDeploymentsJob)))
+                    .AddTrigger(t => t
+                        .WithIdentity($"{nameof(SyncGreenGrassDeploymentsJob)}")
+                        .ForJob(nameof(SyncGreenGrassDeploymentsJob))
+                    .WithSimpleSchedule(s => s
+                            .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes)
+                            .RepeatForever()));
             });
         }