diff --git a/src/AzureIoTHub.Portal.Application/Mappers/AWS/AWSDeviceThingProfile.cs b/src/AzureIoTHub.Portal.Application/Mappers/AWS/AWSDeviceThingProfile.cs index 3f9dddf82..63605c5b2 100644 --- a/src/AzureIoTHub.Portal.Application/Mappers/AWS/AWSDeviceThingProfile.cs +++ b/src/AzureIoTHub.Portal.Application/Mappers/AWS/AWSDeviceThingProfile.cs @@ -34,6 +34,16 @@ public AWSDeviceThingProfile() .ForMember(dest => dest.ThingName, opts => opts.MapFrom(src => src.DeviceName)) .ForMember(dest => dest.Payload, opts => opts.MapFrom(src => EmptyPayload())) .ReverseMap(); + + _ = CreateMap() + .ForMember(dest => dest.Id, opts => opts.MapFrom(src => src.ThingId)) + .ForMember(dest => dest.Name, opts => opts.MapFrom(src => src.ThingName)) + .ForMember(dest => dest.Version, opts => opts.MapFrom(src => src.Version)) + .ForMember(dest => dest.Tags, opts => opts.MapFrom(src => src.Attributes.Select(att => new DeviceTagValue + { + Name = att.Key, + Value = att.Value + }))); } private static MemoryStream EmptyPayload() diff --git a/src/AzureIoTHub.Portal.Domain/Entities/Device.cs b/src/AzureIoTHub.Portal.Domain/Entities/Device.cs index 3bb1fae33..59714969f 100644 --- a/src/AzureIoTHub.Portal.Domain/Entities/Device.cs +++ b/src/AzureIoTHub.Portal.Domain/Entities/Device.cs @@ -3,6 +3,7 @@ namespace AzureIoTHub.Portal.Domain.Entities { + using System.Collections.ObjectModel; using AzureIoTHub.Portal.Domain.Base; public class Device : EntityBase @@ -50,6 +51,6 @@ public class Device : EntityBase /// /// List of custom device tags and their values. /// - public ICollection Tags { get; set; } = default!; + public ICollection Tags { get; set; } = new Collection(); } } diff --git a/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingsJob.cs b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingsJob.cs new file mode 100644 index 000000000..7e6636217 --- /dev/null +++ b/src/AzureIoTHub.Portal.Infrastructure/Jobs/AWS/SyncThingsJob.cs @@ -0,0 +1,159 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Infrastructure.Jobs.AWS +{ + using System.Net; + using System.Threading.Tasks; + using Amazon.IoT; + using Amazon.IoT.Model; + using Amazon.IotData; + using Amazon.IotData.Model; + using AutoMapper; + using AzureIoTHub.Portal.Domain; + using AzureIoTHub.Portal.Domain.Entities; + using AzureIoTHub.Portal.Domain.Repositories; + using Microsoft.Extensions.Logging; + using Quartz; + using Quartz.Util; + + [DisallowConcurrentExecution] + public class SyncThingsJob : IJob + { + + private readonly ILogger logger; + private readonly IMapper mapper; + private readonly IUnitOfWork unitOfWork; + private readonly IDeviceRepository deviceRepository; + private readonly IDeviceModelRepository deviceModelRepository; + private readonly IDeviceTagValueRepository deviceTagValueRepository; + private readonly IAmazonIoT amazonIoTClient; + private readonly IAmazonIotData amazonIoTDataClient; + + public SyncThingsJob( + ILogger logger, + IMapper mapper, + IUnitOfWork unitOfWork, + IDeviceRepository deviceRepository, + IDeviceModelRepository deviceModelRepository, + IDeviceTagValueRepository deviceTagValueRepository, + IAmazonIoT amazonIoTClient, + IAmazonIotData amazonIoTDataClient) + { + this.mapper = mapper; + this.unitOfWork = unitOfWork; + this.deviceRepository = deviceRepository; + this.deviceModelRepository = deviceModelRepository; + this.deviceTagValueRepository = deviceTagValueRepository; + this.amazonIoTClient = amazonIoTClient; + this.amazonIoTDataClient = amazonIoTDataClient; + this.logger = logger; + } + + + public async Task Execute(IJobExecutionContext context) + { + try + { + this.logger.LogInformation("Start of sync Thing Types job"); + + await SyncThingsAsDevices(); + + this.logger.LogInformation("End of sync Thing Types job"); + } + catch (Exception e) + { + this.logger.LogError(e, "Sync Thing Types job has failed"); + } + } + + private async Task SyncThingsAsDevices() + { + var things = await GetAllThings(); + + foreach (var thing in things) + { + //ThingType not specified + if (thing.ThingTypeName.IsNullOrWhiteSpace()) + { + this.logger.LogInformation($"Cannot import device '{thing.ThingName}' since it doesn't have related thing type."); + continue; + } + + //ThingType not find in DB + var deviceModel = this.deviceModelRepository.GetByName(thing.ThingTypeName); + if (deviceModel == null) + { + this.logger.LogWarning($"Cannot import device '{thing.ThingName}'. The ThingType '{thing.ThingTypeName}' doesn't exist"); + continue; + } + + //ThingShadow not specified + var thingShadowRequest = new GetThingShadowRequest() + { + ThingName = thing.ThingName + }; + var thingShadow = await this.amazonIoTDataClient.GetThingShadowAsync(thingShadowRequest); + if (thingShadow.HttpStatusCode.Equals(HttpStatusCode.NotFound)) + { + this.logger.LogWarning($"Cannot import device '{thing.ThingName}' since it doesn't have related thing shadow"); + continue; + } + + //Create or update the thing + await CreateOrUpdateThing(thing, deviceModel); + } + + foreach (var item in (await this.deviceRepository.GetAllAsync()).Where(device => !things.Exists(x => x.ThingId == device.Id))) + { + var deviceEntity = await this.deviceRepository.GetByIdAsync(item.Id, d => d.Tags, d => d.Labels); + this.deviceRepository.Delete(deviceEntity!.Id); + } + + await this.unitOfWork.SaveAsync(); + } + + private async Task> GetAllThings() + { + var things = new List(); + + var response = await amazonIoTClient.ListThingsAsync(); + + foreach (var thing in response.Things) + { + var requestDescribeThing = new DescribeThingRequest + { + ThingName = thing.ThingName + }; + + things.Add(await this.amazonIoTClient.DescribeThingAsync(requestDescribeThing)); + } + + return things; + } + + private async Task CreateOrUpdateThing(DescribeThingResponse thing, DeviceModel deviceModel) + { + var device = this.mapper.Map(thing); + var deviceEntity = await this.deviceRepository.GetByIdAsync(device.Id, d => d.Tags); + device.DeviceModelId = deviceModel.Id; + + if (deviceEntity == null) + { + await this.deviceRepository.InsertAsync(device); + } + else + { + if (deviceEntity.Version >= device.Version) return; + + foreach (var deviceTagEntity in deviceEntity.Tags) + { + this.deviceTagValueRepository.Delete(deviceTagEntity.Id); + } + + _ = this.mapper.Map(device, deviceEntity); + this.deviceRepository.Update(deviceEntity); + } + } + } +} diff --git a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs index 0095297fd..24e9be196 100644 --- a/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs +++ b/src/AzureIoTHub.Portal.Infrastructure/Startup/AWSServiceCollectionExtension.cs @@ -3,6 +3,7 @@ namespace AzureIoTHub.Portal.Infrastructure.Startup { + using System.Net; using Amazon; using Amazon.GreengrassV2; using Amazon.IoT; @@ -19,6 +20,7 @@ namespace AzureIoTHub.Portal.Infrastructure.Startup using AzureIoTHub.Portal.Infrastructure.Services.AWS; using AzureIoTHub.Portal.Models.v10; using Microsoft.Extensions.DependencyInjection; + using Prometheus; using Quartz; public static class AWSServiceCollectionExtension @@ -81,7 +83,15 @@ private static IServiceCollection ConfigureAWSSyncJobs(this IServiceCollection s .AddTrigger(t => t .WithIdentity($"{nameof(SyncThingTypesJob)}") .ForJob(nameof(SyncThingTypesJob)) - .WithSimpleSchedule(s => s + .WithSimpleSchedule(s => s + .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) + .RepeatForever())); + + _ = q.AddJob(j => j.WithIdentity(nameof(SyncThingsJob))) + .AddTrigger(t => t + .WithIdentity($"{nameof(SyncThingsJob)}") + .ForJob(nameof(SyncThingsJob)) + .WithSimpleSchedule(s => s .WithIntervalInMinutes(configuration.SyncDatabaseJobRefreshIntervalInMinutes) .RepeatForever())); diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncThingsJobTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncThingsJobTests.cs new file mode 100644 index 000000000..1d295d619 --- /dev/null +++ b/src/AzureIoTHub.Portal.Tests.Unit/Infrastructure/Jobs/AWS/SyncThingsJobTests.cs @@ -0,0 +1,284 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Tests.Unit.Infrastructure.Jobs.AWS +{ + using System; + using System.Collections.Generic; + using System.Linq.Expressions; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Amazon.IoT; + using Amazon.IoT.Model; + using Amazon.IotData; + using Amazon.IotData.Model; + using AutoFixture; + using AzureIoTHub.Portal.Application.Managers; + using AzureIoTHub.Portal.Domain; + using AzureIoTHub.Portal.Domain.Entities; + using AzureIoTHub.Portal.Domain.Repositories; + using AzureIoTHub.Portal.Infrastructure.Jobs.AWS; + using AzureIoTHub.Portal.Tests.Unit.UnitTests.Bases; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Quartz; + + public class SyncThingsJobTests : BackendUnitTest + { + private IJob syncThingsJob; + + private Mock amazonIoTClient; + private Mock amazonIoTDataClient; + private Mock mockAWSImageManager; + private Mock mockUnitOfWork; + private Mock mockDeviceRepository; + private Mock mockDeviceModelRepository; + private Mock mockDeviceTagValueRepository; + + public override void Setup() + { + base.Setup(); + + this.mockAWSImageManager = MockRepository.Create(); + this.mockUnitOfWork = MockRepository.Create(); + this.mockDeviceRepository = MockRepository.Create(); + this.mockDeviceModelRepository = MockRepository.Create(); + this.mockDeviceTagValueRepository = MockRepository.Create(); + this.amazonIoTClient = MockRepository.Create(); + this.amazonIoTDataClient = MockRepository.Create(); + + _ = ServiceCollection.AddSingleton(this.mockAWSImageManager.Object); + _ = ServiceCollection.AddSingleton(this.mockUnitOfWork.Object); + _ = ServiceCollection.AddSingleton(this.mockDeviceRepository.Object); + _ = ServiceCollection.AddSingleton(this.mockDeviceModelRepository.Object); + _ = ServiceCollection.AddSingleton(this.mockDeviceTagValueRepository.Object); + _ = ServiceCollection.AddSingleton(this.amazonIoTClient.Object); + _ = ServiceCollection.AddSingleton(this.amazonIoTDataClient.Object); + _ = ServiceCollection.AddSingleton(); + + + Services = ServiceCollection.BuildServiceProvider(); + + this.syncThingsJob = Services.GetRequiredService(); + } + + [Test] + public async Task ExecuteNewDeviceDeviceCreated() + { + // Arrange + var mockJobExecutionContext = MockRepository.Create(); + + var expectedDeviceModel = Fixture.Create(); + var newDevice = new Device + { + Id = Fixture.Create(), + Name = Fixture.Create(), + DeviceModel = expectedDeviceModel, + DeviceModelId = expectedDeviceModel.Id, + Version = 1 + }; + + var thingsListing = new ListThingsResponse + { + Things = new List() + { + new ThingAttribute + { + ThingName = newDevice.Name + } + } + }; + + _ = this.amazonIoTClient.Setup(client => client.ListThingsAsync(It.IsAny())) + .ReturnsAsync(thingsListing); + + _ = this.amazonIoTClient.Setup(client => client.DescribeThingAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingResponse() + { + ThingId = newDevice.Id, + ThingName = newDevice.Name, + ThingTypeName = newDevice.DeviceModel.Name, + Version = newDevice.Version + }); + + _ = this.mockDeviceModelRepository + .Setup(x => x.GetByName(newDevice.DeviceModel.Name)) + .Returns(expectedDeviceModel); + + _ = this.amazonIoTDataClient.Setup(client => client.GetThingShadowAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new GetThingShadowResponse() { HttpStatusCode = HttpStatusCode.OK }); + + _ = this.mockDeviceRepository.Setup(repository => repository.GetByIdAsync(newDevice.Id, d => d.Tags)) + .ReturnsAsync((Device)null); + + _ = this.mockDeviceRepository.Setup(repository => repository.InsertAsync(It.IsAny())) + .Returns(Task.CompletedTask); + + _ = this.mockDeviceRepository.Setup(x => x.GetAllAsync(It.IsAny>>(), It.IsAny())) + .ReturnsAsync(new List + { + newDevice, + new Device + { + Id = Guid.NewGuid().ToString() + } + }); + + this.mockDeviceRepository.Setup(x => x.Delete(It.IsAny())).Verifiable(); + + _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) + .Returns(Task.CompletedTask); + + // Act + await this.syncThingsJob.Execute(mockJobExecutionContext.Object); + + // Assert + MockRepository.VerifyAll(); + } + + [Test] + public async Task ExecuteExistingDeviceWithHigherVersionDeviceUpdated() + { + // Arrange + var mockJobExecutionContext = MockRepository.Create(); + + var expectedDeviceModel = Fixture.Create(); + var existingDevice = new Device + { + Id = Fixture.Create(), + Name = Fixture.Create(), + DeviceModel = expectedDeviceModel, + DeviceModelId = expectedDeviceModel.Id, + Version = 1, + Tags = new List() + { + new() + { + Id = Fixture.Create(), + Name = Fixture.Create(), + Value = Fixture.Create() + } + } + }; + + var thingsListing = new ListThingsResponse + { + Things = new List() + { + new ThingAttribute + { + ThingName = existingDevice.Name + } + } + }; + + _ = this.amazonIoTClient.Setup(client => client.ListThingsAsync(It.IsAny())) + .ReturnsAsync(thingsListing); + + _ = this.amazonIoTClient.Setup(client => client.DescribeThingAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingResponse() + { + ThingId = existingDevice.Id, + ThingName = existingDevice.Name, + ThingTypeName = existingDevice.DeviceModel.Name, + Version = 2 + }); + + _ = this.mockDeviceModelRepository + .Setup(x => x.GetByName(existingDevice.DeviceModel.Name)) + .Returns(expectedDeviceModel); + + _ = this.amazonIoTDataClient.Setup(client => client.GetThingShadowAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new GetThingShadowResponse() { HttpStatusCode = HttpStatusCode.OK }); + + _ = this.mockDeviceRepository.Setup(repository => repository.GetByIdAsync(existingDevice.Id, d => d.Tags)) + .ReturnsAsync(existingDevice); + + this.mockDeviceTagValueRepository.Setup(repository => repository.Delete(It.IsAny())).Verifiable(); + + this.mockDeviceRepository.Setup(repository => repository.Update(It.IsAny())).Verifiable(); + + _ = this.mockDeviceRepository.Setup(x => x.GetAllAsync(It.IsAny>>(), It.IsAny())) + .ReturnsAsync(new List + { + existingDevice + }); + + _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) + .Returns(Task.CompletedTask); + + // Act + await this.syncThingsJob.Execute(mockJobExecutionContext.Object); + + // Assert + MockRepository.VerifyAll(); + } + + [Test] + public async Task ExecuteExistingDeviceWithOlderVersionDeviceNotUpdated() + { + // Arrange + var mockJobExecutionContext = MockRepository.Create(); + + var expectedDeviceModel = Fixture.Create(); + var existingDevice = new Device + { + Id = Fixture.Create(), + Name = Fixture.Create(), + DeviceModel = expectedDeviceModel, + DeviceModelId = expectedDeviceModel.Id, + Version = 2 + }; + + var thingsListing = new ListThingsResponse + { + Things = new List() + { + new ThingAttribute + { + ThingName = existingDevice.Name + } + } + }; + + _ = this.amazonIoTClient.Setup(client => client.ListThingsAsync(It.IsAny())) + .ReturnsAsync(thingsListing); + + _ = this.amazonIoTClient.Setup(client => client.DescribeThingAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DescribeThingResponse() + { + ThingId = existingDevice.Id, + ThingName = existingDevice.Name, + ThingTypeName = existingDevice.DeviceModel.Name, + Version = 1 + }); + + _ = this.mockDeviceModelRepository + .Setup(x => x.GetByName(existingDevice.DeviceModel.Name)) + .Returns(expectedDeviceModel); + + _ = this.amazonIoTDataClient.Setup(client => client.GetThingShadowAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new GetThingShadowResponse() { HttpStatusCode = HttpStatusCode.OK }); + + _ = this.mockDeviceRepository.Setup(repository => repository.GetByIdAsync(existingDevice.Id, d => d.Tags)) + .ReturnsAsync(existingDevice); + + _ = this.mockDeviceRepository.Setup(x => x.GetAllAsync(It.IsAny>>(), It.IsAny())) + .ReturnsAsync(new List + { + existingDevice + }); + + _ = this.mockUnitOfWork.Setup(work => work.SaveAsync()) + .Returns(Task.CompletedTask); + + // Act + await this.syncThingsJob.Execute(mockJobExecutionContext.Object); + + // Assert + MockRepository.VerifyAll(); + } + } +}