From d64d791db96298bf2a8080c6963472525977f9e1 Mon Sep 17 00:00:00 2001 From: Kevin BEAUGRAND <9513635+kbeaugrand@users.noreply.github.com> Date: Thu, 25 Aug 2022 12:18:46 +0200 Subject: [PATCH] Feature/add pgsql database connection (#1121) * Add entityframework + PGSql nuget packages * Add PGSql database context + initial database creation --- .github/workflows/build.yml | 4 +-- .github/workflows/codeql.yml | 2 +- .../Server/DevelopmentConfigHandlerTests.cs | 1 + .../Server/ProductionConfigHandlerTests.cs | 1 + src/AzureIoTHub.Portal.sln | 6 ++++ .../Server/AzureIoTHub.Portal.Server.csproj | 10 +++++++ .../Server/ConfigHandler.cs | 3 ++ .../Server/DevelopmentConfigHandler.cs | 2 ++ .../20220822164454_InitialCreate.Designer.cs | 28 +++++++++++++++++++ .../20220822164454_InitialCreate.cs | 20 +++++++++++++ .../PortalDbContextModelSnapshot.cs | 26 +++++++++++++++++ .../Server/Model/PortalDbContext.cs | 14 ++++++++++ .../Server/ProductionConfigHandler.cs | 2 ++ src/AzureIoTHub.Portal/Server/Startup.cs | 19 ++++++++++++- src/docker-compose.dcproj | 18 ++++++++++++ src/docker-compose.override.yml | 13 +++++++++ src/docker-compose.yml | 13 +++++++++ src/launchSettings.json | 15 ++++++++++ 18 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.Designer.cs create mode 100644 src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.cs create mode 100644 src/AzureIoTHub.Portal/Server/Migrations/PortalDbContextModelSnapshot.cs create mode 100644 src/AzureIoTHub.Portal/Server/Model/PortalDbContext.cs create mode 100644 src/docker-compose.dcproj create mode 100644 src/docker-compose.override.yml create mode 100644 src/docker-compose.yml create mode 100644 src/launchSettings.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af7874264..cf01560eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,11 +28,11 @@ jobs: working-directory: src/ - name: Restore dependencies - run: dotnet restore + run: dotnet restore AzureIoTHub.Portal.sln working-directory: src/ - name: Build - run: dotnet build --no-restore -p:ClientAssetsRestoreCommand="npm ci" + run: dotnet build AzureIoTHub.Portal.sln --no-restore -p:ClientAssetsRestoreCommand="npm ci" working-directory: src/ - name: Generate Open API documentation diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index cc733b387..19088c938 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: queries: +security-and-quality,security-extended - name: Build - run: dotnet build --configuration Release + run: dotnet build AzureIoTHub.Portal.sln --configuration Release working-directory: src/ - name: Perform CodeQL Analysis diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Server/DevelopmentConfigHandlerTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Server/DevelopmentConfigHandlerTests.cs index 558127e6a..c1c193300 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Server/DevelopmentConfigHandlerTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Server/DevelopmentConfigHandlerTests.cs @@ -46,6 +46,7 @@ private DevelopmentConfigHandler CreateDevelopmentConfigHandler() [TestCase(ConfigHandler.IoTHubConnectionStringKey, nameof(ConfigHandler.IoTHubConnectionString))] [TestCase(ConfigHandler.DPSConnectionStringKey, nameof(ConfigHandler.DPSConnectionString))] [TestCase(ConfigHandler.StorageAccountConnectionStringKey, nameof(ConfigHandler.StorageAccountConnectionString))] + [TestCase(ConfigHandler.PostgreSQLConnectionStringKey, nameof(ConfigHandler.PostgreSQLConnectionString))] public void SettingsShouldGetValueFromAppSettings(string configKey, string configPropertyName) { // Arrange diff --git a/src/AzureIoTHub.Portal.Tests.Unit/Server/ProductionConfigHandlerTests.cs b/src/AzureIoTHub.Portal.Tests.Unit/Server/ProductionConfigHandlerTests.cs index d614898f7..e72113a99 100644 --- a/src/AzureIoTHub.Portal.Tests.Unit/Server/ProductionConfigHandlerTests.cs +++ b/src/AzureIoTHub.Portal.Tests.Unit/Server/ProductionConfigHandlerTests.cs @@ -36,6 +36,7 @@ private ProductionConfigHandler CreateProductionConfigHandler() [TestCase(ConfigHandler.DPSConnectionStringKey, nameof(ConfigHandler.DPSConnectionString))] [TestCase(ConfigHandler.StorageAccountConnectionStringKey, nameof(ConfigHandler.StorageAccountConnectionString))] [TestCase(ConfigHandler.LoRaKeyManagementCodeKey, nameof(ConfigHandler.LoRaKeyManagementCode))] + [TestCase(ConfigHandler.PostgreSQLConnectionStringKey, nameof(ConfigHandler.PostgreSQLConnectionString))] public void SecretsShouldGetValueFromConnectionStrings(string configKey, string configPropertyName) { // Arrange diff --git a/src/AzureIoTHub.Portal.sln b/src/AzureIoTHub.Portal.sln index 08a397110..3eee2fa1c 100644 --- a/src/AzureIoTHub.Portal.sln +++ b/src/AzureIoTHub.Portal.sln @@ -54,6 +54,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ISSUE_TEMPLATES", "ISSUE_TE ..\.github\ISSUE_TEMPLATE\user_story.md = ..\.github\ISSUE_TEMPLATE\user_story.md EndProjectSection EndProject +Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{B9D2DE01-84DE-461F-998C-20B57E4AA021}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -76,6 +78,10 @@ Global {51FD5B90-B422-47BF-83F2-516520CFB124}.Debug|Any CPU.Build.0 = Debug|Any CPU {51FD5B90-B422-47BF-83F2-516520CFB124}.Release|Any CPU.ActiveCfg = Release|Any CPU {51FD5B90-B422-47BF-83F2-516520CFB124}.Release|Any CPU.Build.0 = Release|Any CPU + {B9D2DE01-84DE-461F-998C-20B57E4AA021}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B9D2DE01-84DE-461F-998C-20B57E4AA021}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9D2DE01-84DE-461F-998C-20B57E4AA021}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B9D2DE01-84DE-461F-998C-20B57E4AA021}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/AzureIoTHub.Portal/Server/AzureIoTHub.Portal.Server.csproj b/src/AzureIoTHub.Portal/Server/AzureIoTHub.Portal.Server.csproj index b4608b885..cdd80deab 100644 --- a/src/AzureIoTHub.Portal/Server/AzureIoTHub.Portal.Server.csproj +++ b/src/AzureIoTHub.Portal/Server/AzureIoTHub.Portal.Server.csproj @@ -44,6 +44,12 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + @@ -51,6 +57,7 @@ + @@ -86,5 +93,8 @@ true $(NoWarn); + Linux + ..\.. + ..\..\docker-compose.dcproj diff --git a/src/AzureIoTHub.Portal/Server/ConfigHandler.cs b/src/AzureIoTHub.Portal/Server/ConfigHandler.cs index 042e9e97a..464e465c2 100644 --- a/src/AzureIoTHub.Portal/Server/ConfigHandler.cs +++ b/src/AzureIoTHub.Portal/Server/ConfigHandler.cs @@ -16,6 +16,7 @@ public abstract class ConfigHandler internal const string DPSServiceEndpointKey = "IoTDPS:ServiceEndpoint"; internal const string DPSIDScopeKey = "IoTDPS:IDScope"; internal const string UseSecurityHeadersKey = "UseSecurityHeaders"; + internal const string PostgreSQLConnectionStringKey = "PostgreSQL:ConnectionString"; internal const string OIDCScopeKey = "OIDC:Scope"; internal const string OIDCAuthorityKey = "OIDC:Authority"; @@ -116,5 +117,7 @@ internal static ConfigHandler Create(IWebHostEnvironment env, IConfiguration con internal abstract string IdeasAuthenticationHeader { get; } internal abstract string IdeasAuthenticationToken { get; } + + internal abstract string PostgreSQLConnectionString { get; } } } diff --git a/src/AzureIoTHub.Portal/Server/DevelopmentConfigHandler.cs b/src/AzureIoTHub.Portal/Server/DevelopmentConfigHandler.cs index 1c259dbd5..1279be89a 100644 --- a/src/AzureIoTHub.Portal/Server/DevelopmentConfigHandler.cs +++ b/src/AzureIoTHub.Portal/Server/DevelopmentConfigHandler.cs @@ -68,5 +68,7 @@ internal DevelopmentConfigHandler(IConfiguration config) internal override string IdeasUrl => this.config.GetValue(IdeasUrlKey, string.Empty); internal override string IdeasAuthenticationHeader => this.config.GetValue(IdeasAuthenticationHeaderKey, "Ocp-Apim-Subscription-Key"); internal override string IdeasAuthenticationToken => this.config.GetValue(IdeasAuthenticationTokenKey, string.Empty); + + internal override string PostgreSQLConnectionString => this.config[PostgreSQLConnectionStringKey]; } } diff --git a/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.Designer.cs b/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.Designer.cs new file mode 100644 index 000000000..aedb2480f --- /dev/null +++ b/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.Designer.cs @@ -0,0 +1,28 @@ +// +using AzureIoTHub.Portal.Server.Model; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace AzureIoTHub.Portal.Server.Migrations +{ + [DbContext(typeof(PortalDbContext))] + [Migration("20220822164454_InitialCreate")] + partial class InitialCreate + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.cs b/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.cs new file mode 100644 index 000000000..20d238f60 --- /dev/null +++ b/src/AzureIoTHub.Portal/Server/Migrations/20220822164454_InitialCreate.cs @@ -0,0 +1,20 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Server.Migrations +{ + using Microsoft.EntityFrameworkCore.Migrations; + + public partial class InitialCreate : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + + } + } +} diff --git a/src/AzureIoTHub.Portal/Server/Migrations/PortalDbContextModelSnapshot.cs b/src/AzureIoTHub.Portal/Server/Migrations/PortalDbContextModelSnapshot.cs new file mode 100644 index 000000000..713552a9a --- /dev/null +++ b/src/AzureIoTHub.Portal/Server/Migrations/PortalDbContextModelSnapshot.cs @@ -0,0 +1,26 @@ +// +using AzureIoTHub.Portal.Server.Model; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace AzureIoTHub.Portal.Server.Migrations +{ + [DbContext(typeof(PortalDbContext))] + partial class PortalDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.8") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/AzureIoTHub.Portal/Server/Model/PortalDbContext.cs b/src/AzureIoTHub.Portal/Server/Model/PortalDbContext.cs new file mode 100644 index 000000000..f90f27997 --- /dev/null +++ b/src/AzureIoTHub.Portal/Server/Model/PortalDbContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) CGI France. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AzureIoTHub.Portal.Server.Model +{ + using Microsoft.EntityFrameworkCore; + + public class PortalDbContext : DbContext + { + public PortalDbContext(DbContextOptions options) + : base(options) + { } + } +} diff --git a/src/AzureIoTHub.Portal/Server/ProductionConfigHandler.cs b/src/AzureIoTHub.Portal/Server/ProductionConfigHandler.cs index d9f43c3d6..c6b198794 100644 --- a/src/AzureIoTHub.Portal/Server/ProductionConfigHandler.cs +++ b/src/AzureIoTHub.Portal/Server/ProductionConfigHandler.cs @@ -30,6 +30,8 @@ internal ProductionConfigHandler(IConfiguration config) internal override string StorageAccountConnectionString => this.config.GetConnectionString(StorageAccountConnectionStringKey); + internal override string PostgreSQLConnectionString => this.config.GetConnectionString(PostgreSQLConnectionStringKey); + internal override int StorageAccountDeviceModelImageMaxAge => this.config.GetValue(StorageAccountDeviceModelImageMaxAgeKey, 86400); internal override bool UseSecurityHeaders => this.config.GetValue(UseSecurityHeadersKey, true); diff --git a/src/AzureIoTHub.Portal/Server/Startup.cs b/src/AzureIoTHub.Portal/Server/Startup.cs index 6d4ba596c..7dc1fbae7 100644 --- a/src/AzureIoTHub.Portal/Server/Startup.cs +++ b/src/AzureIoTHub.Portal/Server/Startup.cs @@ -11,6 +11,7 @@ namespace AzureIoTHub.Portal.Server using AutoMapper; using Azure; using Azure.Storage.Blobs; + using AzureIoTHub.Portal.Server.Model; using Exceptions; using Extensions; using Factories; @@ -28,6 +29,7 @@ namespace AzureIoTHub.Portal.Server using Microsoft.AspNetCore.Mvc.Versioning; using Microsoft.Azure.Devices; using Microsoft.Azure.Devices.Provisioning.Service; + using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -95,6 +97,9 @@ public void ConfigureServices(IServiceCollection services) opts.TokenValidationParameters.ValidateTokenReplay = configuration.OIDCValidateTokenReplay; }); + _ = services + .AddDbContext(opts => opts.UseNpgsql(configuration.PostgreSQLConnectionString)); + _ = services.AddSingleton(configuration); _ = services.AddSingleton(new PortalMetric()); @@ -259,6 +264,7 @@ Specify the authorization token got from your IDP as a header. _ = services.AddSingleton(mapper); _ = services.AddHealthChecks() + .AddDbContextCheck() .AddCheck("iothubHealth") .AddCheck("storageAccountHealth") .AddCheck("tableStorageHealth") @@ -365,11 +371,12 @@ public async void Configure(IApplicationBuilder app, IWebHostEnvironment env) }); }); - var deviceModelImageManager = app.ApplicationServices.GetService(); await deviceModelImageManager?.InitializeDefaultImageBlob()!; await deviceModelImageManager?.SyncImagesCacheControl()!; + + await EnsureDatabaseCreatedAndUpToDate(app)!; } private static void UseApiExceptionMiddleware(IApplicationBuilder app) @@ -392,5 +399,15 @@ private Task HandleApiFallback(HttpContext context) context.Response.StatusCode = StatusCodes.Status404NotFound; return Task.CompletedTask; } + + private static async Task EnsureDatabaseCreatedAndUpToDate(IApplicationBuilder app) + { + using var scope = app.ApplicationServices.CreateScope(); + + using var context = scope.ServiceProvider.GetRequiredService(); + + // Create the database if not exists and migrate it using the database bigration scripts. + await context.Database.MigrateAsync(); + } } } diff --git a/src/docker-compose.dcproj b/src/docker-compose.dcproj new file mode 100644 index 000000000..9cdcce1f7 --- /dev/null +++ b/src/docker-compose.dcproj @@ -0,0 +1,18 @@ + + + + 2.1 + Linux + b9d2de01-84de-461f-998c-20b57e4aa021 + LaunchBrowser + {Scheme}://localhost:{ServicePort}/{Scheme}://{ServiceHost}:{ServicePort} + azureiothub.portal.server + + + + docker-compose.yml + + + + + \ No newline at end of file diff --git a/src/docker-compose.override.yml b/src/docker-compose.override.yml new file mode 100644 index 000000000..c92e398e0 --- /dev/null +++ b/src/docker-compose.override.yml @@ -0,0 +1,13 @@ +version: '3.4' + +services: + azureiothub.portal.server: + environment: + - ASPNETCORE_ENVIRONMENT=Development + - ASPNETCORE_URLS=https://+:443;http://+:80 + ports: + - "80" + - "8001:443" + volumes: + - ${APPDATA}/Microsoft/UserSecrets:/root/.microsoft/usersecrets:ro + - ${APPDATA}/ASP.NET/Https:/root/.aspnet/https:ro diff --git a/src/docker-compose.yml b/src/docker-compose.yml new file mode 100644 index 000000000..80354af8c --- /dev/null +++ b/src/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.4' + +services: + database: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: postgrePassword + azureiothub.portal.server: + image: ${DOCKER_REGISTRY-}azureiothubportalserver + build: + context: . + dockerfile: AzureIoTHub.Portal/Server/Dockerfile diff --git a/src/launchSettings.json b/src/launchSettings.json new file mode 100644 index 000000000..be7d71c2f --- /dev/null +++ b/src/launchSettings.json @@ -0,0 +1,15 @@ +{ + "profiles": { + "Docker Compose": { + "commandName": "DockerCompose", + "commandVersion": "1.0", + "composeLaunchAction": "LaunchBrowser", + "composeLaunchServiceName": "azureiothub.portal.server", + "composeLaunchUrl": "https://localhost:8001", + "serviceActions": { + "azureiothub.portal.server": "StartDebugging", + "database": "StartWithoutDebugging" + } + } + } +}