Merge pull request #4 from Hekku2/feat/process-updates

Update documentation and improve infrastructure

.env-sample

@@ -1,3 +1,7 @@
+# This is just a sample file. You can use `Create-Settings.ps1` to generate
+# these files automatically. However, if the script doesn't work, use this
+# as the base .env file for `docker-compose.yml`
+# This file is NOT needed if docker is not used.
 DISCORD_TOKEN=
 DISCORD_GUILDID=
 DISCORD_CHANNELID=

Create-Environment.ps1

@@ -7,9 +7,13 @@
 
  .PARAMETER SettinsFile
  Settings file that contains environment settings. Defaults to 'developer-settings.json'
+
+ .PARAMETER NoDiscord
+ If set, discord sending is disabled. Useful for CI/CD and testing.
 #>
 param(
- [Parameter()][string]$SettingsFile = 'developer-settings.json'
+ [Parameter()][string]$SettingsFile = 'developer-settings.json',
+ [Parameter()][switch]$NoDiscord
 )
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version Latest
@@ -20,7 +24,7 @@ Write-Host "Reading settings from file $SettingsFile"
 $settingsJson = Get-Content -Raw -Path $SettingsFile | ConvertFrom-Json
 
 Write-Host 'Checking if there is an existing installation...'
-$webSitePackageLocation = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name "func-$($settingsJson.ApplicationName)" -ErrorAction SilentlyContinue | Get-WebSitePackage
+$webSitePackageLocation = Get-AzWebApp -ResourceGroupName $settingsJson.ResourceGroup -Name "func-$($settingsJson.ApplicationName)" -ErrorAction SilentlyContinue | Get-WebSitePackage -ErrorAction SilentlyContinue
 if ($webSitePackageLocation) {
  Write-Host "Function app already exist with website package, using it..."
 }
@@ -35,6 +39,7 @@ Write-Host 'Deploying template...'
 $parameters = @{
  baseName = $settingsJson.ApplicationName
  webSitePackageLocation = $webSitePackageLocation
+ disableDiscordSending = $NoDiscord ? $true : $false
  discordSettings = @{
  token = $settingsJson.DiscordToken
  channelId = $settingsJson.DiscordChannelId

Create-Settings.ps1

@@ -0,0 +1,42 @@
+<#
+ .SYNOPSIS
+ This script generates envvironmetn variable files based on developer-settings.json
+
+ .DESCRIPTION
+ Purpose of this is to make starting the development more convenient by generating environment files.
+
+ .PARAMETER SettinsFile
+ Settings file that contains environment settings. Defaults to 'developer-settings.json'
+#>
+param(
+ [Parameter()][string]$SettingsFile = 'developer-settings.json'
+)
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version Latest
+
+.$PSScriptRoot/scripts/FunctionUtil.ps1
+
+Write-Host "Reading settings from file $SettingsFile"
+$settingsJson = Get-Content -Raw -Path $SettingsFile | ConvertFrom-Json
+
+# Docker compose support
+$dockerEnvFile = "$PSScriptRoot/.env"
+$dockerEnvFileContent = "
+# This file was generated by Create-Settings.ps1. Don't commit this to version control.
+DISCORD_TOKEN=$($settingsJson.DiscordToken)
+DISCORD_GUILDID=$($settingsJson.DiscordGuildId)
+DISCORD_CHANNELID=$($settingsJson.DiscordChannelId)
+"
+Write-Host "Writing $dockerEnvFile"
+$dockerEnvFileContent | Out-File -FilePath $dockerEnvFile -Encoding utf8
+
+# Function Core Tools support
+$funcSettingsFile = "$PSScriptRoot/src/FunctionApp.Isolated/local.settings.json"
+$localSettings = @{
+ IsEncrypted = $false
+ Values = @{
+ AzureWebJobsStorage = 'UseDevelopmentStorage=true'
+ FUNCTIONS_WORKER_RUNTIME = 'dotnet-isolated'
+ }
+}
+$localSettings | ConvertTo-Json | Out-File -FilePath $funcSettingsFile -Encoding utf8

README.md

@@ -7,21 +7,41 @@ Currently this is designed to work with one server and one channel.
 
 ## How does it work?
 
-In short, software periodically selects random image from image storage and sends it to discord channel.
+In short, software periodically selects random image from image storage and
+sends it to discord channel.
 
 In depth
  1. Image sending is periodically triggered by Function App TimerTrigger.
- 1. When Image sending is triggered, software selects a random image from index. Selection is not really random, it prefers less posted images. See [RandomizationService](src/Common/RandomizationService/RandomizationService.cs) for implementation details.
- * If there is no index, software builds an index of images available in source.
- 1. After image is selected, software downloads the image from storage and sends the image to the chosen Discord channel.
- * If Imageis removed, error is logged and image is not sent to channel.
+ 1. When Image sending is triggered, software selects a random image from
+ index. Selection is not really random, it prefers less posted images. See
+ [RandomizationService](src/Common/RandomizationService/RandomizationService.cs)
+ for implementation details.
+ * If there is no index, software builds an index of images available in
+ source.
+ 1. After image is selected, software downloads the image from storage and
+ sends the image to the chosen Discord channel.
+ * If Image is removed, error is logged and image is not sent to channel.
 
 Indexing
  * Indexing is performed automatically if index doesn't exist.
  * Image index can be regenerated by calling the related function.
  * Currently changes in storage don't trigger image index regeneration.
- * The index contains data of the images and how often those have been posted and other similar metadata.
- * Images can also be ignored. Ignored images are not selected by randomization logic.
+ * The index contains data of the images and how often those have been posted
+ and other similar metadata.
+ * Images can also be ignored. Ignored images are not selected by randomization
+ logic.
+
+## Tools
+
+This section lists tools that are used in developing and deploying this
+software. Some are not strictly
+
+Development
+ * Dotnet 8.0 (or later)
+ * Azure Powershell
+ * Bicep with version that supports importing user types (0.28.1 is
+ enough, earlier may also suffice)
+ * Docker (and docker compose).
 
 ## Deployment and running
 

doc/technical-reasoning.md

@@ -0,0 +1,47 @@
+# Technical reasoning
+
+This document describes reasoning behind different architectural and technical
+decisions related to this project. Purpose of this is help with future
+refactoring because the technical landscape and the available Azure services
+may have changed from the time these decisions where made. This should also
+help choosing new implementations I have forgot to think about some aspect
+or I haven't known about some available tool etc.
+
+## Requirements
+
+This sections lists overall requirements.
+
+ 1. Main function of the software (Image sending) is atomatically activated
+ periodically.
+ 2. Software needs to be able to handle HTTP requests from Discord and from
+ users. These should be relatively rare.
+ * Discord MAY add a response time requirement.
+ 1. Software should prefer Managed Identity authentication.
+ 1. Software should be cheap to run and it shouldn't cost much when it's not
+ active.
+ 1. Secrets should be stored securely.
+ 1. Local development with containers should be supported.
+ 1. Infrastructure creation must be automatic with a single command.
+
+## Technical selection
+
+Azure Function App with dotnet-isolated runtime was chosen.
+
+ * Azure
+ * Can handle all of the requirements, but mainly because I'm most familiar
+ with this cloud service provider.
+ * Azure Function App
+ * Should be cheap to run.
+ * Supports HTTP requests
+ * Supports Managed Identity
+ * Can be activated periodically (`TimerTrigger`)
+ * Supports containers for local development.
+ * Isolated worker model
+ * Chosen instead of In-process model, because In-Process support is ending.
+ * Consumption mode for Function App
+ * Should be the cheapest option for this kind of workload.
+ * App Service plan
+ * Chosen instead of Container Apps, because the development experiment with
+ container app was still quite poor.
+ * Azure Key Vault
+ * Is a secure way to store secrets and integrates well with function app.

infra/functions.bicep

@@ -18,6 +18,9 @@ param location string = resourceGroup().location
 @description('Web site package location. Leave empty if none is found.')
 param webSitePackageLocation string = ''
 
+@description('If true, messages are not sent to Discord. This should only be used when testing.')
+param disableDiscordSending bool = false
+
 var hostingPlanName = 'asp-${baseName}'
 var functionAppName = 'func-${baseName}'
 var storageBlobDataOwnerRoleDefinitionId = 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b'
@@ -86,7 +89,11 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
  use32BitWorkerProcess: false
  ftpsState: 'FtpsOnly'
  minTlsVersion: '1.2'
-
+ cors: {
+ allowedOrigins: [
+ 'https://portal.azure.com'
+ ]
+ }
  appSettings: [
  {
  name: 'AzureWebJobsStorage__accountName'
@@ -112,6 +119,10 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
  name: 'APPINSIGHTS_INSTRUMENTATIONKEY'
  value: reference(applicationInsights.id, '2015-05-01').InstrumentationKey
  }
+ {
+ name: 'FeatureSettings__DisableDiscordSending'
+ value: '${disableDiscordSending}'
+ }
  {
  name: '${discordSettingsKey}__Token'
  value: discordSettings.token
@@ -125,24 +136,16 @@ resource functionApp 'Microsoft.Web/sites@2021-02-01' = {
  value: '${discordSettings.channelId}'
  }
  {
- name: '${blobStorageKey}__ConnectionString'
- value: imageStorageSettings.connectionString
- }
- {
- name: '${blobStorageKey}__ContainerName'
- value: imageStorageSettings.containerName
+ name: '${blobStorageKey}__BlobContainerUri'
+ value: imageStorageSettings.blobContainerUri
  }
  {
  name: '${blobStorageKey}__FolderPath'
  value: imageStorageSettings.folderPath
  }
  {
- name: '${imageIndexStorageKey}__ConnectionString'
- value: imageStorageSettings.connectionString
- }
- {
- name: '${imageIndexStorageKey}__ContainerName'
- value: 'index'
+ name: '${imageIndexStorageKey}__BlobContainerUri'
+ value: '${functionStorageAccount.properties.primaryEndpoints.blob}index'
  }
  ]
  }
@@ -163,3 +166,4 @@ resource functionAppFunctionBlobStorageAccess 'Microsoft.Authorization/roleAssig
 }
 
 output functionAppPrincipalId string = functionApp.identity.principalId
+output functionAppResourceId string = functionApp.id

infra/main.bicep

@@ -12,6 +12,9 @@ param location string = resourceGroup().location
 @description('Web site package location. Leave empty if none is found.')
 param webSitePackageLocation string = ''
 
+@description('If true, messages are not sent to Discord. This should only be used when testing.')
+param disableDiscordSending bool = false
+
 module appInsights 'app-insights.bicep' = {
  name: 'application-insights'
  params: {
@@ -39,9 +42,19 @@ resource imageStorage 'Microsoft.Storage/storageAccounts@2022-05-01' = {
  }
 }
 
+var imageContainerName = 'images'
+resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = {
+ parent: imageStorage
+ name: 'default'
+}
+
+resource container 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = {
+ parent: blobServices
+ name: imageContainerName
+}
+
 var imageSettings = {
- connectionString: 'DefaultEndpointsProtocol=https;AccountName=${imageStorage.name};EndpointSuffix=${az.environment().suffixes.storage};AccountKey=${imageStorage.listKeys().keys[0].value}'
- containerName: 'images'
+ blobContainerUri: '${imageStorage.properties.primaryEndpoints.blob}${imageContainerName}'
  folderPath: 'root'
 }
 
@@ -54,5 +67,21 @@ module functions 'functions.bicep' = {
  discordSettings: discordSettings
  imageStorageSettings: imageSettings
  webSitePackageLocation: webSitePackageLocation
+ disableDiscordSending: disableDiscordSending
+ }
+}
+
+// TODO Also this assignment could probably be a separate module etc.
+var storageBlobDataReaderRoleDefinitionId = '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1'
+resource functionAppFunctionBlobStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+ scope: container
+ name: guid(functions.name, storageBlobDataReaderRoleDefinitionId, container.id)
+ properties: {
+ principalId: functions.outputs.functionAppPrincipalId
+ principalType: 'ServicePrincipal'
+ roleDefinitionId: subscriptionResourceId(
+ 'Microsoft.Authorization/roleDefinitions',
+ storageBlobDataReaderRoleDefinitionId
+ )
  }
 }

infra/types.bicep

@@ -7,7 +7,6 @@ type DiscordSettings = {
 
 @export()
 type ImageStorageSettings = {
- connectionString: string
- containerName: string
+ blobContainerUri: string
  folderPath: string
 }

src/Common/BlobStorageImageService/BlobStorageImageSourceOptions.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.DataAnnotations;
+using Microsoft.Identity.Client;
 
 namespace DiscordImagePoster.Common.BlobStorageImageService;
 
@@ -16,9 +17,17 @@ public class BlobStorageImageSourceOptions
 
  /// <summary>
  /// The name of the container where the images are stored.
+ /// This is not needed (nor used) if BlobContainerUri is used.
  /// </summary>
- [Required]
- public required string ContainerName { get; set; }
+ public required string? ContainerName { get; set; }
+
+ /// <summary>
+ /// The URI to the container where the images are stored.
+ /// https://{account_name}.blob.core.windows.net/{container_name}
+ /// 
+ /// If this is used, the ConnectionString is not needed and managed identity is used
+ /// </summary>
+ public required string? BlobContainerUri { get; set; }
 
  /// <summary>
  /// The path to the folder where the images are stored.

src/Common/IndexService/BlobStorageIndexStorageService.cs

@@ -2,29 +2,32 @@
 using Azure.Storage.Blobs;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace DiscordImagePoster.Common.IndexService;
 
 public class BlobStorageIndexStorageService : IIndexStorageService
 {
- private ILogger<BlobStorageIndexStorageService> _logger;
+ private readonly ILogger<BlobStorageIndexStorageService> _logger;
+ private readonly ImageIndexOptions _options;
  private readonly BlobContainerClient _blobContainerClient;
 
- private const string IndexBlobName = "index.json";
-
  public BlobStorageIndexStorageService
  (
  ILogger<BlobStorageIndexStorageService> logger,
+ IOptions<ImageIndexOptions> options,
  [FromKeyedServices(KeyedServiceConstants.ImageIndexBlobContainerClient)] BlobContainerClient blobContainerClient
  )
  {
  _logger = logger;
+ _options = options.Value;
  _blobContainerClient = blobContainerClient;
  }
 
  public async Task<ImageIndex?> GetImageIndexAsync()
  {
- var blobClient = _blobContainerClient.GetBlobClient(IndexBlobName);
+ _logger.LogTrace("Getting image index from {IndexFileName}", _options.IndexFileName);
+ var blobClient = _blobContainerClient.GetBlobClient(_options.IndexFileName);
  var exists = await blobClient.ExistsAsync();
  if (!exists)
  {
@@ -38,9 +41,10 @@ public BlobStorageIndexStorageService
 
  public async Task UpdateIndexAsync(ImageIndex index)
  {
+ _logger.LogTrace("Updating image index to {IndexFileName}", _options.IndexFileName);
  await _blobContainerClient.CreateIfNotExistsAsync();
  var bytes = JsonSerializer.SerializeToUtf8Bytes(index);
- var blobClient = _blobContainerClient.GetBlobClient(IndexBlobName);
+ var blobClient = _blobContainerClient.GetBlobClient(_options.IndexFileName);
  await blobClient.UploadAsync(new MemoryStream(bytes), true);
  }
 }