From 931778a2056b7ab666da6c71098abc9512760f41 Mon Sep 17 00:00:00 2001
From: yophilav <54859653+yophilav@users.noreply.github.com>
Date: Thu, 7 Jan 2021 14:11:24 -0800
Subject: [PATCH] ARM Base Image Update Pipeline (#4160)

First take on automating ARM Base Image update.

Use case scenario:
1. tools/BaseImageUpdate/baseImage.ps1 is a helper script that a developer can use on his powershell to update the ARM base image version.  i.e. `Update-ARM-BaseImages -NewASPNetCoreVersion "2.1.23"`.
2. A developer send out a Pull request with base image changes.
3. A developer uses a pipeline `builds/misc/base-images-publish.yaml` to kick off the build & base image publication.
4. (Optional) As a good measure, once (3) is completed, a developer can run the build image pipeline to verify a successful publication.
---
 builds/misc/base-images-publish.yaml          | 45 +++++++++++
 .../base-image-build-and-publish.yaml         | 67 ++++++++++++++++
 .../base-image-clear-docker-cache.yaml        | 10 +++
 .../base-image-docker-login.yaml              |  6 ++
 tools/BaseImageUpdate/baseImage.ps1           | 77 +++++++++++++++++++
 5 files changed, 205 insertions(+)
 create mode 100644 builds/misc/base-images-publish.yaml
 create mode 100644 builds/misc/templates/base-image-publish/base-image-build-and-publish.yaml
 create mode 100644 builds/misc/templates/base-image-publish/base-image-clear-docker-cache.yaml
 create mode 100644 builds/misc/templates/base-image-publish/base-image-docker-login.yaml
 create mode 100644 tools/BaseImageUpdate/baseImage.ps1

diff --git a/builds/misc/base-images-publish.yaml b/builds/misc/base-images-publish.yaml
new file mode 100644
index 00000000000..19b58eff7c8
--- /dev/null
+++ b/builds/misc/base-images-publish.yaml
@@ -0,0 +1,45 @@
+trigger: none
+pr: none
+
+jobs:
+################################################################################
+  - job: linux_arm32
+################################################################################
+    displayName: Linux ARM32 Base Image Publication
+    pool:
+      name: $(pool.name)
+      demands:
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals ARM
+        - agent-osbits -equals 32
+        - run-base-image-release -equals true
+
+    variables:
+      os: linux
+      arch: arm32v7
+
+    steps:
+    - template: templates/base-image-publish/base-image-clear-docker-cache.yaml
+    - template: templates/base-image-publish/base-image-docker-login.yaml
+    - template: templates/base-image-publish/base-image-build-and-publish.yaml
+
+################################################################################
+  - job: linux_arm64
+################################################################################
+    displayName: Linux ARM64 Base Image Publication
+    pool:
+      name: $(pool.name)
+      demands:
+        - Agent.OS -equals Linux
+        - Agent.OSArchitecture -equals ARM64
+        - agent-osbits -equals 64
+        - run-base-image-release -equals true
+
+    variables:
+      os: linux
+      arch: arm64v8
+
+    steps:
+    - template: templates/base-image-publish/base-image-clear-docker-cache.yaml
+    - template: templates/base-image-publish/base-image-docker-login.yaml
+    - template: templates/base-image-publish/base-image-build-and-publish.yaml
\ No newline at end of file
diff --git a/builds/misc/templates/base-image-publish/base-image-build-and-publish.yaml b/builds/misc/templates/base-image-publish/base-image-build-and-publish.yaml
new file mode 100644
index 00000000000..970b3932f14
--- /dev/null
+++ b/builds/misc/templates/base-image-publish/base-image-build-and-publish.yaml
@@ -0,0 +1,67 @@
+steps:
+- pwsh: |
+    # Config the base image publication
+    $config = New-Object System.Collections.ArrayList
+    # Tuple format:  (ModuleRepoPath,                           BaseImageName,                   ReturningVstsVar)
+    $config.AddRange((
+      [Tuple]::Create("edge-agent",                             "azureiotedge-agent-base",       "edgeAgentTag"),
+      [Tuple]::Create("edge-hub",                               "azureiotedge-hub-base",         "edgeHubTag"),
+      [Tuple]::Create("edge-modules/SimulatedTemperatureSensor","azureiotedge-module-base",      "moduleTag"),
+      [Tuple]::Create("test/modules/TestAnalyzer",              "azureiotedge-module-base-full", "fullModuleTag")
+      ));
+
+    # Download librocksdb from blob storage
+    $artifactLocale = "$(Build.SourcesDirectory)/librocksdb.so"
+    Invoke-WebRequest -Uri $("https://edgebuild.blob.core.windows.net/librocksdb/librocksdb.so.$(arch)") -OutFile $artifactLocale
+
+    # Get all the tag for publication
+    Write-Output "`n`nBase Image Info"
+    foreach ($eachConfig in $config)
+    {
+      # Fetch tags & version from each module Dockerfiles {../$(arch)/base/Dockerfile, ../$(arch)/Dockerfile}
+      # Steps: 
+      #  1. Go to a respective Dockerfile as defined in $config for an underlying base image
+      #  2. Read underlying base image tag using "ARG base_tag="
+      #  3. Go to a Dockerfile of the module to get tagging that refers to underly base image (2)
+      #  4. Set a variable with the tag obtained in (3). This variable has the same name as config by item3 of $config
+      #  5. Output the read values. Format:  <BaseImageName>:<ReferringTag> 🠘 <UnderLyingBaseImageTag>
+      $pathPrefix = "$(Build.SourcesDirectory)/$($eachConfig.Item1)/docker/$(os)/$(arch)"
+      $unberlyingFilePath = $pathPrefix + '/base/Dockerfile' | Resolve-path
+      $underlyingTag = $($(Get-Content $unberlyingFilePath | Select-String "ARG base_tag=") -split "=")[1]
+      $imageFilePath = $pathPrefix + '/Dockerfile' | Resolve-path
+      Set-Variable -Name "$($eachConfig.Item3)" -Value "$($($(Get-Content $imageFilePath | Select-String "ARG base_tag=") -split "=")[1])"
+      Write-Output "$($eachConfig.Item2):$(Get-Variable -Name "$($eachConfig.Item3)" -ValueOnly) 🠘 $underlyingTag"
+
+      # Extract version number from the *-linux* base image
+      # Uses ($config.item3) + 'Version' to return the version string as an environment variable
+      #
+      # Note:
+      #    The output {$(edgeAgentTagVersion), $(edgeHubTagVersion), $(moduleTagVersion), $(fullModuleTagVersion)} are used to publish artifacts
+      Set-Variable -Name "$($eachConfig.Item3)Version" -Value "$($(Get-Variable -Name "$($eachConfig.Item3)" -ValueOnly).split("-")[0])"
+
+      # Copy librocksdb.so to the same directory where base image is going to be built
+      $filePath = "$(Build.SourcesDirectory)/$($eachConfig.Item1)/docker/$(os)/$(arch)/base/librocksdb.so"
+      Copy-Item $artifactLocale -Destination "$filePath" -ErrorAction Stop
+    }
+
+    # Build base images & publish them
+    cd $(Build.SourcesDirectory)
+    $scriptPath = "$(Build.SourcesDirectory)/scripts/linux/createArmBase.sh"
+
+    Write-Output "`n`nBuild & Publish"
+    foreach ($eachConfig in $config)
+    {
+      $versionTag = $(Get-Variable -Name "$($eachConfig.Item3)Version" -ValueOnly)
+      $buildCommand = " ``sh -c ""$scriptPath -d $($eachConfig.Item1) -i $($eachConfig.Item2) -v $versionTag""`` "
+      echo "Command: $buildCommand"
+      Invoke-Expression -Command "$buildCommand"
+
+      # Clean up librocksdb.so that we copied here.  
+      $filePath = "$(Build.SourcesDirectory)/$($eachConfig.Item1)/docker/$(os)/$(arch)/base/librocksdb.so"
+      Remove-Item $filePath
+    }
+
+    # Clean up librocksdb.so
+    Remove-Item $artifactLocale
+  displayName: Publish
+  name: publish
\ No newline at end of file
diff --git a/builds/misc/templates/base-image-publish/base-image-clear-docker-cache.yaml b/builds/misc/templates/base-image-publish/base-image-clear-docker-cache.yaml
new file mode 100644
index 00000000000..974d1061a5d
--- /dev/null
+++ b/builds/misc/templates/base-image-publish/base-image-clear-docker-cache.yaml
@@ -0,0 +1,10 @@
+steps:
+- pwsh: |
+    $isNoModuleRunning = [string]::IsNullOrEmpty($(docker ps -aq))
+    if (!$isNoModuleRunning)
+    {
+      docker stop $(docker ps -aq)
+    }
+
+    docker system prune --volumes -af
+  displayName: Clear cached docker images
\ No newline at end of file
diff --git a/builds/misc/templates/base-image-publish/base-image-docker-login.yaml b/builds/misc/templates/base-image-publish/base-image-docker-login.yaml
new file mode 100644
index 00000000000..a2b39244f5e
--- /dev/null
+++ b/builds/misc/templates/base-image-publish/base-image-docker-login.yaml
@@ -0,0 +1,6 @@
+steps:
+- task: Docker@2
+  displayName: Docker Login
+  inputs:
+    command: login
+    containerRegistry: $(user.dockerConnection)
\ No newline at end of file
diff --git a/tools/BaseImageUpdate/baseImage.ps1 b/tools/BaseImageUpdate/baseImage.ps1
new file mode 100644
index 00000000000..e4253d6a37c
--- /dev/null
+++ b/tools/BaseImageUpdate/baseImage.ps1
@@ -0,0 +1,77 @@
+# Run this on E2E test agent in PWSH
+function Setup-BaseImage-Script
+{
+    # Change current directory to ../iotedge
+    $rootDir = $env:BUILD_SOURCESDIRECTORY
+    if (-not $rootDir) {
+        $rootDir = [IO.Path]::Combine($PSScriptRoot, "..", "..")
+    }
+
+    cd $rootDir
+
+    $isDotNetInstalled = ((gp HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*).DisplayName -Match "Microsoft .NET Core Runtime").Length -gt 0;
+    if ($isDotNetInstalled)
+    {
+        # Cleaning build artifacts as it slow down the search
+        dotnet clean 
+        dotnet clean -c release
+    }
+}
+
+
+function Get-Unique-BaseImages
+{
+    Setup-BaseImage-Script
+
+    Get-ChildItem -Recurse -Filter "Dockerfile" | `
+        Select-String "ARG base_tag=" | `
+        ForEach-Object { $_.Line -replace 'ARG base_tag=', '' } | `
+        Sort-Object -Unique
+}
+
+
+function Update-ARM-BaseImages
+{
+    [CmdletBinding()]
+    param (
+        <# 
+        The new version of ASP .NET Core
+        Ex: The new ASP .NET Core tag is 2.1.23-bionic-arm32, the $NewASPNetCoreVersion = 2.1.23 
+        #>
+        [Parameter(Mandatory)]
+        [string]
+        $NewASPNetCoreVersion
+    )
+
+    Setup-BaseImage-Script
+
+    # Get all the Dockerfile file location
+    $fileLocale = $(Get-ChildItem -Recurse -Filter "Dockerfile" )
+
+    # Replace the underlying ASP .Net Core to the new version
+    # Assuming the ARM64 & ARM32 both use the same *-bionic-arm* ASP .Net Core image tag
+    $baseAspNetLocale = $($($fileLocale | Convert-Path) -like "*\base\*" | Resolve-path)
+    foreach ($file in $baseAspNetLocale)
+    {
+        (Get-Content -Encoding utf8 $file.Path) |
+        Foreach-Object { $_ -replace "ARG base_tag=.*.-bionic-arm", "ARG base_tag=$NewASPNetCoreVersion-bionic-arm" } |
+        Set-Content -Encoding utf8 $file.Path 
+    }
+
+    # Update the places where the base images are used
+    $baseImageLocale = $($fileLocale | Select-String "ARG base_tag=.*.-linux-arm" | Select-Object -Unique Path)
+    foreach ($file in $($baseImageLocale | Resolve-path))
+    {
+        # Increment the last digit by 1
+        $fileContent = (Get-Content $file.Path);
+        $curVersion = $($fileContent -like "ARG base_tag=*" ).split("=")[1].split("-")[0];
+        $splitVersion = $curVersion.split(".", 4);
+        $incrementedSegment = $([int]$splitVersion[3]) + 1;
+        $newBaseImageVersion = $($splitVersion[0,1,2] + $incrementedSegment -join ".");
+
+        # Replace the version
+        $fileContent |
+        Foreach-Object { $_ -replace "ARG base_tag=.*.-linux-arm", "ARG base_tag=$newBaseImageVersion-linux-arm" } |
+        Set-Content $file.Path
+    }
+}
\ No newline at end of file