[1.1] Build rocksdb and arm images in amd64 hosts

builds/misc/images-release.yaml

@@ -2,7 +2,17 @@ trigger: none
 pr: none
 
 name: $(version)
-jobs: 
+
+stages:
+# This stage builds Rocks DB and adds the files to staging directory
+################################################################################
+- template: templates/build-rocksdb.yaml
+################################################################################
+
+- stage: BuildReleaseImages
+ dependsOn: [ BuildRocksDB ]
+
+ jobs: 
 ################################################################################
  - job: linux_dotnet_projects
 ################################################################################
@@ -13,7 +23,7 @@ jobs:
  pool:
  name: $(pool.linux.name)
  demands:
- - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker
+ - ImageOverride -equals agent-aziotedge-ubuntu-20.04-docker 
  steps: 
  # Both docker logins needed for if we need to test this job. In this case images should go to edgebuilds.
  - task: Docker@2
@@ -289,30 +299,51 @@ jobs:
  }
  ]
  signConfigType: inlineSignParams
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact: 'librocksdb'
+ # place in '$(Pipeline.Workspace)/librocksdb'
+ path: '$(Pipeline.Workspace)/librocksdb'
+
+ - task: CopyFiles@2
+ displayName: Copy rocksdb libs
+ inputs:
+ sourceFolder: '$(Pipeline.Workspace)/librocksdb'
+ contents: '**'
+ targetFolder: '$(Build.BinariesDirectory)/publish/librocksdb'
+
  - template: templates/image-linux.yaml
  parameters: 
  imageName: azureiotedge-agent
  name: "Edge Agent"
  project: Microsoft.Azure.Devices.Edge.Agent.Service
  version: $(version)
+ buildx_flag: 'true'
+ use_rocksdb: true
  - template: templates/image-linux.yaml
  parameters: 
  imageName: azureiotedge-hub
  name: "Edge Hub"
  project: Microsoft.Azure.Devices.Edge.Hub.Service
  version: $(version)
+ buildx_flag: 'true'
+ use_rocksdb: true
  - template: templates/image-linux.yaml
  parameters: 
  imageName: azureiotedge-simulated-temperature-sensor
  name: "Temperature Sensor"
  project: SimulatedTemperatureSensor
  version: $(version)
+ buildx_flag: 'true'
  - template: templates/image-linux.yaml
  parameters:
  imageName: azureiotedge-diagnostics
  name: "Diagnostics Module"
  project: IotedgeDiagnosticsDotnet
  version: $(version)
+ buildx_flag: 'true'
+
  - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
  displayName: 'SBOM Generation Task'
  inputs:
@@ -322,6 +353,7 @@ jobs:
  inputs:
  PathtoPublish: '$(Build.BinariesDirectory)/publish'
  ArtifactName: 'publish-linux' 
+
 ################################################################################
  - job: windows
 ################################################################################

builds/misc/templates/build-images.yaml

@@ -26,6 +26,14 @@ stages:
  fi
  displayName: Check changes in runtime sources
  name: check_files
+
+# This stage builds Rocks DB and adds the files to staging directory
+################################################################################
+ - template: build-rocksdb.yaml
+ parameters:
+ dependency: [ CheckBuildImages ]
+################################################################################
+
 ################################################################################
  - stage: BuildExecutables
 ################################################################################
@@ -39,8 +47,8 @@ stages:
  pool:
  name: $(pool.linux.name)
  demands:
- - ImageOverride -equals agent-aziotedge-ubuntu-18.04-docker 
- dependsOn: CheckBuildImages
+ - ImageOverride -equals agent-aziotedge-ubuntu-20.04-docker 
+ dependsOn: BuildRocksDB
  jobs:
  - job: linux_dotnet_projects
  displayName: LinuxDotnet
@@ -57,7 +65,20 @@ stages:
 
  - script: scripts/linux/createArtifactInfo.sh --output-folder '$(Build.BinariesDirectory)/publish' --build-number $(Build.BuildNumber)
  displayName: 'Create Artifact Info File'
-
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact: 'librocksdb'
+ # place in '$(Pipeline.Workspace)/librocksdb'
+ path: '$(Pipeline.Workspace)/librocksdb'
+
+ - task: CopyFiles@2
+ displayName: Copy rocksdb libs
+ inputs:
+ sourceFolder: '$(Pipeline.Workspace)/librocksdb'
+ contents: '**'
+ targetFolder: '$(Build.BinariesDirectory)/publish/librocksdb'
+
  - task: PublishBuildArtifacts@1
  displayName: 'Publish Artifacts'
  inputs:
@@ -70,48 +91,58 @@ stages:
  name: Edge Agent
  imageName: azureiotedge-agent
  project: Microsoft.Azure.Devices.Edge.Agent.Service
+ buildx_flag: 'true'
+ use_rocksdb: true
 
  # Edge Hub
  - template: image-linux.yaml
  parameters:
  name: Edge Hub
  imageName: azureiotedge-hub
  project: Microsoft.Azure.Devices.Edge.Hub.Service
+ buildx_flag: 'true'
+ use_rocksdb: true
 
  # Simulated Temperature Sensor
  - template: image-linux.yaml
  parameters:
  name: Temperature Sensor
  imageName: azureiotedge-simulated-temperature-sensor
  project: SimulatedTemperatureSensor
+ buildx_flag: 'true'
 
  # Dotnet Diagnostics Module
  - template: image-linux.yaml
  parameters:
  name: Diagnostics
  imageName: azureiotedge-diagnostics
  project: IotedgeDiagnosticsDotnet
+ buildx_flag: 'true'
 
  # Temperature Filter
  - template: image-linux.yaml
  parameters:
  name: Temperature Filter
  imageName: azureiotedge-temperature-filter
  project: TemperatureFilter
+ buildx_flag: 'true'
 
  # Load Gen
  - template: image-linux.yaml
  parameters:
  name: Load Gen
  imageName: azureiotedge-load-gen
  project: load-gen
+ buildx_flag: 'true'
 
  # Test Analyzer
  - template: image-linux.yaml
  parameters:
  name: Test Analyzer
  imageName: azureiotedge-analyzer
  project: TestAnalyzer
+ buildx_flag: 'true'
+ use_rocksdb: true
 
  # Functions Sample - Not supported on linux arm64
  - template: image-linux.yaml
@@ -126,97 +157,114 @@ stages:
  name: Direct Method Sender
  imageName: azureiotedge-direct-method-sender
  project: DirectMethodSender
+ buildx_flag: 'true'
 
  # Direct Method Receiver
  - template: image-linux.yaml
  parameters:
  name: Direct Method Receiver
  imageName: azureiotedge-direct-method-receiver
  project: DirectMethodReceiver
+ buildx_flag: 'true'
 
  # Metrics Validator
  - template: image-linux.yaml
  parameters:
  name: Metrics Validator
  imageName: azureiotedge-metrics-validator
  project: MetricsValidator
+ buildx_flag: 'true'
 
  # Number Logger
  - template: image-linux.yaml
  parameters:
  name: Number Logger
  imageName: azureiotedge-number-logger
  project: NumberLogger
+ buildx_flag: 'true'
 
  # Module Restarter
  - template: image-linux.yaml
  parameters:
  name: Module Restarter
  imageName: azureiotedge-module-restarter
  project: ModuleRestarter
+ buildx_flag: 'true'
 
  # Twin Tester
  - template: image-linux.yaml
  parameters:
  name: TwinTester
  imageName: azureiotedge-twin-tester
  project: TwinTester
+ buildx_flag: 'true'
+ use_rocksdb: true
 
  # Relayer
  - template: image-linux.yaml
  parameters:
  name: Relayer
  imageName: azureiotedge-relayer
  project: Relayer
+ buildx_flag: 'true'
 
  # Network Controller
  - template: image-linux.yaml
  parameters:
  name: Network Controller
  imageName: azureiotedge-network-controller
  project: NetworkController
-
+ buildx_flag: 'true'
+ use_rocksdb: true
+
  # TestResultCoordinator
  - template: image-linux.yaml
  parameters:
  name: TestResultCoordinator
  imageName: azureiotedge-test-result-coordinator
  project: TestResultCoordinator
+ buildx_flag: 'true'
+ use_rocksdb: true
 
  # Metrics Collector (test-only)
  - template: image-linux.yaml
  parameters:
  name: Metrics Collector (test-only)
  imageName: azureiotedge-test-metrics-collector
  project: MetricsCollector
+ buildx_flag: 'true'
 
  # Deployment Tester
  - template: image-linux.yaml
  parameters:
  name: DeploymentTester
  imageName: azureiotedge-deployment-tester
  project: DeploymentTester
+ buildx_flag: 'true'
 
  # EdgeHub Restart Tester
  - template: image-linux.yaml
  parameters:
  name: EdgeHubRestartTester
  imageName: azureiotedge-edgehub-restart-tester
  project: EdgeHubRestartTester
+ buildx_flag: 'true'
 
  # Cloud To Device Message Tester
  - template: image-linux.yaml
  parameters:
  name: Cloud To Device Message Tester
  imageName: azureiotedge-c2dmessage-tester
  project: CloudToDeviceMessageTester
+ buildx_flag: 'true'
 
  # Metrics Collector (customer-facing)
  - template: image-linux.yaml
  parameters:
  name: Metrics Collector (customer-facing)
  imageName: azureiotedge-metrics-collector
  project: Microsoft.Azure.Devices.Edge.Azure.Monitor
+ buildx_flag: 'true'
 
 ################################################################################
  # linux - Rust amd64
@@ -477,4 +525,4 @@ stages:
  - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(Build.BuildNumber) -t $(System.DefaultWorkingDirectory)/edgelet/iotedge-proxy/docker/manifest.yaml.template -n microsoft --tags "$(tags)"
  displayName: 'Publish azureiotedge-proxy Manifest'
  - script: scripts/linux/buildManifest.sh -r $(registry.address) -v $(Build.BuildNumber) -t $(System.DefaultWorkingDirectory)/edge-modules/azure-monitor/docker/manifest.yaml.template -n microsoft --tags "$(tags)"
- displayName: 'Publish azureiotedge-metrics-collector Manifest'
+ displayName: 'Publish azureiotedge-metrics-collector Manifest'

builds/misc/templates/build-rocksdb.yaml

@@ -0,0 +1,40 @@
+parameters:
+ dependency: []
+stages: 
+################################################################################
+ - stage: BuildRocksDB
+################################################################################
+ displayName: Build RocksDB
+ pool:
+ name: $(pool.linux.name)
+ demands:
+ - ImageOverride -equals agent-aziotedge-ubuntu-20.04-docker
+ dependsOn: ${{parameters.dependency}}
+ jobs: 
+ - job: BuildRocksDB
+ displayName: Build Rocks DB
+ strategy:
+ matrix:
+ RocksDbAmd64:
+ arch: amd64
+ postfix: amd64
+ RocksDbArm32:
+ arch: arm32v7
+ postfix: armhf
+ RocksDbArm64:
+ arch: arm64v8
+ postfix: arm64
+ steps:
+ - bash: |
+ build_image=rocksdb-build:master-$(postfix)-$(Build.BuildNumber) && \
+ mkdir -p $(Build.ArtifactStagingDirectory)/librocksdb && \
+ cd $(System.DefaultWorkingDirectory)/edge-util/docker/linux/$(arch) && \
+ docker build --tag ${build_image} . && \
+ docker run --rm -v $(Build.ArtifactStagingDirectory)/librocksdb:/artifacts \
+ ${build_image} cp /publish/librocksdb.so.$(postfix) /artifacts
+ displayName: Build and copy out rocksdb lib
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifacts to VSTS'
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/librocksdb'
+ artifactName: 'librocksdb'

builds/misc/templates/image-linux.yaml

@@ -4,8 +4,20 @@ parameters:
  namespace: 'microsoft'
  project: ''
  version: ''
+ buildx_flag: ''
+ use_rocksdb: false
+ bin_dir: ''
 
 steps:
+
+ - task: CopyFiles@2
+ displayName: Copy RocksDB libs into project
+ condition: and(succeeded(), eq('${{ parameters.use_rocksdb }}', 'true'))
+ inputs:
+ sourceFolder: '$(Build.BinariesDirectory)/publish/librocksdb'
+ contents: '**'
+ targetFolder: '$(Build.BinariesDirectory)/publish/${{ parameters.project }}/librocksdb' 
+
  - task: Bash@3
  displayName: Build Image - ${{ parameters.name }} - amd64
  inputs:
@@ -15,10 +27,10 @@ steps:
  displayName: Build Image - ${{ parameters.name }} - arm32
  inputs:
  filePath: scripts/linux/buildImage.sh
- arguments: -r "$(registry.address)" -i "${{ parameters.imageName }}" -n "${{ parameters.namespace }}" -P "${{ parameters.project }}" -v "${{ parameters.version }}" --target-arch armv7l
+ arguments: -r "$(registry.address)" -i "${{ parameters.imageName }}" -n "${{ parameters.namespace }}" -P "${{ parameters.project }}" -v "${{ parameters.version }}" --target-arch armv7l --buildx_flag ${{ parameters.buildx_flag }}
  - task: Bash@3
  displayName: Build Image - ${{ parameters.name }} - arm64 
  condition: and(ne('${{ parameters.name }}', 'Functions Sample'), succeeded())
  inputs:
  filePath: scripts/linux/buildImage.sh
- arguments: -r "$(registry.address)" -i "${{ parameters.imageName }}" -n "${{ parameters.namespace }}" -P "${{ parameters.project }}" -v "${{ parameters.version }}" --target-arch aarch64
+ arguments: -r "$(registry.address)" -i "${{ parameters.imageName }}" -n "${{ parameters.namespace }}" -P "${{ parameters.project }}" -v "${{ parameters.version }}" --target-arch aarch64 --buildx_flag ${{ parameters.buildx_flag }}