Push images to ACR with v2 REST APIs

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bullseye
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-bookworm
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -24,10 +24,6 @@ RUN sudo apt-get update \
     # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed
     && sudo apt-get -y install git iproute2 procps lsb-release figlet build-essential dnsutils openvpn
 
-RUN sudo mkdir -p /dev/net \
-    && sudo mknod /dev/net/tun c 10 200 \
-    && sudo chmod 600 /dev/net/tun
-
 # Save command line history
 RUN echo "export HISTFILE=/home/$USERNAME/commandhistory/.bash_history" >> "/home/$USERNAME/.bashrc" \
     && echo "export PROMPT_COMMAND='history -a'" >> "/home/$USERNAME/.bashrc" \
@@ -38,11 +34,11 @@ RUN echo "export HISTFILE=/home/$USERNAME/commandhistory/.bash_history" >> "/hom
 # docker-client
 RUN sudo apt-get update \
     # libgl dependencies and install Docker CE CLI
-    && sudo apt-get install ffmpeg libsm6 libxext6 apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release -y \
-    && curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | sudo apt-key add - 2>/dev/null \
-    && sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" \
-    && sudo apt-get update \
-    && sudo apt-get install -y docker-ce-cli 
+    && sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release -y \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && sudo apt-get update -y \
+    && sudo apt-get install -y docker-ce-cli
 
 ARG DOCKER_GROUP_ID
 COPY ./scripts/docker-client.sh /tmp/
@@ -79,12 +75,6 @@ RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian
     && apt-get update \
     && apt-get install -y azure-functions-core-tools-4
 
-# Install Powershell
-RUN apt-get update \
-    && apt-get install -y powershell \
-    # Alias Powershell
-    && echo "alias powershell=pwsh" >> "/home/$USERNAME/.bashrc"
-
 #Install npm and nodejs in the Docker dev container
 COPY ./scripts/nodejs.sh /tmp/
 RUN /tmp/nodejs.sh
@@ -104,5 +94,13 @@ RUN pip install azure-cosmos
 RUN pip install pyfiglet
 RUN pip install azure-identity
 RUN pip install azure-keyvault-secrets
+
 # Python Package for Backend Testing
-RUN pip install pytest
+RUN pip install pytest
+
+# Install skopeo - This must be at the end due to pulling from trixie
+RUN sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes install usrmerge
+
+RUN echo 'deb http://ftp.debian.org/debian trixie main' | sudo tee /etc/apt/sources.list.d/debian.trixie.list \
+    && sudo apt-get update \
+    && sudo apt-get install skopeo

Makefile

@@ -9,7 +9,7 @@ help: ## Show this help
 		| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%s\033[0m|%s\n", $$1, $$2}' \
         | column -t -s '|'
 
-deploy: build build-containers infrastructure extract-env deploy-enrichments deploy-search-indexes deploy-webapp deploy-functions ## Deploy infrastructure and application code
+deploy: build infrastructure extract-env deploy-search-indexes deploy-functions deploy-webapp deploy-enrichments ## Deploy infrastructure and application code
 
 build-deploy-webapp: build extract-env deploy-webapp ##Build and Deploy the Webapp
 build-deploy-enrichments: build extract-env deploy-enrichments ##Build and Deploy the Enrichment Webapp
@@ -18,11 +18,6 @@ build-deploy-functions: build extract-env deploy-functions ##Build and Deploy th
 build: ## Build application code
 	@./scripts/build.sh
 
-build-containers: ## Build the docker containers for the function, webapp, and enrichment
-	@./container_images/enrichment_container_image/docker-build.sh
-	@./functions/docker-build.sh
-	@./container_images/webapp_container_image/docker-build.sh
-
 infrastructure: check-subscription ## Deploy infrastructure
 	@./scripts/inf-create.sh
 

README.md

@@ -16,6 +16,7 @@
   - [Deploying to Azure](./docs/deployment/deployment.md#deploy-and-configure-azure-resources)
   - [Troubleshooting Common Issues](./docs/deployment/troubleshooting.md)
   - [Considerations for Production Adoption](./docs/deployment/considerations_production.md)
+- [Secure-Mode Deployment](./docs/deployment/secure_deployment.md)
 - [Enabling optional features](./docs/features/optional_features.md)
 - [Using the app](/docs/deployment/using_ia_first_time.md)
 - [Responsible AI](#responsible-ai)

app/enrichment/requirements.txt

@@ -1,5 +1,8 @@
 #### Any version change made here should also be made and tested for the webapp backend and function apps in /functions and /app/backend
-
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+torchvision
+torchaudio
 sentence-transformers == 2.2.2
 fastapi == 0.109.1
 fastapi-utils == 0.2.1

container_images/enrichment_container_image/Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /app
 COPY app/enrichment /app
 
 # Install any needed packages specified in requirements.txt
-RUN pip install -r /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
 
 # Expose the port the app runs on
 EXPOSE 6000

container_images/enrichment_container_image/docker-build.sh

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 #!/bin/bash
 set -eo pipefail
 
@@ -16,6 +19,14 @@ sudo docker tag enrichmentapp enrichmentapp:$tag
 
 # Output the tag to a file to be used in deployment
 echo -n "$tag" > ./container_images/enrichment_container_image/image_tag.txt
+echo -e "\n"
+
+# Export docker image to the artifacts folder
+echo "Exporting docker image to artifacts folder"
+echo -e "\n"
+rm -rf ./artifacts/enrichmentapp
+mkdir -p ./artifacts/enrichmentapp
+skopeo copy docker-daemon:enrichmentapp:$tag oci:./artifacts/enrichmentapp
 
 echo "Build and tagging complete. Tag: $tag"
 echo -e "\n"

container_images/webapp_container_image/docker-build.sh

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 #!/bin/bash
 set -eo pipefail
 
@@ -13,6 +16,14 @@ sudo docker tag webapp webapp:$tag
 
 # Output the tag to a file to be used in deployment
 echo -n "$tag" > ./container_images/webapp_container_image/image_tag.txt
+echo -e "\n"
+
+# Export docker image to the artifacts folder
+echo "Exporting docker image to artifacts folder"
+echo -e "\n"
+rm -rf ./artifacts/webapp
+mkdir -p ./artifacts/webapp
+skopeo copy docker-daemon:webapp:$tag oci:./artifacts/webapp
 
 echo "Build and tagging complete. Tag: $tag"
 echo -e "\n"

docs/costestimator.md

@@ -17,6 +17,7 @@ The following list of Azure Services will be deployed for IA Accelerator, versio
 - Azure AI Services [:link:](https://azure.microsoft.com/en-ca/pricing/details/cognitive-services/)
 - Azure Cosmos DB [:link:](https://azure.microsoft.com/en-ca/pricing/details/cosmos-db/autoscale-provisioned/)
 - Bing Search Service [:link:](https://www.microsoft.com/en-us/bing/apis/llm-pricing)
+- Container Registry [:link:](https://azure.microsoft.com/en-gb/pricing/details/container-registry)
 - Azure AI Document Intelligence [:link:](https://azure.microsoft.com/en-ca/pricing/details/form-recognizer/#pricing)
 - Azure Function(App Service Plan) [:link:](https://azure.microsoft.com/en-ca/pricing/details/functions/#pricing)
 - Azure Key Vault [:link:](https://azure.microsoft.com/en-us/pricing/details/key-vault/)
@@ -26,7 +27,6 @@ The following list of Azure Services will be deployed for IA Accelerator, versio
 - Azure OpenAI [:link:](https://azure.microsoft.com/en-ca/pricing/details/cognitive-services/openai-service/)
 - Storage account  [:link:](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview)
 - Azure Active Directory [:link:](https://www.microsoft.com/en-sg/security/business/microsoft-entra-pricing?rtc=1)
-- Azure AI Video Indexer [:link:](https://azure.microsoft.com/en-us/pricing/details/video-indexer/)
 
 ---
 **NOTE:**

docs/deployment/deployment.md

@@ -61,7 +61,6 @@ ENABLE_MATH_ASSISTANT | Yes | Defaults to `true`. This feature flag will enable
 ENABLE_TABULAR_DATA_ASSISTANT | Yes | Defaults to `true`. This feature flag will enable the Tabular Data Assistant tab in the Information Assistant website. Read more information about the [Tabular Data Assistant](/docs/features/features.md)
 ENABLE_SHAREPOINT_CONNECTOR | Yes | Defaults to `false`. This feature flag enabled the ability to ingest data from SharePoint document stores into the Information Assistant. When enabled, be sure to set the `SHAREPOINT_TO_SYNC` parameter for your SharePoint sites. Read more about configuring the [SharePoint Connector](/docs/features/sharepoint.md). This feature flag is **NOT** compatible with `AZURE_ENVIRONMENT=AzureUSGovernment`.
 SHAREPOINT_TO_SYNC | No | This is a JSON Array of Objects for SharePoint Sites and their entry folders. The app will crawl down from the folder specified for each site. Specifying "/Shared Documents" will crawl all the documents in your SharePoint. `[{"url": "https://SharePoint.com/", "folder": "/Shared Documents"}]` This will **overwrite** any prior changes you've made to config.json. Information on setting up SharePoint Ingestion can be found here [SharePoint Connector](/docs/features/sharepoint.md)
-ENABLE_MULTIMEDIA | Yes | Defaults to `false`. This feature flag should not be changed at this time. The multimedia feature is still in development. Enabling this feature will deploy an Azure Video Indexer instance in your resource group only.
 REQUIRE_WEBSITE_SECURITY_MEMBERSHIP | Yes | Use this setting to determine whether a user needs to be granted explicit access to the website via an Azure AD Enterprise Application membership (true) or allow the website to be available to anyone in the Azure tenant (false). Defaults to false. If set to true, A tenant level administrator will be required to grant the implicit grant workflow for the Azure AD App Registration manually.
 SECRET_EXPIRATION_DAYS | Yes | Defaults to `120`. Use this setting to set the secret expiration to the current day plus the number of days specified. Key Vault secrets require an expiration date to be compatible with Microsoft's recommended guardrails for Azure Key Vault policy. We have NOT included automatic secret rotation in this deployment. Go [here](https://learn.microsoft.com/en-us/azure/key-vault/keys/how-to-configure-key-rotation) for more information on enabling cryptographic key auto-rotation.
 SKIP_PLAN_CHECK | No | If this value is set to 1, then the Terraform deployment will not stop to allow you to review the planned changes. The default value is 0 in the scripts, which will allow the deployment to stop and confirm you accept the proposed changes before continuing.

docs/deployment/developing_in_a_GitHub_Codespaces.md

@@ -20,6 +20,8 @@ A codespace is a development environment that's hosted in the cloud. You can cus
 1. First you will need to select your repository you forked from the Information Assistant repo
 1. Next, by default you will be on **main** branch, you can switch to specific branch where you would like to create a Codespaces.
 1. Then select options for **Region** and **Machine type**. The "Dev Container configuration" will be pre-populated and does not need to be changed.
+    * The **Machine type** will need to be at least **4-core**.
+![GitHub Codespace machine type: 4-core](image.png).
 1. Next, click on  **Create codespace**
 ![GitHub Codespaces creation](/docs/images/codespace_creation.png)
 

docs/deployment/secure_deployment.md

@@ -13,6 +13,7 @@
 >Secure mode requires a DDOS Protection Plan for Virtual Network Protection. There is a limit of 1 DDOS protection plan for a subscription in a region. You can reuse an existing DDOS plan in your tenant or Info Assistant can deploy one for you.
 >
 
+* [Additional Azure account requirements](#additional-azure-account-requirements)
 * [Overview](#overview)
 * [Architecture](#architecture)
   * [High Level Architecture](#high-level-architecture)
@@ -26,6 +27,14 @@
   * [Secure Communication to Azure](#secure-communication-to-azure)
   * [Secure Communication with Microsoft Cloud for Sovereignty (MCfSov)](#secure-communication-with-microsoft-cloud-for-sovereignty-mcfsov)
 
+## Additional Azure account requirements
+
+In order to deploy the "secure mode" of Information Assistant, you will need the following in addition to the standard [Azure account requirements](/README.md#azure-account-requirements):
+
+* Your GitHub Codespace machine type must be at least **4-core**.
+![alt text](image.png).
+* If you are going to use an existing DDOS that resides in another subscription, you will need to have `TBD` permission on the subscription where the DDOS Protection Plan exists to allow associating to the vnet when it is created.
+
 ## Overview
 
 Information Assistant secure mode is essential when heightened levels of security are necessary. Secure mode is recommended for all production systems. Key features of secure mode include:

functions/docker-build.sh

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 #!/bin/bash
 set -eo pipefail
 
@@ -16,6 +19,14 @@ sudo docker tag functionapp functionapp:$tag
 
 # Output the tag to a file to be used in deployment
 echo -n "$tag" > ./functions/image_tag.txt
+echo -e "\n"
+
+# Export docker image to the artifacts folder
+echo "Exporting docker image to artifacts folder"
+echo -e "\n"
+#rm -rf ./artifacts/functionapp
+mkdir -p ./artifacts/functionapp
+skopeo copy docker-daemon:functionapp:$tag oci:./artifacts/functionapp
 
 echo "Build and tagging complete. Tag: $tag"
 echo -e "\n"

infra/core/host/enrichmentapp/enrichmentapp.tf

@@ -1,7 +1,3 @@
-data "local_file" "image_tag" {
-  filename = "../container_images/enrichment_container_image/image_tag.txt"
-}
-
 // Create Enrichment App Service Plan 
 resource "azurerm_service_plan" "appServicePlan" {
   name                          = var.plan_name
@@ -96,7 +92,7 @@ resource "azurerm_linux_web_app" "enrichmentapp" {
     container_registry_use_managed_identity       = true
 
     application_stack {
-      docker_image_name         = "${var.container_registry}/enrichmentapp:${data.local_file.image_tag.content}"
+      docker_image_name         = "${var.container_registry}/enrichmentapp:latest"
       docker_registry_url       = "https://${var.container_registry}"
       docker_registry_username  = var.container_registry_admin_username
       docker_registry_password  = var.container_registry_admin_password

infra/core/host/functions/functions.tf

@@ -1,24 +1,3 @@
-data "local_file" "image_tag" {
-  filename = "../functions/image_tag.txt"
-}
-
-locals {
-  stripped_container_registry = replace(var.container_registry, "https://", "")
-}
-
-#resource "null_resource" "docker_push" {
-#  provisioner "local-exec" {
-#    command = <<-EOT
-#        printf "%s" ${var.container_registry_admin_password} | docker login --username ${var.container_registry_admin_username} --password-stdin ${var.container_registry}
-#        docker tag functions ${local.stripped_container_registry}/functions:${data.local_file.image_tag.content}
-#        docker push ${local.stripped_container_registry}/functions:${data.local_file.image_tag.content}
-#      EOT
-#  }
-#  triggers = {
-#    always_run = timestamp()
-#  }
-#}
-
 # Terraform resource file to create a service plan for the function app
 resource "azurerm_service_plan" "funcServicePlan" {
   name                = var.plan_name
@@ -120,7 +99,7 @@ resource "azurerm_linux_function_app" "function_app" {
     application_stack {
       docker {
         image_name        = "${var.container_registry}/functionapp"
-        image_tag         = data.local_file.image_tag.content
+        image_tag         = "latest"
         registry_url      = "https://${var.container_registry}"
         registry_username = var.container_registry_admin_username
         registry_password = var.container_registry_admin_password

infra/core/host/webapp/webapp.tf

@@ -1,24 +1,3 @@
-data "local_file" "image_tag" {
-  filename = "../container_images/webapp_container_image/image_tag.txt"
-}
-
-locals {
-  stripped_container_registry = replace(var.container_registry, "https://", "")
-}
-
-#resource "null_resource" "docker_push" {
-#  provisioner "local-exec" {
-#    command = <<-EOT
-#        printf "%s" ${var.container_registry_admin_password} | docker login --username ${var.container_registry_admin_username} --password-stdin ${var.container_registry}
-#        docker tag webapp ${local.stripped_container_registry}/webapp:${data.local_file.image_tag.content}
-#        docker push ${local.stripped_container_registry}/webapp:${data.local_file.image_tag.content}
-#      EOT
-#  }
-#  triggers = {
-#    always_run = timestamp()
-#  }
-#}
-
 # Create the web app service plan
 resource "azurerm_service_plan" "appServicePlan" {
   name                = var.plan_name
@@ -108,7 +87,7 @@ resource "azurerm_linux_web_app" "app_service" {
 
   site_config {
     application_stack {
-      docker_image_name         = "${var.container_registry}/webapp:${data.local_file.image_tag.content}"
+      docker_image_name         = "${var.container_registry}/webapp:latest"
       docker_registry_url       = "https://${var.container_registry}"
       docker_registry_username  = var.container_registry_admin_username
       docker_registry_password  = var.container_registry_admin_password

infra/variables.tf

@@ -77,7 +77,6 @@ variable "enableSharePointConnector" {
 variable "enableMultimedia" {
   type    = bool
   default = false
-
 }
 ////
 

pipelines/templates/deploy-template.yml

@@ -50,11 +50,6 @@ steps:
     displayName: 'Build'
     makeCommand: 'build'
 
-- template: make-command.yml
-  parameters:
-    displayName: 'build-containers'
-    makeCommand: 'build-containers'
-
 - ${{ if eq(parameters.fullBuild, true) }}:
   - template: make-command.yml
     parameters:
@@ -66,23 +61,22 @@ steps:
       displayName: 'Extract environment variables'
       makeCommand: 'extract-env'
 
-  - template: make-command.yml
-    parameters:
-      displayName: 'Deploy Enrichments'
-      makeCommand: 'deploy-enrichments'
-
   - template: make-command.yml
     parameters:
       displayName: 'Deploy search indexes'
       makeCommand: 'deploy-search-indexes'
 
+  - template: make-command.yml
+    parameters:
+      displayName: 'Deploy Azure Functions'
+      makeCommand: 'deploy-functions'   
+
   - template: make-command.yml
     parameters:
       displayName: 'Deploy Web App'
       makeCommand: 'deploy-webapp'
 
   - template: make-command.yml
     parameters:
-      displayName: 'Deploy Azure Functions'
-      makeCommand: 'deploy-functions'   
-
+      displayName: 'Deploy Enrichments'
+      makeCommand: 'deploy-enrichments'