Sync recent changes to windows-new-dir-hierarchy (#8680)

* [MacOS] Add xcode 15.0.1 (#8604)

* Updating readme file for macos-13 version 20231018.14 (#8601)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for macos-13-arm64 version 20231017.1 (#8586)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* [MacOS] improve errexit option handling (#8432)

* [MacOS] improve errexit option handling

* Prevent prompting for override when unzipping

* Explicitly allow failing when installing virtualbox

* Remove libtcl symlinks

* Add debug output

* Updating readme file for ubuntu20 version 20231022.1.1 (#8637)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* Updating readme file for ubuntu22 version 20231022.1.1 (#8636)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* Updating readme file for win19 version 20231023.1.1 (#8644)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for win22 version 20231023.1.1 (#8643)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* [macos] simplify "brew_smart_install" helper (#8639)

* [macos] homebrew: use hardcoded condition for jq installation

we cannot use "jq" if we are asked to install "jq"

* [macos] always use "brew install"

* [macos] add retries to "get_github_package_download_url" helper

* [macos] add retries to chrome install script

* [macos] add retries to OpenJDK install script

* [macos] add retries to miniconda installer

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* fix copy-paste error

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

---------

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* Updating readme file for ubuntu22 version 20231025.1.1 (#8663)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for ubuntu20 version 20231025.1.1 (#8664)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for win22 version 20231024.1.1 (#8668)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Windows Server 2019 (20231025) Image Update (#8667)

* Updating readme file for win19 version 20231025.2.1

* Update Windows2019-Readme.md

---------

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Mikhail Koliada <88318005+mikhailkoliada@users.noreply.github.com>
Co-authored-by: Actions service account <no-reply@github.com>

* [Mac OS] Fix logic for retrying AppleScript execution (#8662)

* [Ubuntu] Add PGP validation for swift (#8652)

* [Windows] Remove out-of-support vs components (#8678)

* [MacOS] Unpin Virtualbox (#8653)

* [MacOS] Unpin Virtualbox

* Remove obsolete virtualbox kext pester test

---------

Co-authored-by: sergei-pyshnoi <121864472+sergei-pyshnoi@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>
Co-authored-by: ilia-shipitsin <125650415+ilia-shipitsin@users.noreply.github.com>
Co-authored-by: Mikhail Koliada <88318005+mikhailkoliada@users.noreply.github.com>

images/linux/scripts/installers/swift.sh

@@ -15,6 +15,19 @@ swift_tar_name="swift-$swift_version-RELEASE-ubuntu$image_label.tar.gz"
 swift_tar_url="https://swift.org/builds/swift-$swift_version-release/ubuntu${image_label//./}/swift-$swift_version-RELEASE/$swift_tar_name"
 download_with_retries $swift_tar_url "/tmp" "$swift_tar_name"
 
+# Verifing pgp signature using official swift pgp key. Referring to https://www.swift.org/install/linux/#Installation-via-Tarball
+# Download swift pgp key
+download_with_retries "https://swift.org/keys/all-keys.asc" "/tmp" "all-keys.asc"
+# Import swift pgp key
+gpg --no-default-keyring --keyring swift --import /tmp/all-keys.asc
+# Download signature file
+download_with_retries "$swift_tar_url.sig" "/tmp" "$swift_tar_name.sig"
+# Verify signature
+gpg --no-default-keyring --keyring swift --verify "/tmp/$swift_tar_name.sig" "/tmp/$swift_tar_name"
+# Remove swift pgp public key with temporary keyring
+rm ~/.gnupg/swift
+
+
 tar xzf /tmp/$swift_tar_name
 
 SWIFT_INSTALL_ROOT="/usr/share/swift"

images/macos/provision/core/android-toolsets.sh

@@ -126,7 +126,7 @@ sdkTools="android-sdk-tools.zip"
 sdkToolsVersion=$(get_toolset_value '.android."sdk-tools"')
 if [ "$sdkToolsVersion" != "null" ]; then
  download_with_retries "https://dl.google.com/android/repository/${sdkToolsVersion}" "." $sdkTools
- unzip -qq $sdkTools -d ${ANDROID_SDK_ROOT}
+ unzip -o -qq $sdkTools -d ${ANDROID_SDK_ROOT}
  rm -f $sdkTools
 fi
 

images/macos/provision/core/chrome.sh

@@ -14,7 +14,8 @@ echo "Google Chrome version is $FULL_CHROME_VERSION"
 # Get Google Chrome versions information
 CHROME_PLATFORM="mac-$arch"
 CHROME_VERSIONS_URL="https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json"
-CHROME_VERSIONS_JSON=$(curl -fsSL "${CHROME_VERSIONS_URL}")
+download_with_retries "$CHROME_VERSIONS_URL" "/tmp" "latest-patch-versions-per-build-with-downloads.json"
+CHROME_VERSIONS_JSON=$(cat /tmp/latest-patch-versions-per-build-with-downloads.json)
 
 # Download and unpack the latest release of Chrome Driver
 CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version')

images/macos/provision/core/commonutils.sh

@@ -13,18 +13,7 @@ done
 cask_packages=$(get_toolset_value '.brew.cask_packages[]')
 for package in $cask_packages; do
  echo "Installing $package..."
- if [[ $package == "virtualbox" ]]; then
- if ! is_Ventura || ! is_VenturaArm64; then
- # VirtualBox 7 crashes
- # macOS host: Dropped all kernel extensions. VirtualBox relies fully on the hypervisor and vmnet frameworks provided by Apple now.
- vbcask_url="https://raw.githubusercontent.com/Homebrew/homebrew-cask/aa3c55951fc9d687acce43e5c0338f42c1ddff7b/Casks/virtualbox.rb"
- download_with_retries $vbcask_url
- brew install ./virtualbox.rb
- rm ./virtualbox.rb
- fi
- else
- brew install --cask $package
- fi
+ brew install --cask $package
 done
 
 # Load "Parallels International GmbH"
@@ -36,18 +25,24 @@ fi
 # System Preferences -> Security & Privacy -> General -> Unlock -> Allow -> Not now
 if is_Monterey; then
  if is_Veertu; then
- retry=5
- while [ $retry -gt 0 ]; do
+ for retry in {4..0}; do
+  echo "Executing AppleScript to change security preferences. Retries left: $retry"
  {
+ set -e
  osascript -e 'tell application "System Events" to get application processes where visible is true'
- }
- osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+ osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+ } && break
+
+ if [ "$retry" -eq 0 ]; then
+ echo "Executing AppleScript failed. No retries left"
+ exit 1
+ fi
 
- retry=$((retry-1))
- echo "retries left "$retry
+ echo "Executing AppleScript failed. Sleeping for 10 seconds and retrying"
  sleep 10
  done
  else
+ echo "Executing AppleScript to change security preferences"
  osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
  fi
 fi
@@ -82,13 +77,5 @@ bazel
 # Install Azure DevOps extension for Azure Command Line Interface
 az extension add -n azure-devops
 
-# Workaround https://github.com/actions/runner-images/issues/4931
-# by making Tcl/Tk paths the same on macOS 10.15 and macOS 11
-if is_Monterey; then
- version=$(brew info tcl-tk --json | jq -r '.[].installed[].version')
- ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtcl8.6.dylib /usr/local/lib/libtcl8.6.dylib
- ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtk8.6.dylib /usr/local/lib/libtk8.6.dylib
-fi
-
 # Invoke tests for all basic tools
 invoke_tests "BasicTools"

images/macos/provision/core/miniconda.sh

@@ -1,9 +1,11 @@
 #!/bin/bash -e -o pipefail
 
-MINICONDA_INSTALLER="/tmp/miniconda.sh"
-curl -fsSL https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $MINICONDA_INSTALLER
-chmod +x $MINICONDA_INSTALLER
-sudo $MINICONDA_INSTALLER -b -p /usr/local/miniconda
+source ~/utils/utils.sh
+
+download_with_retries "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" "/tmp" "miniconda.sh"
+
+chmod +x /tmp/miniconda.sh
+sudo /tmp/miniconda.sh -b -p /usr/local/miniconda
 
 # Chmod with full permissions recursively to avoid permissions restrictions
 sudo chmod -R 777 /usr/local/miniconda

images/macos/provision/core/openjdk.sh

@@ -29,12 +29,12 @@ installOpenJDK() {
  local JAVA_VERSION=$1
 
  # Get link for Java binaries and Java version
- assetUrl=$(curl -fsSL "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot")
+ download_with_retries "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot" "/tmp" "openjdk-hotspot.json"
 
  if [[ $arch == "arm64" ]]; then
- asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")')
+ asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")' /tmp/openjdk-hotspot.json)
  else
- asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")')
+ asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")' /tmp/openjdk-hotspot.json)
  fi
 
  archivePath=$(echo ${asset} | jq -r '.binary.package.link')

images/macos/provision/core/pypy.sh

@@ -11,7 +11,7 @@ function InstallPyPy
  PACKAGE_URL=$1
 
  PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}')
- echo "Downloading tar archive '$PACKAGE_TAR_NAME' - '$PACKAGE_URL'"
+ echo "Downloading tar archive '$PACKAGE_TAR_NAME'"
  PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME"
  download_with_retries $PACKAGE_URL "/tmp" "$PACKAGE_TAR_NAME"
 
@@ -89,4 +89,4 @@ for toolsetVersion in $toolsetVersions; do
  exit 1
  fi
  InstallPyPy $latestMajorPyPyVersion
-done
+done

images/macos/provision/utils/utils.sh

@@ -15,24 +15,31 @@ download_with_retries() {
  local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'"
  fi
 
+ # Save current errexit state and disable it to prevent unexpected exit on error
+ if echo $SHELLOPTS | grep '\(^\|:\)errexit\(:\|$\)' > /dev/null;
+ then
+ local ERR_EXIT_ENABLED=true
+ else
+ local ERR_EXIT_ENABLED=false
+ fi
+ set +e
+
  echo "Downloading '$URL' to '${DEST}/${NAME}'..."
  retries=20
  interval=30
  while [ $retries -gt 0 ]; do
  ((retries--))
- # Temporary disable exit on error to retry on non-zero exit code
- set +e
+ test "$ERR_EXIT_ENABLED" = true && set +e
  http_code=$(eval $COMMAND)
  exit_code=$?
+ test "$ERR_EXIT_ENABLED" = true && set -e
  if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then
  echo "Download completed"
  return 0
  else
  echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left"
  sleep 30
  fi
- # Enable exit on error back
- set -e
  done
 
  echo "Could not download $URL"
@@ -120,98 +127,50 @@ get_brew_os_keyword() {
  fi
 }
 
-should_build_from_source() {
- local tool_name=$1
- local os_name=$2
- # If one of the parsers aborts with an error, 
- # we will get an empty variable notification in the logs
- set -u
-
- # Geting tool info from brew to find available install methods except build from source
- local tool_info=$(brew info --json=v1 $tool_name)
-
- # No need to build from source if a bottle is disabled
- local bottle_disabled=$(echo -E $tool_info | jq ".[0].bottle_disabled")
- if [[ $bottle_disabled == "true" ]]; then
- echo "false"
- return
- fi
-
- # No need to build from source if a universal bottle is available 
- local all_bottle=$(echo -E $tool_info | jq ".[0].bottle.stable.files.all")
- if [[ "$all_bottle" != "null" ]]; then
- echo "false"
- return
- fi
-
- # No need to build from source if a bottle for current OS is available
- local os_bottle=$(echo -E $tool_info | jq ".[0].bottle.stable.files.$os_name")
- if [[ "$os_bottle" != "null" ]]; then
- echo "false"
- return
- fi
-
- # Available method wasn't found - should build from source
- echo "true"
-}
-
 # brew provides package bottles for different macOS versions
 # The 'brew install' command will fail if a package bottle does not exist
 # Use the '--build-from-source' option to build from source in this case
 brew_smart_install() {
  local tool_name=$1
 
- local os_name=$(get_brew_os_keyword)
- if [[ "$os_name" == "null" ]]; then
- echo "$OSTYPE is unknown operating system"
- exit 1
- fi
+ echo "Downloading $tool_name..."
 
- local build_from_source=$(should_build_from_source "$tool_name" "$os_name")
- if $build_from_source; then
- echo "Bottle of the $tool_name for the $os_name was not found. Building $tool_name from source..."
- brew install --build-from-source $tool_name
- else
- echo "Downloading $tool_name..."
+ # get deps & cache em
 
- # get deps & cache em
+ failed=true
+ for i in {1..10}; do
+ brew deps $tool_name > /tmp/$tool_name && failed=false || sleep 60
+ [ "$failed" = false ] && break
+ done
 
- failed=true
- for i in {1..10}; do
- brew deps $tool_name > /tmp/$tool_name && failed=false || sleep 60
- [ "$failed" = false ] && break
- done
+ if [ "$failed" = true ]; then
+ echo "Failed: brew deps $tool_name"
+ exit 1;
+ fi
 
- if [ "$failed" = true ]; then
- echo "Failed: brew deps $tool_name"
- exit 1;
- fi
+ for dep in $(cat /tmp/$tool_name) $tool_name; do
 
- for dep in $(cat /tmp/$tool_name) $tool_name; do
-
- failed=true
- for i in {1..10}; do
- brew --cache $dep >/dev/null && failed=false || sleep 60
- [ "$failed" = false ] && break
- done
-
- if [ "$failed" = true ]; then
- echo "Failed: brew --cache $dep"
- exit 1;
- fi
- done
-
- failed=true
- for i in {1..10}; do
- brew install $tool_name >/dev/null && failed=false || sleep 60
- [ "$failed" = false ] && break
- done
-
- if [ "$failed" = true ]; then
- echo "Failed: brew install $tool_name"
- exit 1;
- fi
+ failed=true
+ for i in {1..10}; do
+ brew --cache $dep >/dev/null && failed=false || sleep 60
+ [ "$failed" = false ] && break
+ done
+
+ if [ "$failed" = true ]; then
+ echo "Failed: brew --cache $dep"
+ exit 1;
+ fi
+ done
+
+ failed=true
+ for i in {1..10}; do
+ brew install $tool_name >/dev/null && failed=false || sleep 60
+ [ "$failed" = false ] && break
+ done
 
+ if [ "$failed" = true ]; then
+ echo "Failed: brew install $tool_name"
+ exit 1;
  fi
 }
 
@@ -240,7 +199,18 @@ get_github_package_download_url() {
 
  [ -n "$API_PAT" ] && authString=(-H "Authorization: token ${API_PAT}")
 
- json=$(curl "${authString[@]}" -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}")
+ failed=true
+ for i in {1..10}; do
+ curl "${authString[@]}" -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}" >/tmp/get_github_package_download_url.json && failed=false || sleep 60
+ [ "$failed" = false ] && break
+ done
+
+ if [ "$failed" = true ]; then
+ echo "Failed: get_github_package_download_url"
+ exit 1;
+ fi
+
+ json=$(cat /tmp/get_github_package_download_url.json)
 
  if [[ "$VERSION" == "latest" ]]; then
  tagName=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0)).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]" | tail -1)

images/macos/tests/BasicTools.Tests.ps1

@@ -67,6 +67,13 @@ Describe "Helm" -Skip:($os.IsMonterey -or $os.IsVentura -or $os.IsVenturaArm64)
  }
 }
 
+Describe "Tcl/Tk" {
+ It "libtcl" {
+ "file /usr/local/lib/libtcl8.6.dylib" | Should -ReturnZeroExitCode
+ "file /usr/local/lib/libtk8.6.dylib" | Should -ReturnZeroExitCode
+ }
+}
+
 Describe "bazelisk" {
  It "bazelisk" {
  "bazelisk version" | Should -ReturnZeroExitCode
@@ -164,9 +171,9 @@ Describe "Homebrew" {
 }
 
 Describe "Kotlin" {
- $kotlinPackages =  @("kapt", "kotlin", "kotlinc", "kotlinc-jvm", "kotlin-dce-js")
+ $kotlinPackages = @("kapt", "kotlin", "kotlinc", "kotlinc-jvm", "kotlin-dce-js")
 
- It "<toolName> is available" -TestCases ($kotlinPackages | ForEach-Object {  @{ toolName = $_ } })  { 
+ It "<toolName> is available" -TestCases ($kotlinPackages | ForEach-Object { @{ toolName = $_ } }) { 
  "$toolName -version" | Should -ReturnZeroExitCode
  }
 }
@@ -187,4 +194,4 @@ Describe "imagemagick" -Skip:($os.IsVentura -or $os.IsVenturaArm64) {
  It "imagemagick" {
  "magick -version" | Should -ReturnZeroExitCode
  }
-}
+}

images/macos/tests/Common.Tests.ps1

@@ -123,12 +123,6 @@ Describe "Go" -Skip:($os.IsVentura -or $os.IsVenturaArm64) {
  }
 }
 
-Describe "VirtualBox" -Skip:($os.IsBigSur -or $os.IsVentura -or $os.IsVenturaArm64) {
- It "Check kext kernel modules" {
- kextstat | Out-String | Should -Match "org.virtualbox.kext"
- }
-}
-
 Describe "CodeQL Bundle" {
  It "Is installed" {
  $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*"