From 88864b5498dc1e79ffb927fc61bd2a28ae5826e7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 3 Nov 2021 18:17:08 -0700 Subject: [PATCH 1/5] adding conditional docker restart to the docker startup yml --- eng/common/testproxy/test-proxy-docker.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 97617b6fd08a7..f77a4f441f52e 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -6,6 +6,18 @@ steps: $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' + # windows has a startup bug where the service is unavailable. See Azure/azure-sdk-tools#2122 + - pwsh: | + try { + docker info + } + catch { + Write-Host "Unable to access docker service, dumping error and restarting." + Write-Host $_ + Get-Service docker | Restart-Service + } + condition: and(succeededOrFailed(), eq(variables['Agent.OS'],'Windows_NT')) + - pwsh: | $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" displayName: 'Run the docker container' From f011938a2863728c4aa10b93fcd74c1899695ee7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:32:36 -0700 Subject: [PATCH 2/5] update the restart query --- eng/common/testproxy/test-proxy-docker.yml | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index f77a4f441f52e..c7e106144b7ae 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -8,14 +8,24 @@ steps: # windows has a startup bug where the service is unavailable. See Azure/azure-sdk-tools#2122 - pwsh: | - try { - docker info - } - catch { - Write-Host "Unable to access docker service, dumping error and restarting." - Write-Host $_ - Get-Service docker | Restart-Service - } + docker info + + # if info failed, we need to restart the service and start docker desktop + if($LASTEXITCODE -ne 0) + { + # Ensure the service is running + Get-Service com.docker.service | Restart-Service + + # Start docker desktop + $dockerExeLocation = (Get-Command docker).Source + + # on windows, docker.exe is shipped here: + # C:\Program Files\Docker\Docker\resources\bin\docker.exe + # Docker desktop lives in the same base path, just up a couple levels: + # C:\Program Files\Docker\Docker\Docker Desktop.exe + $dockerDesktopLocation = Resolve-Path (Join-Path -Path $dockerExeLocation -ChildPath "../../../Docker Desktop.exe") + &$dockerDesktopLocation + } condition: and(succeededOrFailed(), eq(variables['Agent.OS'],'Windows_NT')) - pwsh: | From d0ce9564da825051f647531449c648523e847b05 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 5 Nov 2021 12:06:44 -0700 Subject: [PATCH 3/5] updating the original container create to a retry --- eng/common/testproxy/docker-start-proxy.ps1 | 17 ++++++++++++++++- eng/common/testproxy/test-proxy-docker.yml | 20 +------------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 index 4137ae2aff569..1419cc1496354 100644 --- a/eng/common/testproxy/docker-start-proxy.ps1 +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -64,9 +64,24 @@ if ($Mode -eq "start"){ } # else we need to create it else { + $attempts = 0 Write-Host "Attempting creation of Docker host $CONTAINER_NAME" Write-Host "docker container create -v `"${root}:${Initial}/etc/testproxy`" $LinuxContainerArgs -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage" - docker container create -v "${root}:${Initial}/etc/testproxy" $LinuxContainerArgs -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage + while($attempts -lt 3){ + docker container create -v "${root}:${Initial}/etc/testproxy" $LinuxContainerArgs -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage + + if($LASTEXITCODE -ne 0){ + continue + } + else { + break + } + $attempts += 1 + } + + if($LASTEXITCODE -ne 0){ + exit(1) + } } Write-Host "Attempting start of Docker host $CONTAINER_NAME" diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index c7e106144b7ae..e4e4b2c8cdb0d 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -6,27 +6,9 @@ steps: $(Build.SourcesDirectory)/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' - # windows has a startup bug where the service is unavailable. See Azure/azure-sdk-tools#2122 - pwsh: | docker info - - # if info failed, we need to restart the service and start docker desktop - if($LASTEXITCODE -ne 0) - { - # Ensure the service is running - Get-Service com.docker.service | Restart-Service - - # Start docker desktop - $dockerExeLocation = (Get-Command docker).Source - - # on windows, docker.exe is shipped here: - # C:\Program Files\Docker\Docker\resources\bin\docker.exe - # Docker desktop lives in the same base path, just up a couple levels: - # C:\Program Files\Docker\Docker\Docker Desktop.exe - $dockerDesktopLocation = Resolve-Path (Join-Path -Path $dockerExeLocation -ChildPath "../../../Docker Desktop.exe") - &$dockerDesktopLocation - } - condition: and(succeededOrFailed(), eq(variables['Agent.OS'],'Windows_NT')) + displayName: 'Dump active docker information' - pwsh: | $(Build.SourcesDirectory)/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" From 62c2a7a7656415bd0a7f57c4107b993df77dbc8e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 5 Nov 2021 12:12:54 -0700 Subject: [PATCH 4/5] small bug with the not incrementing due to its location within the while loop --- eng/common/testproxy/docker-start-proxy.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 index 1419cc1496354..3489b2b863f1a 100644 --- a/eng/common/testproxy/docker-start-proxy.ps1 +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -71,12 +71,13 @@ if ($Mode -eq "start"){ docker container create -v "${root}:${Initial}/etc/testproxy" $LinuxContainerArgs -p 5001:5001 -p 5000:5000 --name $CONTAINER_NAME $SelectedImage if($LASTEXITCODE -ne 0){ + $attempts += 1 + Start-Sleep -s 1 continue } else { break } - $attempts += 1 } if($LASTEXITCODE -ne 0){ From 8a14caa44c19e02ddc86875dea084867f0248773 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 5 Nov 2021 16:25:29 -0700 Subject: [PATCH 5/5] add some output to list retries. --- eng/common/testproxy/docker-start-proxy.ps1 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eng/common/testproxy/docker-start-proxy.ps1 b/eng/common/testproxy/docker-start-proxy.ps1 index 3489b2b863f1a..577af6d24d87c 100644 --- a/eng/common/testproxy/docker-start-proxy.ps1 +++ b/eng/common/testproxy/docker-start-proxy.ps1 @@ -72,7 +72,12 @@ if ($Mode -eq "start"){ if($LASTEXITCODE -ne 0){ $attempts += 1 - Start-Sleep -s 1 + Start-Sleep -s 10 + + if($attempts -lt 3){ + Write-Host "Attempt $attempts failed. Retrying." + } + continue } else { @@ -81,6 +86,7 @@ if ($Mode -eq "start"){ } if($LASTEXITCODE -ne 0){ + Write-Host "Unable to successfully create docker container. Exiting." exit(1) } }