From a5bd58759dd6ad22f6c8439a80b5d0c5ebb908fd Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Mon, 8 Jul 2024 18:00:32 +0100 Subject: [PATCH 1/3] winPB: Switch to VS2022 build tools and add build dockerfile + related fixes Notes: - Adds Dockerfile.win2022 which can be used to create a build image comparable to the Linux ones. This will only install VS2022, not the earlier other compilers - Various modifications to support the above including: - Adding more 'win_reboot' tags to tasks to avoid attempting to reboot while running ansible inside the container - Use of gpg2 instead of gpg since that is what the new cygwin seems to have - Since the dockerfile does a base cygwin install, the ansible cygwin role now checks for jq.exe (the most recent addition) instead before deciding whether to skip the cygwin packages - Uses a windows command instead of cygwin (which may not be in the path at that point) as the "Dummy" command in the logs role - Changes the command used for default shortname creation to one that works in containers - Uses Visual Studio Build Tools instead of the community edition (Currently will not take effect if "adoptopenjdk" is skipped but that is skipped in the new dockerfile so it takes effect there) Signed-off-by: Stewart X Addison --- FAQ.md | 1 + ansible/docker/Dockerfile.win2022 | 32 +++++++++++++++++++ .../AdoptOpenJDK_Windows_Playbook/main.yml | 2 +- .../roles/Common/tasks/main.yml | 2 +- .../GPG_signature_verification/tasks/main.yml | 4 +-- .../roles/MSVS_2022/tasks/main.yml | 25 +++++++-------- .../roles/WMF_5.1/tasks/main.yml | 2 +- .../roles/cygwin/tasks/main.yml | 4 ++- .../roles/logs/tasks/main.yml | 7 ++-- .../roles/shortNames/tasks/main.yml | 5 +-- 10 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 ansible/docker/Dockerfile.win2022 diff --git a/FAQ.md b/FAQ.md index b2c552e2ef..6529c738bf 100644 --- a/FAQ.md +++ b/FAQ.md @@ -89,6 +89,7 @@ have at the moment: | [Centos6](./ansible/docker/Dockerfile.CentOS6) | [`adoptopenjdk/centos6_build_image`](https://hub.docker.com/r/adoptopenjdk/centos6_build_image)| linux/amd64 | [GH Actions](.github/workflows/build.yml) | Yes | [Alpine3](./ansible/docker/Dockerfile.Alpine3) | [`adoptopenjdk/alpine3_build_image`](https://hub.docker.com/r/adoptopenjdk/alpine3_build_image) | linux/x64 & linux/arm64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes | [Ubuntu 20.04 (riscv64 only)](./ansible/docker/Dockerfile.Ubuntu2004-riscv64) | [`adoptopenjdk/ubuntu2004_build_image:linux-riscv64`](https://hub.docker.com/r/adoptopenjdk/ubuntu2004_build_image) | linux/riscv64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes +| [Windows Server 2022](./ansible/docker/Dockerfile.win2022) | n/a - restricted | Windows | No
(*) - Caveats: diff --git a/ansible/docker/Dockerfile.win2022 b/ansible/docker/Dockerfile.win2022 new file mode 100644 index 0000000000..21e1e888da --- /dev/null +++ b/ansible/docker/Dockerfile.win2022 @@ -0,0 +1,32 @@ +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +# Specify this with --build-arg PW=SomePassword +ARG PW=T3mp=Passwd + +# Set up cygwin with git and ansible as a bootstrap, and add to system default path +RUN powershell wget -UseBasicParsing https://cygwin.com/setup-x86_64.exe -OutFile setup-x86_64.exe & \ + setup-x86_64.exe --packages git,ansible --download --local-install --delete-orphans --site https://mirrors.kernel.org/sourceware/cygwin --local-package-dir c:\cygwin_packages --root C:\cygwin64 --wait --quiet-mode & \ + C:\cygwin64\bin\git config --system core.autocrlf false & \ + del setup-x86_64.exe & \ + setx PATH "c:\cygwin64\bin;%PATH%" & \ + mkdir c:\temp + +# Set up WinRM for the ansible connection +RUN powershell wget -UseBasicParsing https://raw.githubusercontent.com/ansible/ansible/dd4c56e4d68664e4a50292aa19ea61b15c92287c/examples/scripts/ConfigureRemotingForAnsible.ps1 -OutFile ConfigureRemotingForAnsible.ps1 & \ + PowerShell .\ConfigureRemotingForAnsible.ps1 -CertValidityDays 9999 & \ + PowerShell .\ConfigureRemotingForAnsible.ps1 -EnableCredSSP & \ + PowerShell .\ConfigureRemotingForAnsible.ps1 -ForceNewSSLCert & \ + PowerShell .\ConfigureRemotingForAnsible.ps1 -SkipNetworkProfileCheck + +# Set up WinRM user, clone and run the playbook, then delete the user so it's not in any layer +ENV TERM dumb +RUN net user ansible %PW% /ADD & net localgroup "Administrators" ansible /ADD & net localgroup "Remote Management Users" ansible /ADD & \ + C:\cygwin64\bin\git clone https://github.com/sxa/infrastructure -b windows_docker_support c:/infrastructure & \ + sed -i -e 's/hosts: .*/hosts: localhost/' infrastructure/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml & \ + echo localhost ansible_connection=winrm > infrastructure/ansible/hosts & \ + cd infrastructure\ansible & \ + C:\cygwin64\bin\python3.7m.exe /usr/bin/ansible-playbook -e git_sha=00000000 -e ansible_user=ansible -e ansible_password=%PW% -i hosts \ + --skip-tags=adoptopenjdk,reboot,NTP_TIME,MSVS_2013,MSVS_2017,MSVS_2019 playbooks/AdoptOpenJDK_Windows_Playbook/main.yml & \ + net user ansible /DELETE + +ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml index afc10a15c5..60274091d0 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml @@ -22,7 +22,7 @@ # It can be defined as 'all' or a specific group which the host belongs to. # For example, it can be 'all' or 'x86' for when a host is in the group 'x86'. - name: Ansible Windows playbook - hosts: "{{ Groups | default('build*win*:test*win*') }}" + hosts: "{{ Groups | default('localhost:build:test:perf:jck:!*zos*:!*win*:!*aix*') }}" gather_facts: yes tasks: - name: Load Standard Variables diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/Common/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/Common/tasks/main.yml index 12461a994a..aa46f9d9a1 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/Common/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/Common/tasks/main.yml @@ -64,4 +64,4 @@ win_reboot: reboot_timeout: 1800 when: not (hostname_output.stdout == "") - tags: basic_config + tags: basic_config, reboot diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/GPG_signature_verification/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/GPG_signature_verification/tasks/main.yml index 11e07c75ab..99a8b1d961 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/GPG_signature_verification/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/GPG_signature_verification/tasks/main.yml @@ -24,7 +24,7 @@ - name: Import GPG Signing Key win_shell: | - C:/cygwin64/bin/bash.exe -c "gpg --batch --import c:/temp/gpgkey.asc" + C:/cygwin64/bin/bash.exe -c "gpg2 --batch --import c:/temp/gpgkey.asc" failed_when: false - name: Re-enable gpgagent @@ -39,7 +39,7 @@ win_command: wget -q "{{ signature_link }}" -O sigfile - name: Verify binary - win_shell: C:/cygwin64/bin/bash.exe -c "gpg --verify sigfile {{ file_path }}" + win_shell: C:/cygwin64/bin/bash.exe -c "gpg2 --verify sigfile {{ file_path }}" - name: Remove gnupg directory with Cygwin ( when it didnt exist previously ) win_shell: C:/cygwin64/bin/bash.exe -c "rm -rf ~/.gnupg" diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/MSVS_2022/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/MSVS_2022/tasks/main.yml index 63f63bce49..309943273f 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/MSVS_2022/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/MSVS_2022/tasks/main.yml @@ -10,7 +10,7 @@ - name: Test if VS 2022 is installed win_stat: - path: 'C:\Program Files\Microsoft Visual Studio\2022\Community' + path: 'C:\Program Files\Microsoft Visual Studio\2022' register: vs2022_installed tags: adoptopenjdk @@ -95,40 +95,37 @@ - name: Test if VS 2022 is installed(non adopt) win_stat: - path: 'C:\Program Files\Microsoft Visual Studio\2022\Community' + path: 'C:\Program Files\Microsoft Visual Studio\2022' register: vs2022_installed # Download & Install VS2022 When No Layout & Not AdoptOpenJDK # This is the target that you're redirected to when you go to https://aka.ms/vs/17/release/vs_community.exe - name: Download Visual Studio Community 2022 win_get_url: - url: 'https://aka.ms/vs/17/release/vs_Community.exe' - checksum: 44957c393b3c4dcaf1918f086437a520fc3b3dee83dbe786e5d757f331257856 +# https://learn.microsoft.com/en-us/visualstudio/releases/2022/release-history +# 17.7.34302.85 url: 'https://download.visualstudio.microsoft.com/download/pr/47b236ad-5505-4752-9d2b-5cf9795528bc/87684889f46dec53d1452f4a0ff9fec1ac202a97ebed866718d7c0269e814b28/vs_BuildTools.exe' + url: 'https://download.visualstudio.microsoft.com/download/pr/1d66edfe-3c83-476b-bf05-e8901c62ba7f/bac71effb5a23d7cd1a81e5f628a0c8dcb7e8a07e0aa7077c853ed84a862dceb/vs_BuildTools.exe' # 17.7.3 = 17.7.34024.191 + checksum: bac71effb5a23d7cd1a81e5f628a0c8dcb7e8a07e0aa7077c853ed84a862dceb checksum_algorithm: sha256 - dest: 'C:\temp\vs_community22.exe' + dest: 'C:\temp\vs_BuildTools22.exe' force: no when: (not vs2022_installed.stat.exists) and (windows_version.stdout_lines[0] | regex_search('^(10\.|11\.|2016|2019|2022)')) - name: Run Visual Studio 2022 Installer From Download win_shell: | - Start-Process -Wait -FilePath 'C:\temp\vs_community22.exe' -ArgumentList '--wait --add Microsoft.VisualStudio.Workload.NativeDesktop --includeRecommended --includeOptional --quiet --norestart' + Start-Process -Wait -FilePath 'C:\temp\vs_BuildTools22.exe' -ArgumentList '--wait --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add Microsoft.VisualStudio.Workload.NativeDesktop --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 --add Microsoft.VisualStudio.Component.VC.ATL.ARM64 --add Microsoft.VisualStudio.Component.VC.MFC.ARM64 --includeRecommended --includeOptional --quiet --norestart' args: executable: powershell when: (not vs2022_installed.stat.exists) and (windows_version.stdout_lines[0] | regex_search('^(10\.|11\.|2016|2019|2022)')) register: vs2022_error failed_when: vs2022_error.rc != 0 and vs2022_error.rc != 3010 -- name: Install ARM64 components - win_shell: Start-Process -FilePath 'C:\temp\vs_community22.exe' -Wait -NoNewWindow -ArgumentList - 'modify --installPath "C:\Program Files (x86)\Microsoft Visual Studio\2022\Community" --quiet - --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 --add Microsoft.VisualStudio.Component.VC.ATL.ARM64 --add Microsoft.VisualStudio.Component.VC.MFC.ARM64' - when: (not vs2022_installed.stat.exists) and (windows_version.stdout_lines[0] | regex_search('^(10\.|11\.|2016|2019|2022)')) - +# Note: If this does not find the files, then the NativeDesktop component was not installed - name: Register Visual Studio Community 2022 DIA SDK shared libraries win_command: 'regsvr32 /s "{{ item }}"' with_items: - - C:\Program Files\Microsoft Visual Studio\2022\Community\DIA SDK\bin\msdia140.dll - - C:\Program Files\Microsoft Visual Studio\2022\Community\DIA SDK\bin\amd64\msdia140.dll + - C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\DIA SDK\bin\msdia140.dll + - C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\DIA SDK\bin\amd64\msdia140.dll when: (not vs2022_installed.stat.exists) and (windows_version.stdout_lines[0] | regex_search('^(10\.|11\.|2016|2019|2022)')) - name: Reboot machine after Visual Studio installation diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/WMF_5.1/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/WMF_5.1/tasks/main.yml index 9ac9b2c0d2..79afd650e4 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/WMF_5.1/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/WMF_5.1/tasks/main.yml @@ -35,4 +35,4 @@ when: - (powershell_output.stdout < '5') - hotfix_install.reboot_required - tags: WMF + tags: WMF,reboot diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/cygwin/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/cygwin/tasks/main.yml index 443bd11eec..fdde87f728 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/cygwin/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/cygwin/tasks/main.yml @@ -4,7 +4,7 @@ ########## - name: Test if Cygwin is already installed win_stat: - path: 'C:\cygwin64' + path: 'C:\cygwin64\bin\jq.exe' register: cygwin_installed tags: cygwin @@ -15,6 +15,8 @@ when: not cygwin_installed.stat.exists tags: cygwin +# If you update this with a new package, modify the "Test +# if installed" to look for something in the new package - name: Install Cygwin win_shell: | Start-Process -Wait -FilePath 'C:\temp\cygwin.exe' -ArgumentList '--packages autoconf,automake,bsdtar,cmake,cpio,curl,gcc-core,git,gnupg,grep,jq,libtool,make,mingw64-x86_64-gcc-core,perl,rsync,unzip,wget,zip --quiet-mode --download --local-install --delete-orphans --site https://mirrors.kernel.org/sourceware/cygwin/ --local-package-dir C:\cygwin_packages --root C:\cygwin64' diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/logs/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/logs/tasks/main.yml index 160fca0c14..9a0adc85a0 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/logs/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/logs/tasks/main.yml @@ -7,14 +7,13 @@ # This task doesn't actually matter, aslong as it runs and is registered. The timestamp for the registered variable is used - name: Dummy task to get timestamp - win_command: whoami + win_command: cmd /c echo hello register: date_output # Accounts for cases where playbook executor is windows and its executing on localhost -- name: Get Latest git commit SHA (Windows Localhost) - win_command: git rev-parse HEAD +- name: Get Latest git commit SHA (Windows local container) + win_command: C:\cygwin64\bin\git -C C:/infrastructure rev-parse HEAD register: git_output - delegate_to: localhost ignore_errors: yes when: - git_sha is not defined diff --git a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/shortNames/tasks/main.yml b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/shortNames/tasks/main.yml index 028463cc4e..a425048cc1 100644 --- a/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/shortNames/tasks/main.yml +++ b/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/roles/shortNames/tasks/main.yml @@ -22,8 +22,9 @@ tags: - shortnames -- name: Enable shortnames on drive C:/ - win_shell: "fsutil 8dot3name set C: 0" +# Note that using "set C: 0" did not work in containers +- name: Enable shortnames + win_shell: "fsutil 8dot3name set 0" when: (not enabled_shortnames.stdout) tags: - shortnames From 3e6e5dfbfe86d7d9404387ba969f59e1b2cb9312 Mon Sep 17 00:00:00 2001 From: Stewart X Addison <6487691+sxa@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:02:07 +0100 Subject: [PATCH 2/3] Use ENV TERM=dumb Co-authored-by: George Adams --- ansible/docker/Dockerfile.win2022 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/docker/Dockerfile.win2022 b/ansible/docker/Dockerfile.win2022 index 21e1e888da..e33339be4d 100644 --- a/ansible/docker/Dockerfile.win2022 +++ b/ansible/docker/Dockerfile.win2022 @@ -19,7 +19,8 @@ RUN powershell wget -UseBasicParsing https://raw.githubusercontent.com/ansible/a PowerShell .\ConfigureRemotingForAnsible.ps1 -SkipNetworkProfileCheck # Set up WinRM user, clone and run the playbook, then delete the user so it's not in any layer -ENV TERM dumb +ENV TERM=dumb + RUN net user ansible %PW% /ADD & net localgroup "Administrators" ansible /ADD & net localgroup "Remote Management Users" ansible /ADD & \ C:\cygwin64\bin\git clone https://github.com/sxa/infrastructure -b windows_docker_support c:/infrastructure & \ sed -i -e 's/hosts: .*/hosts: localhost/' infrastructure/ansible/playbooks/AdoptOpenJDK_Windows_Playbook/main.yml & \ From 9c9e5cee9029cda93efd2a2ae3985f8a13484098 Mon Sep 17 00:00:00 2001 From: Stewart X Addison Date: Mon, 12 Aug 2024 14:57:15 +0100 Subject: [PATCH 3/3] Set USER to ContainerUser Signed-off-by: Stewart X Addison --- ansible/docker/Dockerfile.win2022 | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/docker/Dockerfile.win2022 b/ansible/docker/Dockerfile.win2022 index e33339be4d..94f4b6e628 100644 --- a/ansible/docker/Dockerfile.win2022 +++ b/ansible/docker/Dockerfile.win2022 @@ -31,3 +31,4 @@ RUN net user ansible %PW% /ADD & net localgroup "Administrators" ansible /ADD & net user ansible /DELETE ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] +USER ContainerUser