From 553543aaf3d08f6f3c8a47fd4de516a10c08f042 Mon Sep 17 00:00:00 2001 From: Keeley Hammond Date: Mon, 21 Nov 2022 07:24:26 -0800 Subject: [PATCH] ci: bake appveyor images automatically, run sync on depshash change (#35396) * chore: update yml formatting for parser * ci: bake appveyor images automatically, run sync on depshash change * chore: clean up .yml files * chore: bump to e-110.0.5415.0 --- .github/workflows/update_appveyor_image.yml | 62 ++++ appveyor-bake.yml | 52 +++ appveyor.yml | 374 +++++++++----------- build/profile_toolchain.py | 56 +-- script/lib/utils.js | 1 + script/prepare-appveyor.js | 219 ++++++++++++ script/release/ci-release-build.js | 2 +- script/setup-win-for-dev.bat | 40 +-- spec/api-browser-view-spec.ts | 4 +- spec/api-browser-window-spec.ts | 7 +- 10 files changed, 529 insertions(+), 288 deletions(-) create mode 100644 .github/workflows/update_appveyor_image.yml create mode 100644 appveyor-bake.yml create mode 100644 script/prepare-appveyor.js diff --git a/.github/workflows/update_appveyor_image.yml b/.github/workflows/update_appveyor_image.yml new file mode 100644 index 00000000000000..9d84c7d9598d89 --- /dev/null +++ b/.github/workflows/update_appveyor_image.yml @@ -0,0 +1,62 @@ +name: Update AppVeyor Image + +# Run chron daily Mon-Fri +on: + schedule: + - cron: '0 8 * * 1-5' # runs 8:00 every business day (see https://crontab.guru) + +permissions: + contents: write + pull-requests: write + +jobs: + bake-appveyor-image: + name: Bake AppVeyor Image + permissions: + contents: write + pull-requests: write # to create a new PR with updated Appveyor images + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Yarn install + run: | + node script/yarn.js install --frozen-lockfile + - name: Set Repo for Commit + run: git config --global --add safe.directory $GITHUB_WORKSPACE + - name: Check AppVeyor Image + env: + APPVEYOR_TOKEN: ${{ secrets.APPVEYOR_TOKEN }} + run: | + node ./script/prepare-appveyor + if [ -f ./image_version.txt ]; then + echo "APPVEYOR_IMAGE_VERSION="$(cat image_version.txt)"" >> $GITHUB_ENV + rm image_version.txt + fi + - name: (Optionally) Update Appveyor Image + if: ${{ env.APPVEYOR_IMAGE_VERSION }} + uses: mikefarah/yq@v4.27.2 + with: + cmd: yq '.image = "${{ env.APPVEYOR_IMAGE_VERSION }}"' "appveyor.yml" > "appveyor2.yml" + - name: (Optionally) Generate Commit Diff + if: ${{ env.APPVEYOR_IMAGE_VERSION }} + run: | + diff -w -B appveyor.yml appveyor2.yml > appveyor.diff || true + patch -f appveyor.yml < appveyor.diff + rm appveyor2.yml appveyor.diff + - name: (Optionally) Commit and Pull Request + if: ${{ env.APPVEYOR_IMAGE_VERSION }} + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.ACTIONS_GITHUB_TOKEN }} + commit-message: 'build: update appveyor image to latest version' + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: bump-appveyor-image + delete-branch: true + title: 'build: update appveyor image to latest version' + body: | + This PR updates appveyor.yml to the latest baked image, ${{ env.APPVEYOR_IMAGE_VERSION }}. \ No newline at end of file diff --git a/appveyor-bake.yml b/appveyor-bake.yml new file mode 100644 index 00000000000000..26bfc782a62ac2 --- /dev/null +++ b/appveyor-bake.yml @@ -0,0 +1,52 @@ +# The config is used to bake appveyor images, not for running CI jobs. +# The config expects the following environment variables to be set: +# - "APPVEYOR_BAKE_IMAGE" e.g. 'electron-99.0.4767.0'. Name of the image to be baked. +# Typically named after the Chromium version on which the image is built. +# This can be set dynamically in the prepare-appveyor script. + +version: 1.0.{build} +build_cloud: electronhq-16-core +image: Windows_Default_Appveyor +environment: + GIT_CACHE_PATH: C:\Users\appveyor\libcc_cache + ELECTRON_OUT_DIR: Default + ELECTRON_ENABLE_STACK_DUMPING: 1 + MOCHA_REPORTER: mocha-multi-reporters + MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap + GOMA_FALLBACK_ON_AUTH_FAILURE: true + DEPOT_TOOLS_WIN_TOOLCHAIN: 0 + PYTHONIOENCODING: UTF-8 +build_script: + - ps: Resize-Partition -DriveLetter C -Size (256GB) # ensure initial partition size + - ps: Get-Partition -DriveLetter C + - git config --global core.longpaths true + - cd .. + - mkdir src + - ps: git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + - ps: $env:PATH="$pwd\depot_tools;$env:PATH" + - update_depot_tools.bat + - ps: Move-Item $env:APPVEYOR_BUILD_FOLDER -Destination src\electron + - src\electron\script\setup-win-for-dev.bat + - >- + gclient config + --name "src\electron" + --unmanaged + %GCLIENT_EXTRA_ARGS% + "https://github.com/electron/electron" + - ps: cd src\electron + - ps: node script\generate-deps-hash.js + - ps: $depshash = Get-Content .\.depshash -Raw + - ps: Copy-Item -path .\.depshash -destination ..\.depshash + - ps: cd ..\.. + - gclient sync --with_branch_heads --with_tags --nohooks + - ps: regsvr32 /s "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\DIA SDK\bin\amd64\msdia140.dll" +on_image_bake: + - ps: >- + echo "Baking image: $env:APPVEYOR_BAKE_IMAGE at dir $PWD" + - ps: Remove-Item -Recurse -Force $pwd\depot_tools + - ps: Remove-Item -Recurse -Force $pwd\src\electron +# Uncomment these lines to enable RDP +#on_finish: +# - ps: >- +# $env:APPVEYOR_RDP_PASSWORD = "electron" +# $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index 86dcc96ea8a458..4c13990ed6056d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,23 +24,24 @@ # https://www.appveyor.com/docs/build-configuration/#custom-environment-variables version: 1.0.{build} -build_cloud: electron-16-core -image: vs2019bt-16.16.11 +build_cloud: electronhq-16-core +image: e-110.0.5415.0 environment: - GIT_CACHE_PATH: C:\Users\electron\libcc_cache + GIT_CACHE_PATH: C:\Users\appveyor\libcc_cache ELECTRON_OUT_DIR: Default ELECTRON_ENABLE_STACK_DUMPING: 1 ELECTRON_ALSO_LOG_TO_STDERR: 1 MOCHA_REPORTER: mocha-multi-reporters MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap GOMA_FALLBACK_ON_AUTH_FAILURE: true + DEPOT_TOOLS_WIN_TOOLCHAIN: 0 + PYTHONIOENCODING: UTF-8 matrix: - - job_name: Build - - - job_name: Test - job_depends_on: Build + - job_name: Build + - job_name: Test + job_depends_on: Build clone_folder: C:\projects\src\electron @@ -50,167 +51,129 @@ matrix: for: - - - matrix: + - matrix: only: - job_name: Build init: - - ps: >- - if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) { - Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild - } + - ps: >- + if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) { + Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild + } build_script: - - ps: | - node script/yarn.js install --frozen-lockfile - node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH - if ($LASTEXITCODE -eq 0) { - Write-warning "Skipping tests for doc only change"; Exit-AppveyorBuild - } - $global:LASTEXITCODE = 0 - - cd .. - - ps: Write-Host "Building $env:GN_CONFIG build" - - git config --global core.longpaths true - - update_depot_tools.bat - - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" - $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE - } - - git clone https://github.com/electron/build-tools.git - - cd build-tools - - npm install - - mkdir third_party - - ps: >- - node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" - - ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)" - - ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)" - - cd ..\.. - - ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR - - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $goma_login = python $env:LOCAL_GOMA_DIR\goma_auth.py info - if ($goma_login -eq 'Login as Fermi Planck') { - Write-warning "Goma authentication is correct"; - } else { - Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token."; - $host.SetShouldExit(1) + - ps: | + node script/yarn.js install --frozen-lockfile + node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH + if ($LASTEXITCODE -eq 0) { + Write-warning "Skipping build for doc only change"; Exit-AppveyorBuild } - } - - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools" - - ps: >- - if ($env:GN_CONFIG -ne 'release') { - $env:NINJA_STATUS="[%r processes, %f/%t @ %o/s : %es] " - } - - >- - gclient config - --name "src\electron" - --unmanaged - %GCLIENT_EXTRA_ARGS% - "https://github.com/electron/electron" - - ps: >- - if ($env:GN_CONFIG -eq 'release') { - $env:RUN_GCLIENT_SYNC="true" - } else { - cd src\electron - node script\generate-deps-hash.js - $depshash = Get-Content .\.depshash -Raw - $zipfile = "Z:\$depshash.7z" - cd ..\.. - if (Test-Path -Path $zipfile) { - # file exists, unzip and then gclient sync - 7z x -y $zipfile -mmt=14 -aoa - if (-not (Test-Path -Path "src\buildtools")) { - # the zip file must be corrupt - resync - $env:RUN_GCLIENT_SYNC="true" - if ($env:TARGET_ARCH -ne 'ia32') { - # only save on x64/woa to avoid contention saving - $env:SAVE_GCLIENT_SRC="true" - } + $global:LASTEXITCODE = 0 + - cd .. + - ps: Write-Host "Building $env:GN_CONFIG build" + - git config --global core.longpaths true + - ps: >- + if (Test-Path -Path "$pwd\depot_tools") { + Remove-Item -Recurse -Force $pwd\depot_tools + } + - ps: >- + if (Test-Path -Path "$pwd\build-tools") { + Remove-Item -Recurse -Force $pwd\build-tools + } + - ps: git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + - ps: $env:PATH="$pwd\depot_tools;$env:PATH" + - ps: >- + if (Test-Path -Path "$pwd\src\electron") { + Remove-Item -Recurse -Force $pwd\src\electron + } + - ps: >- + if (Test-Path 'env:RAW_GOMA_AUTH') { + $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" + $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE + } + - git clone https://github.com/electron/build-tools.git + - cd build-tools + - npm install + - mkdir third_party + - ps: >- + node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })" + - ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)" + - ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)" + - cd ..\.. + - ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR + - ps: >- + if (Test-Path 'env:RAW_GOMA_AUTH') { + $goma_login = python $env:LOCAL_GOMA_DIR\goma_auth.py info + if ($goma_login -eq 'Login as Fermi Planck') { + Write-warning "Goma authentication is correct"; } else { - # update angle - cd src\third_party\angle - git remote set-url origin https://chromium.googlesource.com/angle/angle.git - git fetch - cd ..\..\.. - } - } else { - # file does not exist, gclient sync, then zip - $env:RUN_GCLIENT_SYNC="true" - if ($env:TARGET_ARCH -ne 'ia32') { - # only save on x64/woa to avoid contention saving - $env:SAVE_GCLIENT_SRC="true" + Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token."; + $host.SetShouldExit(1) } } - } - - if "%RUN_GCLIENT_SYNC%"=="true" ( gclient sync ) - - ps: >- - if ($env:SAVE_GCLIENT_SRC -eq 'true') { - # archive current source for future use - # only run on x64/woa to avoid contention saving - $(7z a $zipfile src -xr!android_webview -xr!electron -xr'!*\.git' -xr!third_party\blink\web_tests -xr!third_party\blink\perf_tests -slp -t7z -mmt=30) - if ($LASTEXITCODE -ne 0) { - Write-warning "Could not save source to shared drive; continuing anyway" + - ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools" + - ps: >- + if ($env:GN_CONFIG -ne 'release') { + $env:NINJA_STATUS="[%r processes, %f/%t @ %o/s : %es] " + } + - gclient config --name "src\electron" --unmanaged %GCLIENT_EXTRA_ARGS% "https://github.com/electron/electron" + # Patches are applied in the image bake. Check depshash to see if patches have changed. + - ps: $env:RUN_GCLIENT_SYNC="false" + - ps: $depshash_baked = Get-Content .\src\.depshash -Raw + - ps: cd src\electron + - ps: node script\generate-deps-hash.js + - ps: $depshash = Get-Content .\.depshash -Raw + - ps: cd ..\.. + - ps: >- + if ($depshash_baked -ne $depshash) { + $env:RUN_GCLIENT_SYNC="true" } - # build time generation of file gen/angle/angle_commit.h depends on - # third_party/angle/.git - # https://chromium-review.googlesource.com/c/angle/angle/+/2074924 - $(7z a $zipfile src\third_party\angle\.git) - if ($LASTEXITCODE -ne 0) { - Write-warning "Failed to add third_party\angle\.git; continuing anyway" + - if "%RUN_GCLIENT_SYNC%"=="true" ( gclient sync --with_branch_heads --with_tags ) else ( gclient runhooks ) + - cd src + - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn + - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% " + - gn check out/Default //electron:electron_lib + - gn check out/Default //electron:electron_app + - gn check out/Default //electron/shell/common/api:mojo + - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) + - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default ) + - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" + - ninja -C out/ffmpeg electron:electron_ffmpeg_zip + - ninja -C out/Default electron:electron_dist_zip + - ninja -C out/Default shell_browser_ui_unittests + - gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args + # Remove unused args from mksnapshot_args + - ps: >- + Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') } | Set-Content out/Default/mksnapshot_args + - ninja -C out/Default electron:electron_mksnapshot_zip + - cd out\Default + - 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S + - cd ..\.. + - ninja -C out/Default electron:hunspell_dictionaries_zip + - ninja -C out/Default electron:electron_chromedriver_zip + - ninja -C out/Default third_party/electron_node:headers + - python %LOCAL_GOMA_DIR%\goma_ctl.py stat + - ps: >- + Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json + - python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json + - 7z a node_headers.zip out\Default\gen\node_headers + - ps: >- + if ($env:GN_CONFIG -eq 'release') { + # Needed for msdia140.dll on 64-bit windows + $env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin" + ninja -C out/Default electron:electron_symbols } - # build time generation of file dawn/common/Version_autogen.h depends on third_party/dawn/.git/HEAD - # https://dawn-review.googlesource.com/c/dawn/+/83901 - $(7z a $zipfile src\third_party\dawn\.git) - if ($LASTEXITCODE -ne 0) { - Write-warning "Failed to add third_party\dawn\.git; continuing anyway" + - ps: >- + if ($env:GN_CONFIG -eq 'release') { + python3 electron\script\zip-symbols.py + appveyor-retry appveyor PushArtifact out/Default/symbols.zip + } else { + # It's useful to have pdb files when debugging testing builds that are + # built on CI. + 7z a pdb.zip out\Default\*.pdb } - } - - cd src - - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn - - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% " - - gn check out/Default //electron:electron_lib - - gn check out/Default //electron:electron_app - - gn check out/Default //electron/shell/common/api:mojo - - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) - - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default ) - - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" - - ninja -C out/ffmpeg electron:electron_ffmpeg_zip - - ninja -C out/Default electron:electron_dist_zip - - ninja -C out/Default shell_browser_ui_unittests - - gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args - - ps: >- - # Remove unused args from mksnapshot_args + - python3 electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.win.%TARGET_ARCH%.manifest - Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') } | Set-Content out/Default/mksnapshot_args - - ninja -C out/Default electron:electron_mksnapshot_zip - - cd out\Default - - 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S - - cd ..\.. - - ninja -C out/Default electron:hunspell_dictionaries_zip - - ninja -C out/Default electron:electron_chromedriver_zip - - ninja -C out/Default third_party/electron_node:headers - - python %LOCAL_GOMA_DIR%\goma_ctl.py stat - - python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json - - 7z a node_headers.zip out\Default\gen\node_headers - - ps: >- - if ($env:GN_CONFIG -eq 'release') { - # Needed for msdia140.dll on 64-bit windows - $env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin" - ninja -C out/Default electron:electron_symbols - } - - ps: >- - if ($env:GN_CONFIG -eq 'release') { - python3 electron\script\zip-symbols.py - appveyor-retry appveyor PushArtifact out/Default/symbols.zip - } else { - # It's useful to have pdb files when debugging testing builds that are - # built on CI. - 7z a pdb.zip out\Default\*.pdb - } - - python3 electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.win.%TARGET_ARCH%.manifest - deploy_script: - cd electron - ps: >- @@ -227,7 +190,7 @@ for: } on_finish: # Uncomment this lines to enable RDP - #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - cd C:\projects\src - if exist out\Default\windows_toolchain_profile.json ( appveyor-retry appveyor PushArtifact out\Default\windows_toolchain_profile.json ) - if exist out\Default\dist.zip (appveyor-retry appveyor PushArtifact out\Default\dist.zip) @@ -241,57 +204,55 @@ for: - ps: >- if ((Test-Path "pdb.zip") -And ($env:GN_CONFIG -ne 'release')) { appveyor-retry appveyor PushArtifact pdb.zip - } - - - - matrix: + } + - matrix: only: - job_name: Test init: - - ps: | - if ($env:RUN_TESTS -ne 'true') { - Write-warning "Skipping tests for $env:APPVEYOR_PROJECT_NAME"; Exit-AppveyorBuild - } - if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) { - Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild - } + - ps: | + if ($env:RUN_TESTS -ne 'true') { + Write-warning "Skipping tests for $env:APPVEYOR_PROJECT_NAME"; Exit-AppveyorBuild + } + if(($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME -split "/")[0] -eq ($env:APPVEYOR_REPO_NAME -split "/")[0]) { + Write-warning "Skipping PR build for branch"; Exit-AppveyorBuild + } build_script: - - ps: | - node script/yarn.js install --frozen-lockfile - node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH - if ($LASTEXITCODE -eq 0) { - Write-warning "Skipping tests for doc only change"; Exit-AppveyorBuild - } - $global:LASTEXITCODE = 0 - - ps: | - cd .. - mkdir out\Default - cd .. - # Download build artifacts - $apiUrl = 'https://ci.appveyor.com/api' - $build_info = Invoke-RestMethod -Method Get -Uri "$apiUrl/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/builds/$env:APPVEYOR_BUILD_ID" - $artifacts_to_download = @('dist.zip','shell_browser_ui_unittests.exe','chromedriver.zip','ffmpeg.zip','node_headers.zip','mksnapshot.zip','electron.lib') - foreach ($job in $build_info.build.jobs) { - if ($job.name -eq "Build") { - $jobId = $job.jobId - foreach($artifact_name in $artifacts_to_download) { - if ($artifact_name -eq 'shell_browser_ui_unittests.exe' -Or $artifact_name -eq 'electron.lib') { - $outfile = "src\out\Default\$artifact_name" - } else { - $outfile = $artifact_name + - ps: | + node script/yarn.js install --frozen-lockfile + node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH + if ($LASTEXITCODE -eq 0) { + Write-warning "Skipping build for doc only change"; Exit-AppveyorBuild + } + $global:LASTEXITCODE = 0 + - cd .. + - mkdir out\Default + - cd .. + - ps: | + # Download build artifacts + $apiUrl = 'https://ci.appveyor.com/api' + $build_info = Invoke-RestMethod -Method Get -Uri "$apiUrl/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/builds/$env:APPVEYOR_BUILD_ID" + $artifacts_to_download = @('dist.zip','shell_browser_ui_unittests.exe','chromedriver.zip','ffmpeg.zip','node_headers.zip','mksnapshot.zip','electron.lib') + foreach ($job in $build_info.build.jobs) { + if ($job.name -eq "Build") { + $jobId = $job.jobId + foreach($artifact_name in $artifacts_to_download) { + if ($artifact_name -eq 'shell_browser_ui_unittests.exe' -Or $artifact_name -eq 'electron.lib') { + $outfile = "src\out\Default\$artifact_name" + } else { + $outfile = $artifact_name + } + Invoke-RestMethod -Method Get -Uri "$apiUrl/buildjobs/$jobId/artifacts/$artifact_name" -OutFile $outfile } - Invoke-RestMethod -Method Get -Uri "$apiUrl/buildjobs/$jobId/artifacts/$artifact_name" -OutFile $outfile } } - } - - ps: | - $out_default_zips = @('dist.zip','chromedriver.zip','mksnapshot.zip') - foreach($zip_name in $out_default_zips) { - 7z x -y -osrc\out\Default $zip_name - } - - ps: 7z x -y -osrc\out\ffmpeg ffmpeg.zip - - ps: 7z x -y -osrc node_headers.zip + - ps: | + $out_default_zips = @('dist.zip','chromedriver.zip','mksnapshot.zip') + foreach($zip_name in $out_default_zips) { + 7z x -y -osrc\out\Default $zip_name + } + - ps: 7z x -y -osrc\out\ffmpeg ffmpeg.zip + - ps: 7z x -y -osrc node_headers.zip test_script: # Workaround for https://github.com/appveyor/ci/issues/2420 @@ -301,15 +262,22 @@ for: New-Item .\out\Default\gen\node_headers\Release -Type directory Copy-Item -path .\out\Default\electron.lib -destination .\out\Default\gen\node_headers\Release\node.lib - cd electron + # Explicitly set npm_config_arch because the .env doesn't persist + - ps: >- + if ($env:TARGET_ARCH -eq 'ia32') { + $env:npm_config_arch = "ia32" + } - echo Running main test suite & node script/yarn test -- --trace-uncaught --runners=main --enable-logging=file --log-file=%cd%\electron.log - echo Running native test suite & node script/yarn test -- --trace-uncaught --runners=native --enable-logging=file --log-file=%cd%\electron.log - cd .. - - echo Verifying non proprietary ffmpeg & python3 electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg + - echo Verifying non proprietary ffmpeg & python electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg - echo "About to verify mksnapshot" - - echo Verifying mksnapshot & python3 electron\script\verify-mksnapshot.py --build-dir out\Default --source-root %cd% + - echo Verifying mksnapshot & python electron\script\verify-mksnapshot.py --build-dir out\Default --source-root %cd% - echo "Done verifying mksnapshot" - - echo Verifying chromedriver & python3 electron\script\verify-chromedriver.py --build-dir out\Default --source-root %cd% + - echo Verifying chromedriver & python electron\script\verify-chromedriver.py --build-dir out\Default --source-root %cd% - echo "Done verifying chromedriver" - - on_finish: + + # Uncomment these lines to enable RDP + # on_finish: + # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - if exist electron\electron.log ( appveyor-retry appveyor PushArtifact electron\electron.log ) \ No newline at end of file diff --git a/build/profile_toolchain.py b/build/profile_toolchain.py index 4251315e0a16a1..6e51a7eaa14237 100755 --- a/build/profile_toolchain.py +++ b/build/profile_toolchain.py @@ -5,8 +5,6 @@ import os import optparse import json -import re -import subprocess sys.path.append("%s/../../build" % os.path.dirname(os.path.realpath(__file__))) @@ -36,56 +34,10 @@ def calculate_hash(root): return CalculateHash('.', None) def windows_installed_software(): - powershell_command = [ - "Get-CimInstance", - "-Namespace", - "root\cimv2", - "-Class", - "Win32_product", - "|", - "Select", - "vendor,", - "description,", - "@{l='install_location';e='InstallLocation'},", - "@{l='install_date';e='InstallDate'},", - "@{l='install_date_2';e='InstallDate2'},", - "caption,", - "version,", - "name,", - "@{l='sku_number';e='SKUNumber'}", - "|", - "ConvertTo-Json", - ] - - proc = subprocess.Popen( - ["powershell.exe", "-Command", "-"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - stdout, _ = proc.communicate(" ".join(powershell_command).encode("utf-8")) - - if proc.returncode != 0: - raise RuntimeError("Failed to get list of installed software") - - # On AppVeyor there's other output related to PSReadline, - # so grab only the JSON output and ignore everything else - json_match = re.match( - r".*(\[.*{.*}.*\]).*", stdout.decode("utf-8"), re.DOTALL - ) - - if not json_match: - raise RuntimeError( - "Couldn't find JSON output for list of installed software" - ) - - # Filter out missing keys - return list( - map( - lambda info: {k: info[k] for k in info if info[k]}, - json.loads(json_match.group(1)), - ) - ) + # file_path = os.path.join(os.getcwd(), 'installed_software.json') + # return json.loads(open('installed_software.json').read().decode('utf-8')) + f = open('installed_software.json', encoding='utf-8-sig') + return json.load(f) def windows_profile(): diff --git a/script/lib/utils.js b/script/lib/utils.js index 44c81dc86aca02..b562b03faaebdc 100644 --- a/script/lib/utils.js +++ b/script/lib/utils.js @@ -129,6 +129,7 @@ module.exports = { getElectronExec, getOutDir, getAbsoluteElectronExec, + handleGitCall, ELECTRON_DIR, SRC_DIR }; diff --git a/script/prepare-appveyor.js b/script/prepare-appveyor.js new file mode 100644 index 00000000000000..9960b877f5df67 --- /dev/null +++ b/script/prepare-appveyor.js @@ -0,0 +1,219 @@ +if (!process.env.CI) require('dotenv-safe').load(); + +const assert = require('assert'); +const fs = require('fs'); +const got = require('got'); +const path = require('path'); +const { handleGitCall, ELECTRON_DIR } = require('./lib/utils.js'); +const { Octokit } = require('@octokit/rest'); +const octokit = new Octokit(); + +const APPVEYOR_IMAGES_URL = 'https://ci.appveyor.com/api/build-clouds'; +const APPVEYOR_JOB_URL = 'https://ci.appveyor.com/api/builds'; +const ROLLER_BRANCH_PATTERN = /^roller\/chromium$/; + +const DEFAULT_BUILD_CLOUD_ID = '1598'; +const DEFAULT_BUILD_CLOUD = 'electronhq-16-core'; +const DEFAULT_BAKE_BASE_IMAGE = 'Windows_Default_Appveyor'; +const DEFAULT_BUILD_IMAGE = 'Windows_Default_Appveyor'; + +const appveyorBakeJob = 'electron-bake-image'; +const appVeyorJobs = { + 'electron-x64': 'electron-x64-testing', + 'electron-woa': 'electron-woa-testing', + 'electron-ia32': 'electron-ia32-testing' +}; + +async function makeRequest ({ auth, username, password, url, headers, body, method }) { + const clonedHeaders = { + ...(headers || {}) + }; + if (auth?.bearer) { + clonedHeaders.Authorization = `Bearer ${auth.bearer}`; + } + + const options = { + headers: clonedHeaders, + body, + method + }; + + if (username || password) { + options.username = username; + options.password = password; + } + + const response = await got(url, options); + + if (response.statusCode < 200 || response.statusCode >= 300) { + console.error('Error: ', `(status ${response.statusCode})`, response.body); + throw new Error(`Unexpected status code ${response.statusCode} from ${url}`); + } + return JSON.parse(response.body); +} + +async function checkAppVeyorImage (options) { + const IMAGE_URL = `${APPVEYOR_IMAGES_URL}/${options.cloudId}`; + const requestOpts = { + url: IMAGE_URL, + auth: { + bearer: process.env.APPVEYOR_TOKEN + }, + headers: { + 'Content-Type': 'application/json' + }, + method: 'GET' + }; + + try { + const { settings } = await makeRequest(requestOpts); + const { cloudSettings } = settings; + return cloudSettings.images.find(image => image.name === `${options.imageVersion}`) || null; + } catch (err) { + console.log('Could not call AppVeyor: ', err); + } +} + +async function getPullRequestId (targetBranch) { + const prsForBranch = await octokit.pulls.list({ + owner: 'electron', + repo: 'electron', + state: 'open', + head: `electron:${targetBranch}` + }); + if (prsForBranch.data.length === 1) { + return prsForBranch.data[0].number; + } else { + return null; + } +} + +function useAppVeyorImage (targetBranch, options) { + const validJobs = Object.keys(appVeyorJobs); + if (options.job) { + assert(validJobs.includes(options.job), `Unknown AppVeyor CI job name: ${options.job}. Valid values are: ${validJobs}.`); + callAppVeyorBuildJobs(targetBranch, options.job, options); + } else { + validJobs.forEach((job) => callAppVeyorBuildJobs(targetBranch, job, options)); + } +} + +async function callAppVeyorBuildJobs (targetBranch, job, options) { + console.log(`Using AppVeyor image ${options.version} for ${job}`); + + const pullRequestId = await getPullRequestId(targetBranch); + const environmentVariables = { + APPVEYOR_BUILD_WORKER_CLOUD: DEFAULT_BUILD_CLOUD, + APPVEYOR_BUILD_WORKER_IMAGE: options.version, + ELECTRON_OUT_DIR: 'Default', + ELECTRON_ENABLE_STACK_DUMPING: 1, + ELECTRON_ALSO_LOG_TO_STDERR: 1, + GOMA_FALLBACK_ON_AUTH_FAILURE: true, + DEPOT_TOOLS_WIN_TOOLCHAIN: 0, + PYTHONIOENCODING: 'UTF-8' + }; + + const requestOpts = { + url: APPVEYOR_JOB_URL, + auth: { + bearer: process.env.APPVEYOR_TOKEN + }, + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + accountName: 'electron-bot', + projectSlug: appVeyorJobs[job], + branch: targetBranch, + pullRequestId: pullRequestId || undefined, + commitId: options.commit || undefined, + environmentVariables + }), + method: 'POST' + }; + + try { + const { version } = await makeRequest(requestOpts); + const buildUrl = `https://ci.appveyor.com/project/electron-bot/${appVeyorJobs[job]}/build/${version}`; + console.log(`AppVeyor CI request for ${job} successful. Check status at ${buildUrl}`); + } catch (err) { + console.log('Could not call AppVeyor: ', err); + } +} + +async function bakeAppVeyorImage (targetBranch, options) { + console.log(`Baking a new AppVeyor image for ${options.version}, on build cloud ${options.cloudId}`); + + const environmentVariables = { + APPVEYOR_BUILD_WORKER_CLOUD: DEFAULT_BUILD_CLOUD, + APPVEYOR_BUILD_WORKER_IMAGE: DEFAULT_BAKE_BASE_IMAGE, + APPVEYOR_BAKE_IMAGE: options.version + }; + + const requestOpts = { + url: APPVEYOR_JOB_URL, + auth: { + bearer: process.env.APPVEYOR_TOKEN + }, + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + accountName: 'electron-bot', + projectSlug: appveyorBakeJob, + branch: targetBranch, + commitId: options.commit || undefined, + environmentVariables + }), + method: 'POST' + }; + + try { + const { version } = await makeRequest(requestOpts); + const bakeUrl = `https://ci.appveyor.com/project/electron-bot/${appveyorBakeJob}/build/${version}`; + console.log(`AppVeyor image bake request for ${options.version} successful. Check bake status at ${bakeUrl}`); + } catch (err) { + console.log('Could not call AppVeyor: ', err); + } +} + +async function prepareAppVeyorImage (opts) { + const branch = await handleGitCall(['rev-parse', '--abbrev-ref', 'HEAD'], ELECTRON_DIR); + if (ROLLER_BRANCH_PATTERN.test(branch)) { + useAppVeyorImage(branch, { ...opts, version: DEFAULT_BUILD_IMAGE, cloudId: DEFAULT_BUILD_CLOUD_ID }); + } else { + // eslint-disable-next-line no-control-regex + const versionRegex = new RegExp('chromium_version\':\n +\'(.+?)\',', 'm'); + const deps = fs.readFileSync(path.resolve(__dirname, '..', 'DEPS'), 'utf8'); + const [, CHROMIUM_VERSION] = versionRegex.exec(deps); + + const cloudId = opts.cloudId || DEFAULT_BUILD_CLOUD_ID; + const imageVersion = opts.imageVersion || `e-${CHROMIUM_VERSION}`; + const image = await checkAppVeyorImage({ cloudId, imageVersion }); + + if (image && image.name) { + console.log(`Image exists for ${image.name}. Continuing AppVeyor jobs using ${cloudId}.\n`); + } else { + console.log(`No AppVeyor image found for ${imageVersion} in ${cloudId}. + Creating new image for ${imageVersion}, using Chromium ${CHROMIUM_VERSION} - job will run after image is baked.`); + await bakeAppVeyorImage(branch, { ...opts, version: imageVersion, cloudId }); + + // write image to temp file if running on CI + if (process.env.CI) fs.writeFileSync('./image_version.txt', imageVersion); + } + } +} + +module.exports = prepareAppVeyorImage; + +// Load or bake AppVeyor images for Windows CI. +// Usage: prepare-appveyor.js [--cloudId=CLOUD_ID] [--appveyorJobId=xxx] [--imageVersion=xxx] +// [--commit=sha] [--branch=branch_name] +if (require.main === module) { + const args = require('minimist')(process.argv.slice(2)); + prepareAppVeyorImage(args) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/script/release/ci-release-build.js b/script/release/ci-release-build.js index ef3616374b4c81..1121f7b56125db 100644 --- a/script/release/ci-release-build.js +++ b/script/release/ci-release-build.js @@ -204,7 +204,7 @@ async function callAppVeyor (targetBranch, job, options) { console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`); const environmentVariables = { ELECTRON_RELEASE: 1, - APPVEYOR_BUILD_WORKER_CLOUD: 'libcc-20' + APPVEYOR_BUILD_WORKER_CLOUD: 'electronhq-16-core' }; if (!options.ghRelease) { diff --git a/script/setup-win-for-dev.bat b/script/setup-win-for-dev.bat index e2ee59f5abbb7b..8a4744602a4936 100644 --- a/script/setup-win-for-dev.bat +++ b/script/setup-win-for-dev.bat @@ -1,9 +1,8 @@ REM Parameters vs_buildtools.exe download link and wsdk version @ECHO OFF -SET buildtools_link=https://download.visualstudio.microsoft.com/download/pr/d7691cc1-82e6-434f-8e9f-a612f85b4b76/c62179f8cbbb58d4af22c21e8d4e122165f21615f529c94fad5cc7e012f1ef08/vs_BuildTools.exe -SET wsdk10_link=https://go.microsoft.com/fwlink/p/?LinkId=845298 -SET wsdk=10SDK.18362 +SET wsdk10_link=https://go.microsoft.com/fwlink/?linkid=2164145 +SET wsdk=10SDK.20348 REM Check for disk space Rem 543210987654321 @@ -44,24 +43,6 @@ IF NOT "%1"=="" ( if not exist "C:\TEMP\" mkdir C:\TEMP -REM Download vs_buildtools.exe to C:\TEMP\vs_buildtools.exe -powershell -command "& { iwr %buildtools_link% -OutFile C:\TEMP\vs_buildtools.exe }" - -REM Install Visual Studio Toolchain -C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ^ - --installPath "%ProgramFiles(x86)%/Microsoft Visual Studio/2019/Community" ^ - --add Microsoft.VisualStudio.Workload.VCTools ^ - --add Microsoft.VisualStudio.Component.VC.140 ^ - --add Microsoft.VisualStudio.Component.VC.ATLMFC ^ - --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ^ - --add Microsoft.VisualStudio.Component.VC.MFC.ARM64 ^ - --add Microsoft.VisualStudio.Component.Windows%wsdk% ^ - --includeRecommended - -REM Install Windows SDK -powershell -command "& { iwr %wsdk10_link% -OutFile C:\TEMP\wsdk10.exe }" -C:\TEMP\wsdk10.exe /features /quiet - REM Install chocolatey to further install dependencies set chocolateyUseWindowsCompression='true' @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" ^ @@ -69,14 +50,19 @@ set chocolateyUseWindowsCompression='true' -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" -REM Install nodejs python git and yarn needed dependencies -choco install -y nodejs python2 git yarn windows-sdk-10-version-1903-windbg -call C:\ProgramData\chocolatey\bin\RefreshEnv.cmd -SET PATH=C:\Python27\;C:\Python27\Scripts;%PATH% +REM Install Visual Studio Toolchain +choco install visualstudio2019buildtools --package-parameters "--quiet --wait --norestart --nocache --installPath ""%ProgramFiles(x86)%/Microsoft Visual Studio/2019/Community"" --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.VC.140 --add Microsoft.VisualStudio.Component.VC.ATLMFC --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 --add Microsoft.VisualStudio.Component.VC.MFC.ARM64 --add Microsoft.VisualStudio.Component.Windows%wsdk% --includeRecommended" -pip install pywin32 +REM Install Windows SDK +powershell -command "& { iwr %wsdk10_link% -OutFile C:\TEMP\wsdk10.exe }" +C:\TEMP\wsdk10.exe /features /quiet + +REM Install nodejs python git and yarn needed dependencies +choco install -y nodejs-lts python2 git yarn +choco install python --version 3.7.9 +choco install windows-sdk-10-version-2004-windbg call C:\ProgramData\chocolatey\bin\RefreshEnv.cmd -pip2 install pywin32 +SET PATH=C:\Python27\;C:\Python27\Scripts;C:\Python39\;C:\Python39\Scripts;%PATH% REM Setup Depot Tools git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git C:\depot_tools diff --git a/spec/api-browser-view-spec.ts b/spec/api-browser-view-spec.ts index e8b2c05b1beb34..89ee7af47d0b27 100644 --- a/spec/api-browser-view-spec.ts +++ b/spec/api-browser-view-spec.ts @@ -63,7 +63,7 @@ describe('BrowserView module', () => { }); // Linux and arm64 platforms (WOA and macOS) do not return any capture sources - ifit(process.platform !== 'linux' && process.arch !== 'arm64')('sets the background color to transparent if none is set', async () => { + ifit(process.platform === 'darwin' && process.arch === 'x64')('sets the background color to transparent if none is set', async () => { const display = screen.getPrimaryDisplay(); const WINDOW_BACKGROUND_COLOR = '#55ccbb'; @@ -87,7 +87,7 @@ describe('BrowserView module', () => { }); // Linux and arm64 platforms (WOA and macOS) do not return any capture sources - ifit(process.platform !== 'linux' && process.arch !== 'arm64')('successfully applies the background color', async () => { + ifit(process.platform === 'darwin' && process.arch === 'x64')('successfully applies the background color', async () => { const WINDOW_BACKGROUND_COLOR = '#55ccbb'; const VIEW_BACKGROUND_COLOR = '#ff00ff'; const display = screen.getPrimaryDisplay(); diff --git a/spec/api-browser-window-spec.ts b/spec/api-browser-window-spec.ts index 68bb96c614c488..9b6eb24f753e53 100644 --- a/spec/api-browser-window-spec.ts +++ b/spec/api-browser-window-spec.ts @@ -5519,7 +5519,7 @@ describe('BrowserWindow module', () => { }); // Linux and arm64 platforms (WOA and macOS) do not return any capture sources - ifit(process.platform !== 'linux' && process.arch !== 'arm64')('should not display a visible background', async () => { + ifit(process.platform === 'darwin' && process.arch !== 'x64')('should not display a visible background', async () => { const display = screen.getPrimaryDisplay(); const backgroundWindow = new BrowserWindow({ @@ -5557,7 +5557,7 @@ describe('BrowserWindow module', () => { expect(areColorsSimilar(rightHalfColor, HexColors.RED)).to.be.true(); }); - ifit(process.platform !== 'linux' && process.arch !== 'arm64')('Allows setting a transparent window via CSS', async () => { + ifit(process.platform === 'darwin')('Allows setting a transparent window via CSS', async () => { const display = screen.getPrimaryDisplay(); const backgroundWindow = new BrowserWindow({ @@ -5597,7 +5597,8 @@ describe('BrowserWindow module', () => { describe('"backgroundColor" option', () => { afterEach(closeAllWindows); - ifit(process.platform !== 'linux' && process.arch !== 'arm64')('should display the set color', async () => { + // Linux/WOA doesn't return any capture sources. + ifit(process.platform === 'darwin')('should display the set color', async () => { const display = screen.getPrimaryDisplay(); const w = new BrowserWindow({