From 429b5db6bf08f3fa4eaf094617783721dc5dfa6b Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Mon, 21 Mar 2022 10:17:56 -0400 Subject: [PATCH 1/5] Secondary draft PR for Azure to GHA migration --- .github/workflows/tests.yml | 202 +++++++++++++++++++++++------------- 1 file changed, 129 insertions(+), 73 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 956f9448e2..316d87802a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,6 +111,7 @@ jobs: windows: runs-on: windows-2019 strategy: + max-parallel: 10 fail-fast: false matrix: python-version: [ '3.7', '3.8' ] @@ -123,6 +124,7 @@ jobs: - python-version: '3.9' conda-version: canary test-type: parallel + timeout-minutes: 120 env: pytest-replay: --replay-record-dir=pytest-replay/ --replay-base-name=Windows-${{ matrix.conda-version }}-Py${{ matrix.python-version }} @@ -133,99 +135,153 @@ jobs: fetch-depth: 0 - name: Set temp dirs correctly - shell: bash -l {0} + shell: cmd # https://github.com/actions/virtual-environments/issues/712 run: | echo "TMPDIR=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 with: - condarc-file: ci/github/.condarc python-version: ${{ matrix.python-version }} - - name: Install Conda-Canary if canary version - shell: bash -l {0} - if: matrix.conda-version == 'canary' + - name: Install vcpython27.msi run: | - conda install -y -c conda-canary conda + $wc = New-Object net.webclient + $wc.Downloadfile("https://github.com/GeoNode/geonode-win-installer/raw/ffb76c7cbf1d6b4970c6c25f79c3c7682a3aa035/VCForPython27.msi", "VCForPython27.msi") + Start-Process "VCForPython27.msi" /qn -Wait + shell: pwsh - - name: Install default Conda if release version - shell: bash -l {0} - if: matrix.conda-version == 'release' + - name: Install miniconda run: | - conda install -y conda + rmdir C:\Strawberry /s /q + choco install wget + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe + start /wait "" Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /S /D=%UserProfile%\Miniconda3 + "%UserProfile%\Miniconda3\condabin\conda.bat" init + conda info -a + shell: cmd - - name: Source Scripts - shell: bash -l {0} + - name: Patch vs2008 run: | - source ci/azurepipelines/install_conda_build_test_deps - pip install -e . - conda info -a - mkdir %UserProfile%\cbtmp_serial - mkdir %UserProfile%\cbtmp - mkdir ./pytest-replay + Set-PSDebug -Trace 1 + $batchcontent = @" + ECHO ON + SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 + DIR "%vcpython%" + CALL "%vcpython%\vcvarsall.bat" %* + "@ + $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" + $batchPath = "$batchDir" + "\vcvarsall.bat" + New-Item -Path $batchPath -ItemType "file" -Force + Set-Content -Value $batchcontent -Path $batchPath + Get-ChildItem -Path $batchDir + Get-ChildItem -Path ($batchDir + '\..') + set LIB + shell: pwsh -# - name: Source Scripts -# shell: cmd -# run: | -# python -c "import sys; print(sys.version)" -# python -c "import sys; print(sys.executable)" -# python -c "import sys; print(sys.prefix)" -# call conda update -q --all||exit 1 -# call conda install -q pip python-libarchive-c pytest git pytest-cov jinja2 m2-patch flake8 mock requests contextlib2 chardet glob2 perl pyflakes pycrypto posix m2-git anaconda-client numpy beautifulsoup4 pytest-xdist pytest-mock filelock pkginfo psutil pytz tqdm conda-package-handling pytest-azurepipelines||exit 1 -# call conda install pytest-replay pytest-rerunfailures -y||exit 1 -# pip install --no-deps . -# conda-build --version -# pushd .. && git clone https://github.com/conda/conda_build_test_recipe && popd -# mkdir %UserProfile%\cbtmp_serial -# mkdir %UserProfile%\cbtmp -# for /d %%F in (%UserProfile%\cbtmp_serial\*) do rd /s /q "%%F" -# for /d %%F in (%UserProfile%\cbtmp\*) do rd /s /q "%%F" -# call conda create -n blarg -yq --download-only python=2.7||exit 1 -# call conda create -n blarg -yq --download-only python=3.8||exit 1 -# call conda create -n blarg -yq --download-only python cmake||exit 1 -# mkdir $(Build.ArtifactStagingDirectory)\\pytest-replay -# set "PYTEST_REPLAY_OPTIONS=--replay-record-dir=$(Build.ArtifactStagingDirectory)\\pytest-replay --replay-base-name=Win-%CONDA_VERSION%-Py%PYTHON_VERSION%" -# echo "##vso[task.setvariable variable=PYTEST_REPLAY_OPTIONS]%PYTEST_REPLAY_OPTIONS%" + - name: conda init + run: | + echo on + set PATH + doskey conda="call %UserProfile%\Miniconda3\condabin\conda.bat" $* + doskey /macros + call "%UserProfile%\Miniconda3\condabin\conda.bat" init + set PATH + shell: cmd - - name: Run Serial tests - shell: bash -l {0} - if: matrix.test-type == 'serial' + - name: Configuration run: | - pytest \ - --color=yes \ - -vv \ - -n 0 \ - --basetemp %UserProfile%\cbtmp_serial \ - --cov conda_build \ - --cov-report xml \ - -m "serial" tests \ - --test-run-title="Windows-SerialTests" \ - --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" \ - ${{ env.pytest-replay }} + echo on + set PATH + call %UserProfile%\Miniconda3\condabin\activate.bat base||exit 1 + set PATH + call conda install python="%PYTHON_VERSION%" -y||exit 1 + if "%CONDA_VERSION%" == "canary" (call conda update -c conda-canary conda||exit 1) else (call conda update -q conda||exit 1) + call conda config --set always_yes yes + call conda config --set auto_update_conda no + call conda info + python -c "import sys; print(sys.version)" + python -c "import sys; print(sys.executable)" + python -c "import sys; print(sys.prefix)" + call conda update -q --all||exit 1 + call conda install -q pip python-libarchive-c pytest git pytest-cov jinja2 m2-patch flake8 mock requests contextlib2 chardet glob2 perl pyflakes pycrypto posix m2-git anaconda-client numpy beautifulsoup4 pytest-xdist pytest-mock filelock pkginfo psutil pytz tqdm conda-package-handling pytest-azurepipelines||exit 1 + call conda install pytest-replay pytest-rerunfailures -y||exit 1 + echo safety_checks: disabled >> %UserProfile%\.condarc + echo local_repodata_ttl: 1800 >> %UserProfile%\.condarc + call conda install -q py-lief||exit 1 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + pip install --no-deps . + conda-build --version + pushd .. && git clone https://github.com/conda/conda_build_test_recipe && popd + mkdir %UserProfile%\cbtmp_serial + mkdir %UserProfile%\cbtmp + for /d %%F in (%UserProfile%\cbtmp_serial\*) do rd /s /q "%%F" + for /d %%F in (%UserProfile%\cbtmp\*) do rd /s /q "%%F" + call conda create -n blarg -yq --download-only python=2.7||exit 1 + call conda create -n blarg -yq --download-only python=3.8||exit 1 + call conda create -n blarg -yq --download-only python cmake||exit 1 + mkdir ${{ github.workspace }}\\pytest-replay + set "PYTEST_REPLAY_OPTIONS=--replay-record-dir=${{ github.workspace }}\\pytest-replay --replay-base-name=Win-%CONDA_VERSION%-Py%PYTHON_VERSION%" + echo "##vso[task.setvariable variable=PYTEST_REPLAY_OPTIONS]%PYTEST_REPLAY_OPTIONS%" + shell: cmd - - name: Run Parallel tests - shell: bash -l {0} - if: matrix.test-type == 'parallel' + - name: Serial Tests + if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'serial')) run: | - call pytest \ - --color=yes \ - -vv \ - -n auto \ - --basetemp %UserProfile%\cbtmp \ - --cov conda_build \ - --cov-append \ - --cov-report xml \ - -m "not serial" tests \ - --test-run-title="Windows-ParallelTests" \ - --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" \ - ${{ env.pytest-replay }} + echo on + set PATH + call %UserProfile%\Miniconda3\condabin\activate.bat base||exit 1 + set PATH + call conda install -y conda-verify||exit 1 + set PERL= + set LUA= + set R= + pytest + --color=yes + -vv + -n 0 + --basetemp %UserProfile%\cbtmp_serial + --cov conda_build + --cov-report xml + -m "serial" tests + --test-run-title="Windows-SerialTests" + --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" + ${{ env.pytest-replay }} + shell: cmd - - name: Upload Pytest Replay - uses: actions/upload-artifact@v2 + - name: Parallel Tests + if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'parallel')) + run: | + echo on + set PATH + call %UserProfile%\Miniconda3\condabin\activate.bat base||exit 1 + set PATH + :: call conda remove -y conda-verify||exit 1 + set PERL= + set LUA= + set R= + call pytest + --color=yes + -vv + -n auto + --basetemp %UserProfile%\cbtmp + --cov conda_build + --cov-append + --cov-report xml + -m "not serial" tests + --test-run-title="Windows-ParallelTests" + --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" + ${{ env.pytest-replay }} + shell: cmd + env: + VS90COMNTOOLS: C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin + LIB: + - uses: actions/upload-artifact@v2 with: name: Windows_replay-py-${{ matrix.python-version }}_${{ matrix.conda-version }}_${{ matrix.test-type }} path: pytest-replay/ + if: always() From 3e0a75828f59c7c6cfacc360d393a245fd5a6cc9 Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Mon, 21 Mar 2022 18:27:14 -0400 Subject: [PATCH 2/5] Tweak syntax for tests --- .github/workflows/tests.yml | 59 +++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 316d87802a..b1f635faed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,7 +111,6 @@ jobs: windows: runs-on: windows-2019 strategy: - max-parallel: 10 fail-fast: false matrix: python-version: [ '3.7', '3.8' ] @@ -126,7 +125,7 @@ jobs: test-type: parallel timeout-minutes: 120 env: - pytest-replay: --replay-record-dir=pytest-replay/ --replay-base-name=Windows-${{ matrix.conda-version }}-Py${{ matrix.python-version }} + serial_or_parallel: '' steps: - name: Checkout repository @@ -229,7 +228,7 @@ jobs: echo "##vso[task.setvariable variable=PYTEST_REPLAY_OPTIONS]%PYTEST_REPLAY_OPTIONS%" shell: cmd - - name: Serial Tests + - name: Run Serial Tests if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'serial')) run: | echo on @@ -240,20 +239,21 @@ jobs: set PERL= set LUA= set R= - pytest - --color=yes - -vv - -n 0 - --basetemp %UserProfile%\cbtmp_serial - --cov conda_build - --cov-report xml - -m "serial" tests - --test-run-title="Windows-SerialTests" - --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" - ${{ env.pytest-replay }} + > + py.test + --color=yes + -vv + -n 0 + --basetemp %UserProfile%\cbtmp_serial + --cov conda_build + --cov-report xml + -m "serial" tests + --test-run-title="Windows-SerialTests" + --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" + ${{ env.pytest-replay }} shell: cmd - - name: Parallel Tests + - name: Run Parallel Tests if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'parallel')) run: | echo on @@ -264,24 +264,25 @@ jobs: set PERL= set LUA= set R= - call pytest - --color=yes - -vv - -n auto - --basetemp %UserProfile%\cbtmp - --cov conda_build - --cov-append - --cov-report xml - -m "not serial" tests - --test-run-title="Windows-ParallelTests" - --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" - ${{ env.pytest-replay }} + > + py.test + --color=yes + -vv + -n auto + --basetemp %UserProfile%\cbtmp + --cov conda_build + --cov-append + --cov-report xml + -m "not serial" tests + --test-run-title="Windows-ParallelTests" + --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" + ${{ env.pytest-replay }} shell: cmd env: VS90COMNTOOLS: C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin LIB: - uses: actions/upload-artifact@v2 with: - name: Windows_replay-py-${{ matrix.python-version }}_${{ matrix.conda-version }}_${{ matrix.test-type }} - path: pytest-replay/ + path: ${{ github.workspace }}/pytest-replay + name: Win-${{ env.CONDA_VERSION }}-Python${{ matrix.python-version }}-${{ env. parameters.serial_or_parallel }} if: always() From d7bbbe12b8cce568b8613d6483095c729df02563 Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Mon, 21 Mar 2022 18:41:46 -0400 Subject: [PATCH 3/5] Tweak syntax for tests --- .github/workflows/tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b1f635faed..631bfcae06 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -239,8 +239,7 @@ jobs: set PERL= set LUA= set R= - > - py.test + py.test --color=yes -vv -n 0 @@ -264,8 +263,7 @@ jobs: set PERL= set LUA= set R= - > - py.test + py.test --color=yes -vv -n auto From 2ba43c84ecde6017cea53caa9f30ced634d1fcc1 Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Tue, 22 Mar 2022 10:42:23 -0400 Subject: [PATCH 4/5] Update line breaks to see if pytest flags are correctly recognized --- .github/workflows/tests.yml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 631bfcae06..68d36ce95d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -239,17 +239,7 @@ jobs: set PERL= set LUA= set R= - py.test - --color=yes - -vv - -n 0 - --basetemp %UserProfile%\cbtmp_serial - --cov conda_build - --cov-report xml - -m "serial" tests - --test-run-title="Windows-SerialTests" - --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" - ${{ env.pytest-replay }} + py.test --color=yes -vv -n 0 --basetemp %UserProfile%\cbtmp_serial --cov conda_build --cov-report xml -m "serial" --test-run-title="Windows-SerialTests" --test-run-title="Windows-SerialTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" ${{ env.pytest-replay }} shell: cmd - name: Run Parallel Tests @@ -263,18 +253,7 @@ jobs: set PERL= set LUA= set R= - py.test - --color=yes - -vv - -n auto - --basetemp %UserProfile%\cbtmp - --cov conda_build - --cov-append - --cov-report xml - -m "not serial" tests - --test-run-title="Windows-ParallelTests" - --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" - ${{ env.pytest-replay }} + py.test --color=yes -vv -n auto --basetemp %UserProfile%\cbtmp --cov conda_build --cov-append --cov-report xml -m "not serial" --test-run-title="Windows-ParallelTests" --test-run-title="Windows-ParallelTests-${{ matrix.conda-version }}-Py${{ matrix.python-version }}" ${{ env.pytest-replay }} shell: cmd env: VS90COMNTOOLS: C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin From 7cbfe1d5929015b2f10ff112c6249cf723c312f5 Mon Sep 17 00:00:00 2001 From: Bianca Henderson Date: Tue, 22 Mar 2022 11:41:21 -0400 Subject: [PATCH 5/5] Utilize build matrix to call parallel vs serial tests --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68d36ce95d..be335caeb5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -229,7 +229,7 @@ jobs: shell: cmd - name: Run Serial Tests - if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'serial')) + if: matrix.test-type == 'serial' run: | echo on set PATH @@ -243,7 +243,7 @@ jobs: shell: cmd - name: Run Parallel Tests - if: (success() && ('${{ env. parameters.serial_or_parallel }}' == 'parallel')) + if: matrix.test-type == 'parallel' run: | echo on set PATH