[CI] Fix CI (open-mmlab#2029)

* [CI] Fix CI * fix wrong command * remove mmcv * fix lint error * add pytorch install * fix pytorch installation * fix mmengine error * fix mmcv installation in pr_stage * fix docstring coverage in lint and delete cu102 in pr_stage windows * fix lint.yml and reset test.yml * ignore some ut in build_windows of pr_stage * test merge stage test * fix mmseg dependencies in pr_stage_test.yml * delete redundant lines in pr_stage and fix mmseg dependencies in mr_stage * fix error in merge_stage * delete python -m in merge_stage * fix error in merge_stage * let mmcv installation before mmengine * fix error of mmcv not found * fix ut error in merge)stage_test.yml * fix build_windows ut in metge_stage * fix error * fix windows error of merge_stag * Update .github/workflows/merge_stage_test.yml * Update .github/workflows/merge_stage_test.yml * Update .github/workflows/merge_stage_test.yml * fix error * delete skip timm ut * add requitements/optinal.txt in test.yml * Update .github/workflows/merge_stage_test.yml Co-authored-by: Miao Zheng <76149310+MeowZheng@users.noreply.github.com>
MeowZheng · Nov 1, 2022 · ce6a03a · ce6a03a
1 parent 8d2817e
commit ce6a03a
Show file tree

Hide file tree

Showing 3 changed files with 352 additions and 15 deletions.
diff --git a/.circleci/test.yml b/.circleci/test.yml
@@ -49,13 +49,13 @@ jobs:
  - run:
  name: Configure Python & pip
  command: |
- python -m pip install --upgrade pip
- python -m pip install wheel
+ pip install --upgrade pip
+ pip install wheel
  - run:
  name: Install PyTorch
  command: |
  python -V
- python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
+ pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
  - run:
  name: Install mmseg dependencies
  command: |
@@ -67,14 +67,14 @@ jobs:
  - run:
  name: Build and install
  command: |
- python -m pip install -e .
+ pip install -e .
  - run:
  name: Run unittests
  command: |
- python -m pip install timm
- python -m coverage run --branch --source mmseg -m pytest tests/
- python -m coverage xml
- python -m coverage report -m
+ pip install timm
+ coverage run --branch --source mmseg -m pytest tests/
+ coverage xml
+ coverage report -m
  - run:
  name: Skip timm unittests and generate coverage report
  command: |
@@ -101,8 +101,8 @@ jobs:
  # Cloning repos in VM since Docker doesn't have access to the private key
  name: Clone Repos
  command: |
- git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine
- git clone -b dev-1.x --depth 1 ssh://git@github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
+ git clone -b main --depth 1 https://github.com/open-mmlab/mmengine.git /home/circleci/mmengine
+ git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
  - run:
  name: Build Docker image
  command: |
@@ -115,16 +115,16 @@ jobs:
  docker exec mmseg pip install -U openmim 'importlib-metadata<2'
  docker exec mmseg mim install 'mmcv>=2.0.0rc1'
  docker exec mmseg pip install -e /mmclassification
- docker exec mmseg python -m pip install -r requirements.txt
+ docker exec mmseg pip install -r requirements/tests.txt -r requirements/optional.txt
  - run:
  name: Build and install
  command: |
  docker exec mmseg pip install -e .
  - run:
  name: Run unittests
  command: |
- docker exec mmseg python -m pip install timm
- docker exec mmseg python -m pytest tests/
+ docker exec mmseg pip install timm
+ docker exec mmseg pytest tests/
 workflows:
  pr_stage_lint:
  when: << pipeline.parameters.lint_only >>
@@ -159,8 +159,8 @@ workflows:
  - lint
  - build_cpu:
  name: maximum_version_cpu
- torch: 1.9.0
- torchvision: 0.10.0
+ torch: 1.12.1
+ torchvision: 0.13.1
  python: 3.9.0
  requires:
  - minimum_version_cpu

diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml
@@ -0,0 +1,196 @@
+name: merge_stage_test
+
+on:
+ push:
+ paths-ignore:
+ - 'README.md'
+ - 'README_zh-CN.md'
+ - 'docs/**'
+ - 'demo/**'
+ - '.dev_scripts/**'
+ - '.circleci/**'
+ branches:
+ - dev-1.x
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build_cpu_py:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.6, 3.8, 3.9]
+ torch: [1.8.1]
+ include:
+ - torch: 1.8.1
+ torchvision: 0.9.1
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Upgrade pip
+ run: pip install pip --upgrade
+ - name: Install Pillow
+ run: pip install Pillow==6.2.2
+ if: ${{matrix.torchvision == '0.4.2'}}
+ - name: Install PyTorch
+ run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+ - name: Install mmseg dependencies
+ run: |
+ python -V
+ pip install -U openmim
+ pip install git+https://github.com/open-mmlab/mmengine.git
+ mim install 'mmcv>=2.0.0rc1'
+ pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
+ - name: Install unittest dependencies
+ run: pip install -r requirements/tests.txt -r requirements/optional.txt
+ - name: Build and install
+ run: rm -rf .eggs && pip install -e .
+ - name: Run unittests and generate coverage report
+ run: |
+ pip install timm
+ coverage run --branch --source mmseg -m pytest tests/
+ coverage xml
+ coverage report -m
+ build_cpu_pt:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ torch: [1.6.0, 1.7.1, 1.8.1, 1.9.1, 1.10.1, 1.11.0]
+ include:
+ - torch: 1.6.0
+ torchvision: 0.7.0
+ - torch: 1.7.1
+ torchvision: 0.8.2
+ - torch: 1.8.1
+ torchvision: 0.9.1
+ - torch: 1.9.1
+ torchvision: 0.10.1
+ - torch: 1.10.1
+ torchvision: 0.11.2
+ - torch: 1.11.0
+ torchvision: 0.12.0
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Upgrade pip
+ run: pip install pip --upgrade
+ - name: Install Pillow
+ run: pip install Pillow==6.2.2
+ - name: Install PyTorch
+ run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+ - name: Install mmseg dependencies
+ run: |
+ python -V
+ pip install -U openmim
+ pip install git+https://github.com/open-mmlab/mmengine.git
+ mim install 'mmcv>=2.0.0rc1'
+ pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
+ - name: Install unittest dependencies
+ run: pip install -r requirements/tests.txt -r requirements/optional.txt
+ - name: Build and install
+ run: rm -rf .eggs && pip install -e .
+ - name: Run unittests and generate coverage report
+ run: |
+ pip install timm
+ coverage run --branch --source mmseg -m pytest tests/
+ coverage xml
+ coverage report -m
+ # Only upload coverage report for python3.7 && pytorch1.8.1 cpu
+ - name: Upload coverage to Codecov
+ if: ${{matrix.torch == '1.8.1' && matrix.python-version == '3.7'}}
+ uses: codecov/codecov-action@v2
+ with:
+ files: ./coverage.xml
+ flags: unittests
+ env_vars: OS,PYTHON
+ name: codecov-umbrella
+ fail_ci_if_error: false
+
+ build_cu102:
+ runs-on: ubuntu-18.04
+ container:
+ image: pytorch/pytorch:1.8.1-cuda10.2-cudnn7-devel
+ strategy:
+ matrix:
+ python-version: [3.7]
+ include:
+ - torch: 1.8.1
+ cuda: 10.2
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Upgrade pip
+ run: pip install pip --upgrade
+ - name: Fetch GPG keys
+ run: |
+ apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
+ apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
+ - name: Install Python-dev
+ run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
+ if: ${{matrix.python-version != 3.9}}
+ - name: Install system dependencies
+ run: |
+ apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6
+ - name: Install mmseg dependencies
+ run: |
+ python -V
+ pip install -U openmim
+ pip install git+https://github.com/open-mmlab/mmengine.git
+ mim install 'mmcv>=2.0.0rc1'
+ pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
+ - name: Install unittest dependencies
+ run: pip install -r requirements/tests.txt -r requirements/optional.txt
+ - name: Build and install
+ run: |
+ python setup.py check -m -s
+ TORCH_CUDA_ARCH_LIST=7.0 pip install -e .
+ build_windows:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [windows-2022]
+ python: [3.7]
+ platform: [cpu, cu111]
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Upgrade pip
+ run: pip install pip --upgrade
+ - name: Install OpenCV
+ run: pip install opencv-python>=3
+ - name: Install PyTorch
+ run: pip install torch==1.8.1+${{matrix.platform}} torchvision==0.9.1+${{matrix.platform}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+ - name: Install mmseg dependencies
+ run: |
+ python -V
+ pip install -U openmim
+ pip install git+https://github.com/open-mmlab/mmengine.git
+ mim install 'mmcv>=2.0.0rc1'
+ pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
+ - name: Install unittest dependencies
+ run: pip install -r requirements/tests.txt -r requirements/optional.txt
+ - name: Build and install
+ run: pip install -e .
+ - name: Run unittests
+ run: |
+ pip install timm
+ coverage run --branch --source mmseg -m pytest tests/ --ignore tests\test_models\test_forward.py tests\test_models\test_backbones\test_beit.py
+ - name: Generate coverage report
+ run: |
+ coverage xml
+ coverage report -m