From 25317472b1327bfe92029800a20efcc1ca7f88ef Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 17 Apr 2023 23:16:08 -0400 Subject: [PATCH 1/5] updated CI --- .github/workflows/ci.yml | 76 +++++++++++++++++++++++++++++ .github/workflows/python-app.yml | 84 -------------------------------- requirements-tests.txt | 3 +- 3 files changed, 78 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..ae7596b33 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ + +name: CI + +on: + push: + branches: [ devel ] + pull_request: + branches: [ devel ] + +jobs: + + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m venv testenv + . testenv/bin/activate + python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade . + python -m pip install -r requirements-tests.txt + - name: Lint with flake8 and pylint + run: | + . testenv/bin/activate + LOG=`git log -n 1 | grep Merge` + NEW=`echo $LOG | cut -d ' ' -f2` + OLD=`echo $LOG | cut -d ' ' -f3` + (test -z "$NEW" || test -z "$OLD") && echo 'nothing to lint' && exit 0 + DIFF=`git diff --name-only --diff-filter=db $NEW..$OLD` + FILTERED=$(for f in $DIFF; do echo $f; done | \ + grep -e '\.py$' | sed "s|^|$PWD/|") + test -z "$FILTERED" && echo 'nothing to lint' && exit 0 + flake8 $FILTERED + pylint $FILTERED + + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m venv testenv + . testenv/bin/activate + python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade . + python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel + python -m pip install git+https://github.com/radical-cybertools/radical.pilot.git@devel + python -m pip install -r requirements-tests.txt + - name: Test with pytest + env: + RADICAL_DEBUG: TRUE + RADICAL_DEBUG_HELPER: TRUE + run: | + . testenv/bin/activate + radical-stack + coverage run --source=radical.utils -m pytest -ra -vvv --showlocals tests/unittests/ + coverage xml + - uses: codecov/codecov-action@v3 + if: ${{ matrix.python-version == '3.7' }} + with: + files: ./coverage.xml + diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index cdbf5e8a3..000000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,84 +0,0 @@ - -# This workflow will install Python dependencies, run tests and lint with -# a single version of Python -# -# For more information see: -# https://help.github.com/actions/language-and-framework-guides/ \ -# using-python-with-github-actions - -name: Build - -on: - push: - branches: [ devel ] - pull_request: - branches: [ devel ] - -jobs: - - tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m venv testenv - . testenv/bin/activate - python -m pip install --upgrade pip setuptools - python -m pip install --upgrade . - python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel - python -m pip install git+https://github.com/radical-cybertools/radical.pilot.git@devel - python -m pip install coverage codecov - python -m pip install pytest pytest-timeout - - name: Test with pytest - env: - RADICAL_DEBUG: TRUE - RADICAL_DEBUG_HELPER: TRUE - LOC: testenv/lib/python3.7/site-packages - run: | - . testenv/bin/activate - radical-stack - coverage run --include=$LOC/radical/utils/* -m pytest -ra --timeout=600 -vvv --showlocals tests/unittests/ - coverage xml - - name: Codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - - linting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m venv testenv - . testenv/bin/activate - python -m pip install --upgrade pip setuptools - python -m pip install --upgrade . - python -m pip install flake8 flake8-per-file-ignores - python -m pip install pytest pylint - - name: Lint with flake8 and pylint - run: | - . testenv/bin/activate - LOG=`git log -n 1 | grep Merge || true` - NEW=`echo $LOG | cut -d ' ' -f2` - OLD=`echo $LOG | cut -d ' ' -f3` - test -z "$NEW" && NEW=HEAD - test -z "$OLD" && OLD=HEAD - DIFF=`git diff --name-only --diff-filter=db $NEW..$OLD` - DIFF=$(echo $DIFF | grep -o -e '\b[^ ]*.py\b' | sed "s|^|$PWD/|") - test -z "$DIFF" && echo 'nothing to flake8' || flake8 $DIFF - test -z "$DIFF" && echo 'nothing to pylint' || pylint $DIFF - diff --git a/requirements-tests.txt b/requirements-tests.txt index b2071f71e..f1abac4cc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,9 +2,10 @@ -r requirements.txt pytest +pytest-timeout pylint flake8 +flake8-per-file-ignores coverage -mock==2.0.0. pudb From 9130e1b3e3521394e20c8595169e20e676fa50fe Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Mon, 17 Apr 2023 23:33:20 -0400 Subject: [PATCH 2/5] return back coverage run using `include` option --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae7596b33..85b6f2bf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,10 +64,11 @@ jobs: env: RADICAL_DEBUG: TRUE RADICAL_DEBUG_HELPER: TRUE + LOC: testenv/lib/python${{ matrix.python-version }}/site-packages run: | . testenv/bin/activate radical-stack - coverage run --source=radical.utils -m pytest -ra -vvv --showlocals tests/unittests/ + coverage run --include=$LOC/radical/utils/* -m pytest -ra -vvv --showlocals tests/unittests/ coverage xml - uses: codecov/codecov-action@v3 if: ${{ matrix.python-version == '3.7' }} From e2e94d464232ecc54d6542eec5772d0e5d2b61da Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Tue, 30 Apr 2024 11:21:17 -0400 Subject: [PATCH 3/5] last iteration to sync with RP's CI --- .github/workflows/ci.yml | 11 ++++------- requirements-ci.txt | 20 ++++++++++++++++++++ requirements-tests.txt | 11 ----------- requirements.txt | 1 - 4 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 requirements-ci.txt delete mode 100644 requirements-tests.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85b6f2bf4..d3d84ccc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,12 +23,12 @@ jobs: python -m venv testenv . testenv/bin/activate python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade . - python -m pip install -r requirements-tests.txt + python -m pip install -r requirements-ci.txt - name: Lint with flake8 and pylint run: | . testenv/bin/activate - LOG=`git log -n 1 | grep Merge` + radical-stack + LOG=`git log -n 1 | grep Merge` || echo 'no merging' && exit 0 NEW=`echo $LOG | cut -d ' ' -f2` OLD=`echo $LOG | cut -d ' ' -f3` (test -z "$NEW" || test -z "$OLD") && echo 'nothing to lint' && exit 0 @@ -56,10 +56,7 @@ jobs: python -m venv testenv . testenv/bin/activate python -m pip install --upgrade pip setuptools wheel - python -m pip install --upgrade . - python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel - python -m pip install git+https://github.com/radical-cybertools/radical.pilot.git@devel - python -m pip install -r requirements-tests.txt + python -m pip install -r requirements-ci.txt - name: Test with pytest env: RADICAL_DEBUG: TRUE diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 000000000..ae892d6c5 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,20 @@ + +# provided branches for RCT components can be edited, +# if there is a dependency with non-devel branch + +# default RCT branch for CI runs is "devel" + +radical.saga @ git+https://github.com/radical-cybertools/radical.utils@devel +radical.pilot @ git+https://github.com/radical-cybertools/radical.gtod@devel + +# RP from the current branch +. + +pytest +pytest-timeout +pylint +flake8 +flake8-per-file-ignores +coverage +pudb + diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index f1abac4cc..000000000 --- a/requirements-tests.txt +++ /dev/null @@ -1,11 +0,0 @@ - --r requirements.txt - -pytest -pytest-timeout -pylint -flake8 -flake8-per-file-ignores -coverage -pudb - diff --git a/requirements.txt b/requirements.txt index 556a8dcd1..b55e00ed8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ - colorama msgpack netifaces From 9bdde64d47a27b22e04e73058f15f4e9303a4fb0 Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Tue, 30 Apr 2024 11:22:57 -0400 Subject: [PATCH 4/5] fixed typo --- requirements-ci.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index ae892d6c5..355c5b649 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -4,8 +4,8 @@ # default RCT branch for CI runs is "devel" -radical.saga @ git+https://github.com/radical-cybertools/radical.utils@devel -radical.pilot @ git+https://github.com/radical-cybertools/radical.gtod@devel +radical.saga @ git+https://github.com/radical-cybertools/radical.saga@devel +radical.pilot @ git+https://github.com/radical-cybertools/radical.pilot@devel # RP from the current branch . From 14e0ef3444059696267963d34ace88bde0f33fcb Mon Sep 17 00:00:00 2001 From: Mikhail Titov Date: Tue, 30 Apr 2024 11:35:40 -0400 Subject: [PATCH 5/5] added CODECOV_TOKEN (CI) --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3d84ccc6..6008fb60a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: coverage xml - uses: codecov/codecov-action@v3 if: ${{ matrix.python-version == '3.7' }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: files: ./coverage.xml