From 1efd48d61f871d6833158ad18a2a2920bf515d77 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 10:32:42 +0200 Subject: [PATCH 01/14] circle CI --- .circleci/artifact_path | 1 + .circleci/config.yml | 137 ++++++++++++++++++++++++++++++++++++++++ docs/requirements.txt | 5 ++ 3 files changed, 143 insertions(+) create mode 100644 .circleci/artifact_path create mode 100644 .circleci/config.yml create mode 100644 docs/requirements.txt diff --git a/.circleci/artifact_path b/.circleci/artifact_path new file mode 100644 index 000000000..aa9acb865 --- /dev/null +++ b/.circleci/artifact_path @@ -0,0 +1 @@ +0/docs/build/html/index.html diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..9701ad14a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,137 @@ +# Tagging a commit with [circle front] will build the front page and perform test-doc. +# Tagging a commit with [circle full] will build everything. +version: 2 +jobs: + build_docs: + docker: + - image: circleci/python:3.7-stretch + steps: + - checkout + - run: + name: Set BASH_ENV + command: | + echo "set -e" >> $BASH_ENV + echo "export DISPLAY=:99" >> $BASH_ENV + echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV + echo "BASH_ENV:" + cat $BASH_ENV + + - run: + name: Merge with upstream + command: | + echo $(git log -1 --pretty=%B) | tee gitlog.txt + echo ${CI_PULL_REQUEST//*pull\//} | tee merge.txt + if [[ $(cat merge.txt) != "" ]]; then + echo "Merging $(cat merge.txt)"; + git remote add upstream git://github.com/PythonOT/POT.git; + git pull --ff-only upstream "refs/pull/$(cat merge.txt)/merge"; + git fetch upstream master; + fi + + # Load our data + - restore_cache: + keys: + - data-cache-0 + - pip-cache + + - run: + name: Spin up Xvfb + command: | + /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1400x900x24 -ac +extension GLX +render -noreset; + + # https://github.com/ContinuumIO/anaconda-issues/issues/9190#issuecomment-386508136 + # https://github.com/golemfactory/golem/issues/1019 + - run: + name: Fix libgcc_s.so.1 pthread_cancel bug + command: | + sudo apt-get install qt5-default + + - run: + name: Get Python running + command: | + python -m pip install --user --upgrade --progress-bar off pip + python -m pip install --user --upgrade --progress-bar off -r requirements.txt + python -m pip install --user --upgrade --progress-bar off -r docs/requirements.txt + python -m pip install --user --upgrade --progress-bar off ipython "https://api.github.com/repos/sphinx-gallery/sphinx-gallery/zipball/master" memory_profiler + python -m pip install --user -e . + + - save_cache: + key: pip-cache + paths: + - ~/.cache/pip + + # Look at what we have and fail early if there is some library conflict + - run: + name: Check installation + command: | + which python + python -c "import ot" + + # Build docs + - run: + name: make html + command: | + cd docs; + make html; + + # Save the outputs + - store_artifacts: + path: docs/build/html/ + destination: dev + - persist_to_workspace: + root: docs/build + paths: + - html + + deploy: + docker: + - image: circleci/python:3.6-jessie + steps: + - attach_workspace: + at: /tmp/build + - run: + name: Fetch docs + command: | + set -e + mkdir -p ~/.ssh + echo -e "Host *\nStrictHostKeyChecking no" > ~/.ssh/config + chmod og= ~/.ssh/config + if [ ! -d ~/PythonOT.github.io ]; then + git clone git@github.com:/PythonOT/PythonOT.github.io.git ~/PythonOT.github.io --depth=1 + fi + - run: + name: Deploy docs + command: | + set -e; + if [ "${CIRCLE_BRANCH}" == "master" ]; then + git config --global user.email "circle@PythonOT.com"; + git config --global user.name "Circle CI"; + cd ~/PythonOT.github.io; + git checkout master + git remote -v + git fetch origin + git reset --hard origin/master + git clean -xdf + echo "Deploying dev docs for ${CIRCLE_BRANCH}."; + cp -a /tmp/build/html/* .; + touch .nojekyll; + git add -A; + git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; + git push origin master; + else + echo "No deployment (build: ${CIRCLE_BRANCH})."; + fi + +workflows: + version: 2 + + default: + jobs: + - build_docs + - deploy: + requires: + - build_docs + filters: + branches: + only: + - master diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..1fe37c259 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +sphinx_gallery +sphinx_rtd_theme +numpydoc +memory_profiler +pillow From f141b7ecec8f69db29fb3dcf1efd1d00d96db87d Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 10:53:59 +0200 Subject: [PATCH 02/14] add artifact redirect --- .github/workflows/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..ca10c8c2c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,12 @@ +on: [status] +jobs: + circleci_artifacts_redirector_job: + runs-on: ubuntu-latest + name: Run CircleCI artifacts redirector + steps: + - name: GitHub Action step + uses: larsoner/circleci-artifacts-redirector-action@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + artifact-path: 0/docs/build/html/index.html + circleci-jobs: build_doc From 8c6a976c9d1225e469e22f7ef1c700eaf1e08dd7 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 11:00:24 +0200 Subject: [PATCH 03/14] try codecov setup --- .github/workflows/main.yml | 2 +- .travis.yml | 5 ++++- codecov.yml | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca10c8c2c..1a8a759c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,4 +9,4 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} artifact-path: 0/docs/build/html/index.html - circleci-jobs: build_doc + circleci-jobs: build_docs diff --git a/.travis.yml b/.travis.yml index b9cc88ffb..3f63867d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ before_install: install: - pip install -r requirements.txt - pip install -U "numpy>=1.14" scipy # for numpy array formatting in doctests - - pip install flake8 pytest "pytest-cov<2.6" + - pip install flake8 pytest "pytest-cov<2.6" codecov - pip install -U "sklearn" - pip install . # command to run tests + check syntax style @@ -45,3 +45,6 @@ script: - flake8 examples/ ot/ test/ - python -m pytest -v test/ ot/ --doctest-modules --ignore ot/gpu/ --cov=ot # - py.test ot test +after_script: + # Need to run from source dir to execute "git" commands + - codecov; diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..1e7b888a8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,14 @@ +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: auto + threshold: 0.01 + patch: false + changes: false +comment: + layout: "header, diff, sunburst, uncovered" + behavior: default From 555f8426a8cb66ae5255885ce555037dc8c72c53 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 11:11:19 +0200 Subject: [PATCH 04/14] add codecov button --- .github/workflows/main.yml | 1 - README.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a8a759c3..6b52555c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,5 @@ jobs: - name: GitHub Action step uses: larsoner/circleci-artifacts-redirector-action@master with: - repo-token: ${{ secrets.GITHUB_TOKEN }} artifact-path: 0/docs/build/html/index.html circleci-jobs: build_docs diff --git a/README.md b/README.md index 931a2521a..64630e63b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![PyPI version](https://badge.fury.io/py/POT.svg)](https://badge.fury.io/py/POT) [![Anaconda Cloud](https://anaconda.org/conda-forge/pot/badges/version.svg)](https://anaconda.org/conda-forge/pot) [![Build Status](https://travis-ci.org/PythonOT/POT.svg?branch=master)](https://travis-ci.org/PythonOT/POT) +[![Codecov Status](https://codecov.io/gh/PythonOT/POT/branch/master/graph/badge.svg)](https://codecov.io/gh/PythonOT/POT) [![Documentation Status](https://readthedocs.org/projects/pot/badge/?version=latest)](http://pot.readthedocs.io/en/latest/?badge=latest) [![Downloads](https://pepy.tech/badge/pot)](https://pepy.tech/project/pot) [![Anaconda downloads](https://anaconda.org/conda-forge/pot/badges/downloads.svg)](https://anaconda.org/conda-forge/pot) From f48d51579d843f12a73f29d44374bb63e627238d Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 11:19:41 +0200 Subject: [PATCH 05/14] try to push doc to github.io --- .circleci/config.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9701ad14a..455b7006b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,24 +103,24 @@ jobs: name: Deploy docs command: | set -e; - if [ "${CIRCLE_BRANCH}" == "master" ]; then - git config --global user.email "circle@PythonOT.com"; - git config --global user.name "Circle CI"; - cd ~/PythonOT.github.io; - git checkout master - git remote -v - git fetch origin - git reset --hard origin/master - git clean -xdf - echo "Deploying dev docs for ${CIRCLE_BRANCH}."; - cp -a /tmp/build/html/* .; - touch .nojekyll; - git add -A; - git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; - git push origin master; - else - echo "No deployment (build: ${CIRCLE_BRANCH})."; - fi + # if [ "${CIRCLE_BRANCH}" == "master" ]; then + git config --global user.email "circle@PythonOT.com"; + git config --global user.name "Circle CI"; + cd ~/PythonOT.github.io; + git checkout master + git remote -v + git fetch origin + git reset --hard origin/master + git clean -xdf + echo "Deploying dev docs for ${CIRCLE_BRANCH}."; + cp -a /tmp/build/html/* .; + touch .nojekyll; + git add -A; + git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; + git push origin master; + # else + # echo "No deployment (build: ${CIRCLE_BRANCH})."; + # fi workflows: version: 2 From d3ebb36fc4a3614c5640d731ea793c0afd400e4e Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 11:23:34 +0200 Subject: [PATCH 06/14] run gh actions all the time --- .github/workflows/main.yml | 11 ++++++++++- .github/workflows/pythonpackage.yml | 10 +++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b52555c3..46f278a34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,13 @@ -on: [status] +on: + push: + branches: + - '**' + create: + branches: + - 'master' + tags: + - '**' + jobs: circleci_artifacts_redirector_job: runs-on: ubuntu-latest diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cb3baf898..d728edd8e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -1,6 +1,14 @@ name: Test Package -on: [push] +on: + push: + branches: + - '**' + create: + branches: + - 'master' + tags: + - '**' jobs: build: From 624fd44c729a3b761d09a01b71338c0332b6b9fe Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 11:24:42 +0200 Subject: [PATCH 07/14] fix --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 46f278a34..07686d9e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,5 +16,6 @@ jobs: - name: GitHub Action step uses: larsoner/circleci-artifacts-redirector-action@master with: + repo-token: ${{ secrets.GITHUB_TOKEN }} artifact-path: 0/docs/build/html/index.html circleci-jobs: build_docs From 5c88642a27e6c330a5c795898c0de7f3b4c0cd8d Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:00:03 +0200 Subject: [PATCH 08/14] fix? --- .circleci/config.yml | 3 ++- .github/workflows/main.yml | 13 ++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 455b7006b..ea5981e7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,4 +134,5 @@ workflows: filters: branches: only: - - master + # - master + - doc_ci_build diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07686d9e1..7153fe66a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,13 +1,4 @@ -on: - push: - branches: - - '**' - create: - branches: - - 'master' - tags: - - '**' - +on: [status] jobs: circleci_artifacts_redirector_job: runs-on: ubuntu-latest @@ -17,5 +8,5 @@ jobs: uses: larsoner/circleci-artifacts-redirector-action@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} - artifact-path: 0/docs/build/html/index.html + artifact-path: 0/dev/index.html circleci-jobs: build_docs From 8497cd523085ea69d949a0de67db252b36a81d8e Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:03:44 +0200 Subject: [PATCH 09/14] run tests in GH actions --- .github/workflows/pythonpackage.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d728edd8e..d60acd28b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -29,10 +29,18 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install flake8 pytest "pytest-cov<2.6" codecov + pip install -U "sklearn" - name: Lint with flake8 run: | - pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Run tests + run: | + pip install . + python -m pytest -v test/ ot/ --doctest-modules --ignore ot/gpu/ --cov=ot + - name: Upload codecov + run: | + codecov From bdf608bfb417a15545d581aa2d57160401aacb6e Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:09:53 +0200 Subject: [PATCH 10/14] fix? --- .github/workflows/pythonpackage.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d60acd28b..c4b0165a4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,7 +17,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [2.7, 3.5, 3.6, 3.7] + python-version: [3.5, 3.6, 3.7, 3.8] steps: - uses: actions/checkout@v1 @@ -37,9 +37,11 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Install POT + run: | + pip install -e . - name: Run tests run: | - pip install . python -m pytest -v test/ ot/ --doctest-modules --ignore ot/gpu/ --cov=ot - name: Upload codecov run: | From 3e10d08ab7bb1a58053505e6e5bb6c8715e90854 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:12:40 +0200 Subject: [PATCH 11/14] fix? --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea5981e7b..6817880ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,10 @@ workflows: jobs: - build_docs - deploy: + steps: + - add_ssh_keys: + fingerprints: + - "ff:30:7b:ba:bd:5c:ec:27:49:12:11:cb:78:aa:c2:6e" requires: - build_docs filters: From 8da537d72414a7ce347f39ef64c6c819610c660c Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:14:21 +0200 Subject: [PATCH 12/14] fix? --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6817880ff..80fe9350b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,9 +130,9 @@ workflows: - build_docs - deploy: steps: - - add_ssh_keys: - fingerprints: - - "ff:30:7b:ba:bd:5c:ec:27:49:12:11:cb:78:aa:c2:6e" + - add_ssh_keys: + fingerprints: + - "ff:30:7b:ba:bd:5c:ec:27:49:12:11:cb:78:aa:c2:6e" requires: - build_docs filters: From 9b66b194cdfc6b7e4b35488fa093d936fa08e011 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:15:01 +0200 Subject: [PATCH 13/14] fix? --- .circleci/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80fe9350b..ea5981e7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,10 +129,6 @@ workflows: jobs: - build_docs - deploy: - steps: - - add_ssh_keys: - fingerprints: - - "ff:30:7b:ba:bd:5c:ec:27:49:12:11:cb:78:aa:c2:6e" requires: - build_docs filters: From 3eed3cad36ec5f5691dc029853bc36bfad5ee389 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 21 Apr 2020 12:25:36 +0200 Subject: [PATCH 14/14] all good --- .circleci/config.yml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea5981e7b..9701ad14a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,24 +103,24 @@ jobs: name: Deploy docs command: | set -e; - # if [ "${CIRCLE_BRANCH}" == "master" ]; then - git config --global user.email "circle@PythonOT.com"; - git config --global user.name "Circle CI"; - cd ~/PythonOT.github.io; - git checkout master - git remote -v - git fetch origin - git reset --hard origin/master - git clean -xdf - echo "Deploying dev docs for ${CIRCLE_BRANCH}."; - cp -a /tmp/build/html/* .; - touch .nojekyll; - git add -A; - git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; - git push origin master; - # else - # echo "No deployment (build: ${CIRCLE_BRANCH})."; - # fi + if [ "${CIRCLE_BRANCH}" == "master" ]; then + git config --global user.email "circle@PythonOT.com"; + git config --global user.name "Circle CI"; + cd ~/PythonOT.github.io; + git checkout master + git remote -v + git fetch origin + git reset --hard origin/master + git clean -xdf + echo "Deploying dev docs for ${CIRCLE_BRANCH}."; + cp -a /tmp/build/html/* .; + touch .nojekyll; + git add -A; + git commit -m "CircleCI update of dev docs (${CIRCLE_BUILD_NUM})."; + git push origin master; + else + echo "No deployment (build: ${CIRCLE_BRANCH})."; + fi workflows: version: 2 @@ -134,5 +134,4 @@ workflows: filters: branches: only: - # - master - - doc_ci_build + - master