diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f68080d05bea6..268b3f9693be2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,3 +95,310 @@ jobs: name: Benchmarks log path: asv_bench/benchmarks.log if: failure() + + - name: Create build output artifact + run: | + echo $STEPS_CONTEXT > build_output.json + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + if: job.status == 'failure' + + - name: Upload build output artifact + uses: actions/upload-artifact@master + with: + name: build_output_checks + path: build_output.json + if: job.status == 'failure' + + web_and_docs: + name: Web and docs + runs-on: ubuntu-latest + steps: + + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + + - name: Checkout + uses: actions/checkout@v1 + + - name: Setup environment and build pandas + run: ci/setup_env.sh + + - name: Build website + run: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + + - name: Build documentation + run: | + source activate pandas-dev + doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + + # This can be removed when the ipython directive fails when there are errors, + # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) + - name: Check ipython directive errors + run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" + + - name: Merge website and docs + run: | + mkdir -p pandas_web/docs + cp -r web/build/* pandas_web/ + cp -r doc/build/html/* pandas_web/docs/ + + - name: Create artifact + run: | + tar -czvf pandas_web.tar.gz pandas_web + + - name: Upload artifact + uses: actions/upload-artifact@master + with: + name: pandas_web + path: pandas_web.tar.gz + + - name: Install rclone + run: sudo apt install rclone -y + + - name: Set up Rclone + run: | + RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf + mkdir -p `dirname $RCLONE_CONFIG_PATH` + echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH + echo "type = swift" >> $RCLONE_CONFIG_PATH + echo "env_auth = false" >> $RCLONE_CONFIG_PATH + echo "auth_version = 3" >> $RCLONE_CONFIG_PATH + echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH + echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH + echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH + echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH + echo "domain = default" >> $RCLONE_CONFIG_PATH + echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH + echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH + echo "region = BHS" >> $RCLONE_CONFIG_PATH + + - name: Sync web + run: | + REMOTE_PATH="dev" + if [[ "${{ github.event_name }}" == "pr" ]]; then + REMOTE_PATH="$REMOTE_PATH/pr/$PR_NUMBER" + fi + rclone sync pandas_web ovh_cloud_pandas_web:$REMOTE_PATH + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + + - name: Create build output artifact + run: | + echo $STEPS_CONTEXT > build_output.json + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + if: job.status == 'failure' + + - name: Upload build output artifact + uses: actions/upload-artifact@master + with: + name: build_output_web_and_docs + path: build_output.json + if: job.status == 'failure' + + tests: + name: Tests + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + name: [py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np] + include: + + # regular tests + + - name: py37 + os: ubuntu-latest + env_file: travis-37.yaml + pattern: "not slow and not network" + + - name: py37-locale + os: ubuntu-latest + env_file: azure-37-locale.yaml + pattern: "not slow and not network" + locale_override: zh_CN.UTF-8 + + - name: py36-minimum-versions + os: ubuntu-latest + env_file: azure-36-minimum_versions.yaml + pattern: "not slow and not network" + + - name: py36-macos + os: macos-latest + env_file: azure-macos-36.yaml + pattern: "not slow and not network" + + - name: py36-32bits + os: ubuntu-latest + env_file: azure-36-32bit.yaml + pattern: "not slow and not network" + bits32: "yes" + + - name: py36-locale-slow # TODO: this is not slow, see pattern + os: ubuntu-latest + env_file: azure-36-locale_slow.yaml + pattern: "not slow and not network" + locale_override: it_IT.UTF-8 + + - name: py36-np15-windows + os: windows-latest + env_file: azure-windows-36.yaml + pattern: "not slow and not network" + + - name: py37-np141-windows + os: windows-latest + env_file: azure-windows-37.yaml + pattern: "not slow and not network" + + # TODO: This is commented in azure - https://github.com/pandas-dev/pandas/issues/29432 + - name: py37-numpydev + os: ubuntu-latest + env_file: azure-37-numpydev.yaml + pattern: "not slow and not network" + pandas_testing_mode: deprecate + # TODO env vars pending to add and use in the workflow + # TEST_ARGS: "-W error" + # EXTRA_APT: "xsel" + + # slow tests + + - name: py36-locale-slow-old-np + os: ubuntu-latest + env_file: azure-36-locale.yaml + pattern: "slow" + locale_override: zh_CN.UTF-8 + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + + - name: Setup environment and build pandas + shell: bash + run: ci/setup_env.sh + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + BITS32: ${{ matrix.bits32 }} + + - name: Run tests + shell: bash + run: | + source activate pandas-dev + ci/run_tests.sh + env: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + db_tests: + name: DB Tests + runs-on: ${{ matrix.os }} + + services: + postgres: + image: postgres:latest + ports: + - 5432/tcp + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + mysql: + image: mysql:latest + ports: + - 3306/tcp + env: + MYSQL_ROOT_PASSWORD: password + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + name: [py36-cov, py36-locale-zh] + include: + + - name: py36-cov + os: ubuntu-latest + env_file: travis-36-cov.yaml + pattern: "(not slow and not network) or (single and db)" + pandas_testing_mode: deprecate + coverage: true + + - name: py36-locale-zh + os: ubuntu-latest + env_file: travis-36-locale.yaml + pattern: "(not slow and not network) or (single and db)" + locale_override: zh_CN.UTF-8 + + steps: + - name: Setting conda path + run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}" + + - name: Checkout + uses: actions/checkout@v1 + + - name: Create PostgreSQL database + run: | + sudo apt-get install -y postgresql-client + psql --host 127.0.0.1 \ + --port ${{ job.services.postgres.ports[5432] }} \ + --username postgres \ + --command 'CREATE DATABASE pandas_nosetest;' + + - name: Create MySQL database + run: | + mysql --host 127.0.0.1 \ + --port ${{ job.services.mysql.ports[3306] }} \ + --user root \ + -ppassword \ + --execute "CREATE DATABASE pandas_nosetest;" + + - name: Setup environment and build pandas + run: | + ci/setup_env.sh + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + - name: Run tests + run: | + source activate pandas-dev + ci/run_tests.sh + env: + PATTERN: ${{ matrix.pattern }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }} + COVERAGE: ${{ matrix.coverage }} + LOCALE_OVERRIDE: ${{ matrix.locale_override }} + + + finalize: + name: Finalize build + runs-on: ubuntu-latest + if: true + # TODO run when all the other jobs finish: + needs: [checks, web_and_docs] + steps: + + - name: Download checks artifact + uses: actions/download-artifact@master + with: + name: build_output_checks + path: build_output_checks.json + + - name: Extract errors from checks artifact + if: true # TODO temporary for debugging + run: | + export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:" + # TODO get actual errors + export MSG="$MSG\n- [Actual messages from the log are not yet extracted](https://github.com/pandas-dev/pandas/actions)" + echo "{\"body\": \"$MSG\"}" > payload.json + + - name: + run: | + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$( /dev/null 2>&1 - echo "Using ccache" - export PATH=/usr/local/opt/ccache/libexec:$PATH - gcc=$(which gcc) - echo "gcc: $gcc" - CCACHE=$(which ccache) - echo "ccache: $CCACHE" -else - echo "Not using ccache" -fi - echo "source deactivate" source deactivate @@ -139,13 +119,4 @@ echo echo "conda list" conda list -# Install DB for Linux -if [ "${TRAVIS_OS_NAME}" == "linux" ]; then - echo "installing dbs" - mysql -e 'create database pandas_nosetest;' - psql -c 'create database pandas_nosetest;' -U postgres -else - echo "not using dbs on non-linux Travis builds or Azure Pipelines" -fi - echo "done"