diff --git a/.circleci/Dockerfile.cypress b/.ci/Dockerfile.cypress similarity index 100% rename from .circleci/Dockerfile.cypress rename to .ci/Dockerfile.cypress diff --git a/.circleci/docker-compose.circle.yml b/.ci/docker-compose.ci.yml similarity index 100% rename from .circleci/docker-compose.circle.yml rename to .ci/docker-compose.ci.yml diff --git a/.circleci/docker-compose.cypress.yml b/.ci/docker-compose.cypress.yml similarity index 97% rename from .circleci/docker-compose.cypress.yml rename to .ci/docker-compose.cypress.yml index da57925f11..2a6feaf261 100644 --- a/.circleci/docker-compose.cypress.yml +++ b/.ci/docker-compose.cypress.yml @@ -44,7 +44,7 @@ services: ipc: host build: context: ../ - dockerfile: .circleci/Dockerfile.cypress + dockerfile: .ci/Dockerfile.cypress depends_on: - server - worker diff --git a/.circleci/docker_build b/.ci/docker_build similarity index 100% rename from .circleci/docker_build rename to .ci/docker_build diff --git a/.circleci/pack b/.ci/pack similarity index 100% rename from .circleci/pack rename to .ci/pack diff --git a/.circleci/update_version b/.ci/update_version similarity index 100% rename from .circleci/update_version rename to .ci/update_version diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 0109051768..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,179 +0,0 @@ -version: 2.0 - -build-docker-image-job: &build-docker-image-job - docker: - - image: circleci/node:14.17 - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: sudo apt update - - run: sudo apt install python3-pip - - run: .circleci/update_version - - run: sudo npm install --global --force yarn@1.22.10 - - run: .circleci/docker_build -jobs: - backend-lint: - docker: - - image: circleci/python:3.7.0 - steps: - - checkout - - run: sudo pip install flake8 - - run: ./bin/flake8_tests.sh - backend-unit-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.circle.yml - COMPOSE_PROJECT_NAME: redash - docker: - - image: circleci/buildpack-deps:xenial - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: - name: Build Docker Images - command: | - set -x - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build --build-arg skip_ds_deps=true --build-arg skip_frontend_build=true - docker-compose up -d - sleep 10 - - run: - name: Create Test Database - command: docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;" - - run: - name: List Enabled Query Runners - command: docker-compose run --rm redash manage ds list_types - - run: - name: Run Tests - command: docker-compose run --name tests redash tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ - - run: - name: Copy Test Results - command: | - mkdir -p /tmp/test-results/unit-tests - docker cp tests:/app/coverage.xml ./coverage.xml - docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - when: always - - store_test_results: - path: /tmp/test-results - - store_artifacts: - path: coverage.xml - frontend-lint: - environment: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - checkout - - run: mkdir -p /tmp/test-results/eslint - - run: sudo npm install --global --force yarn@1.22.10 - - run: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: yarn lint:ci - - store_test_results: - path: /tmp/test-results - frontend-unit-tests: - environment: - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - checkout - - run: sudo apt update - - run: sudo apt install python3-pip - - run: sudo npm install --global --force yarn@1.22.10 - - run: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: - name: Run App Tests - command: yarn test - - run: - name: Run Visualizations Tests - command: (cd viz-lib && yarn test) - - run: yarn lint - frontend-e2e-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.cypress.yml - COMPOSE_PROJECT_NAME: cypress - PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== - CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th - CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx - CYPRESS_INSTALL_BINARY: 0 - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - docker: - - image: circleci/node:14.17 - steps: - - setup_remote_docker: - version: 19.03.13 - - checkout - - run: - name: Enable Code Coverage report for master branch - command: | - if [ "$CIRCLE_BRANCH" = "master" ]; then - echo 'export CODE_COVERAGE=true' >> $BASH_ENV - source $BASH_ENV - fi - - run: sudo npm install --global --force yarn@1.22.10 - - run: - name: Install frontend dependencies - command: yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - - run: - name: Setup Redash server - command: | - yarn cypress build - yarn cypress start -- --skip-db-seed - docker-compose run cypress yarn cypress db-seed - - run: - name: Execute Cypress tests - command: yarn cypress run-ci - - run: - name: "Failure: output container logs to console" - command: | - docker-compose logs - when: on_fail - - run: - name: Copy Code Coverage results - command: | - docker cp cypress:/usr/src/app/coverage ./coverage || true - when: always - - store_artifacts: - path: coverage - build-docker-image: *build-docker-image-job - build-preview-docker-image: *build-docker-image-job -workflows: - version: 2 - build: - jobs: - - backend-lint - - backend-unit-tests: - requires: - - backend-lint - - frontend-lint - - frontend-unit-tests: - requires: - - backend-lint - - frontend-lint - - frontend-e2e-tests: - requires: - - frontend-lint - - build-preview-docker-image: - requires: - - backend-unit-tests - - frontend-unit-tests - - frontend-e2e-tests - filters: - branches: - only: - - master - - hold: - type: approval - requires: - - backend-unit-tests - - frontend-unit-tests - - frontend-e2e-tests - filters: - branches: - only: - - /release\/.*/ - - build-docker-image: - requires: - - hold diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..7d00b665b9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +name: Tests +on: + push: + branches: + - master + pull_request: +env: + NODE_VERSION: 14.17 +jobs: + backend-lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - run: sudo pip install flake8 + - run: ./bin/flake8_tests.sh + + backend-unit-tests: + runs-on: ubuntu-22.04 + needs: backend-lint + env: + COMPOSE_FILE: .ci/docker-compose.ci.yml + COMPOSE_PROJECT_NAME: redash + COMPOSE_DOCKER_CLI_BUILD: 1 + DOCKER_BUILDKIT: 1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Build Docker Images + run: | + set -x + docker-compose build --build-arg skip_ds_deps=true --build-arg skip_frontend_build=true + docker-compose up -d + sleep 10 + - name: Create Test Database + run: docker-compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;" + - name: List Enabled Query Runners + run: docker-compose -p redash run --rm redash manage ds list_types + - name: Run Tests + run: docker-compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report xml --cov=redash --cov-config .coveragerc tests/ + - name: Copy Test Results + run: | + mkdir -p /tmp/test-results/unit-tests + docker cp tests:/app/coverage.xml ./coverage.xml + docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml + - name: Store Test Results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: /tmp/test-results + - name: Store Coverage Results + uses: actions/upload-artifact@v3 + with: + name: coverage + path: coverage.xml + + frontend-lint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Run Lint + run: yarn lint:ci + - name: Store Test Results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: /tmp/test-results + + frontend-unit-tests: + runs-on: ubuntu-22.04 + needs: frontend-lint + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Run App Tests + run: yarn test + - name: Run Visualizations Tests + run: cd viz-lib && yarn test + - run: yarn lint + + frontend-e2e-tests: + runs-on: ubuntu-22.04 + needs: frontend-lint + env: + COMPOSE_FILE: .ci/docker-compose.cypress.yml + COMPOSE_PROJECT_NAME: cypress + PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA== + CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th + CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx + CYPRESS_INSTALL_BINARY: 0 + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + - name: Enable Code Coverage Report For Master Branch + if: endsWith(github.ref, '/master') + run: | + echo "CODE_COVERAGE=true" >> $GITHUB_ENV + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.10 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Setup Redash Server + run: | + set -x + yarn cypress build + yarn cypress start -- --skip-db-seed + docker-compose run cypress yarn cypress db-seed + - name: Execute Cypress Tests + run: yarn cypress run-ci + - name: "Failure: output container logs to console" + if: failure() + run: docker-compose logs + - name: Copy Code Coverage Results + run: docker cp cypress:/usr/src/app/coverage ./coverage || true + - name: Store Coverage Results + uses: actions/upload-artifact@v3 + with: + name: coverage + path: coverage diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 5fd080bb31..583346e85a 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -63,10 +63,10 @@ function runCypressCI() { PERCY_TOKEN_ENCODED, CYPRESS_PROJECT_ID_ENCODED, CYPRESS_RECORD_KEY_ENCODED, - CIRCLE_REPOSITORY_URL, + GITHUB_REPOSITORY, } = process.env; - if (CIRCLE_REPOSITORY_URL && CIRCLE_REPOSITORY_URL.includes("getredash/redash")) { + if (GITHUB_REPOSITORY === "getredash/redash") { if (PERCY_TOKEN_ENCODED) { process.env.PERCY_TOKEN = atob(`${PERCY_TOKEN_ENCODED}`); } diff --git a/requirements_dev.txt b/requirements_dev.txt index 1c32a3d30f..f169eb53a2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,6 +6,7 @@ mock==3.0.5 # PyMongo and Athena dependencies are needed for some of the unit tests: # (this is not perfect and we should resolve this in a different way) pymongo[srv,tls]==3.9.0 +boto3>=1.10.0,<1.11.0 botocore>=1.13,<1.14.0 PyAthena>=1.5.0,<=1.11.5 ptvsd==4.3.2