[Build] Add Github workflows

.github/workflows/bashlib.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# default command to run when the `run` input is empty
+default-setup-command() {
+  pip-install
+}
+
+# install python dependencies
+pip-install() {
+  cd $GITHUB_WORKSPACE
+
+  # Don't use pip cache as it doesn't seem to help much.
+  # cache-restore pip
+
+  echo "::group::Install Python pacakges"
+  pip install -r requirements.txt
+  pip install -r requirements-dev.txt
+  pip install -e ".[postgres,mysql]"
+  echo "::endgroup::"
+
+  # cache-save pip
+}
+
+# prepare (lint and build) frontend code
+npm-install() {
+  cd $GITHUB_WORKSPACE/superset-frontend
+
+  cache-restore npm
+
+  echo "::group::Install npm packages"
+  echo "npm: $(npm --version)"
+  echo "node: $(node --version)"
+  npm ci
+  echo "::endgroup::"
+
+  cache-save npm
+}
+
+build-assets() {
+  cd $GITHUB_WORKSPACE/superset-frontend
+
+  echo "::group::Build static assets"
+  npm run build -- --no-progress
+  echo "::endgroup::"
+}
+
+npm-build() {
+  if [[ $1 = '--no-cache' ]]; then
+    build-assets
+  else
+    cache-restore assets
+    if [[ -f $GITHUB_WORKSPACE/superset/static/assets/manifest.json ]]; then
+      echo 'Skip frontend build because static assets already exist.'
+    else
+      build-assets
+      cache-save assets
+    fi
+  fi
+}
+
+cypress-install() {
+  cd $GITHUB_WORKSPACE/superset-frontend/cypress-base
+
+  cache-restore cypress
+
+  echo "::group::Install Cypress"
+  npm ci
+  echo "::endgroup::"
+
+  cache-save cypress
+}
+
+testdata() {
+  cd $GITHUB_WORKSPACE
+
+  echo "::group::Load test data"
+  superset db upgrade
+  superset load_test_users
+  superset load_examples --load-test-data
+  superset init
+  echo "::endgroup::"
+}

.github/workflows/caches.js

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// always use absolute directory
+const workspaceDirectory = process.env.GITHUB_WORKSPACE;
+const homeDirectory = process.env.HOME;
+
+// Multi-layer cache definition
+module.exports = {
+  pip: {
+    path: [`${homeDirectory}/.cache/pip`],
+    hashFiles: [`${workspaceDirectory}/requirements*.txt`],
+  },
+  npm: {
+    path: [`${homeDirectory}/.npm`],
+    hashFiles: ['superset-frontend/package-lock.json'],
+  },
+  assets: {
+    path: [
+      `${workspaceDirectory}/superset/static/assets`,
+    ],
+    hashFiles: [
+      `${workspaceDirectory}/superset-frontend/src/**/*`,
+      `${workspaceDirectory}/superset-frontend/package-lock.json`,
+    ],
+  },
+  cypress: {
+    path: [`${homeDirectory}/.cache/Cypress`],
+    hashFiles: [
+      `${workspaceDirectory}/superset-frontend/cypress-base/package-lock.json`,
+    ],
+  },
+};

.github/workflows/license-check.yml

@@ -0,0 +1,20 @@
+name: License
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Generate fossa report
+      run: ./scripts/fossa.sh
+    - name: Run license check
+      run: ./scripts/check_license.sh

.github/workflows/superset-backend.yml

@@ -0,0 +1,161 @@
+name: Backend
+
+on:
+  # only build on direct push to `master` branch
+  push:
+    branches: [ master ]
+    paths:
+    - superset/**/*.py
+    - tests/**
+    - setup.py
+    - requirements*.txt
+  # but also build on pull requests to any branch
+  # (the so-called feature branch)
+  pull_request:
+    paths:
+    - superset/**/*.py
+    - tests/**
+    - setup.py
+    - requirements*.txt
+
+jobs:
+  python-lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6]
+    env:
+      PYTHON_LINT_TARGET: setup.py superset tests
+      CI: github-actions
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Cache mypy
+      uses: actions/cache@v1
+      with:
+        path: .mypy_cache
+        key: ${{ runner.os }}-mypy-${{ hashFiles('./requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-mypy-
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: ktmud/cached-dependencies@v1
+    - name: black
+      run: black --check $(echo $PYTHON_LINT_TARGET)
+    - name: mypy
+      run: mypy $(echo $PYTHON_LINT_TARGET)
+    - name: isort
+      run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET)
+    - name: pylint
+      # `-j 0` run Pylint in parallel
+      run: pylint -j 0 superset
+
+  python-test-postgres:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # run unit tests in multiple version just for fun
+        # (3.8 is not supported yet, some dependencies need an update)
+        python-version: [3.6, 3.7]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.superset_test_config
+      REDIS_PORT: 16379
+    services:
+      postgres:
+        image: postgres:10.1-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: ktmud/cached-dependencies@v1
+    - name: Initialize database
+      run: |
+        psql "postgresql://superset:superset@127.0.0.1:15432/superset" <<- EOF
+          DROP SCHEMA IF EXISTS sqllab_test_db;
+          CREATE SCHEMA sqllab_test_db;
+          DROP SCHEMA IF EXISTS admin_database;
+          CREATE SCHEMA admin_database;
+        EOF
+    - name: Python unit tests (PostgreSQL)
+      env:
+        SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      run: |
+        ./scripts/python_tests.sh
+
+  python-test-mysql:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6]
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      SUPERSET_CONFIG: tests.superset_test_config
+      REDIS_PORT: 16379
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=5s
+          --health-timeout=2s
+          --health-retries=3
+        ports:
+          - 13306:3306
+      redis:
+        image: redis:5-alpine
+        options: --entrypoint redis-server
+        ports:
+          - 16379:6379
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: ktmud/cached-dependencies@v1
+    - name: Initialize database
+      run: |
+        mysql -h 127.0.0.1 -P 13306 -u root --password=root <<- EOF
+          DROP DATABASE IF EXISTS superset;
+          CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+          DROP DATABASE IF EXISTS sqllab_test_db;
+          CREATE DATABASE sqllab_test_db DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+          DROP DATABASE IF EXISTS admin_database;
+          CREATE DATABASE admin_database DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+          CREATE USER 'superset'@'%' IDENTIFIED BY 'superset';
+          GRANT ALL ON *.* TO 'superset'@'%';
+          FLUSH PRIVILEGES;
+        EOF
+    - name: Python unit tests (MySQL)
+      env:
+        SUPERSET__SQLALCHEMY_DATABASE_URI: |
+          mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
+      run: |
+        ./scripts/python_tests.sh
+    - name: Upload code coverage
+      run: |
+        bash <(curl -s https://codecov.io/bash)

.github/workflows/superset-e2e.yml

@@ -0,0 +1,66 @@
+name: E2E
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  cypress:
+    name: Cypress
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: ['chrome']
+    env:
+      SUPERSET_CONFIG: tests.superset_test_config
+      PYTHONPATH: ${{ github.workspace }}
+      REDIS_PORT: 16379
+      CI: github-actions
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+    services:
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+    - name: Install dependencies
+      uses: ktmud/cached-dependencies@v1
+      with:
+        run: |
+          npm-install && npm-build
+          pip-install && testdata
+          cypress-install
+    - name: Cypress run all
+      env:
+        CYPRESS_GROUP: Default
+        CYPRESS_PATH: 'cypress/integration/*/*'
+      run: |
+        flask run -p 8081 --no-debugger &
+        sleep 3 # wait for the Flask app to start
+        cd ${{ github.workspace }}/superset-frontend/cypress-base/
+        npm run cypress -- run \
+          --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}"
+        # --record --group "${{ env.CYPRESS_GROUP }}" --ci-build-id ${{ github.event_name }}-${{ github.run_id }}
+    # Follow step relies on previously installed Cypress
+    - name: Cypress run SQL Lab (with backend persist)
+      env:
+        SUPERSET_CONFIG: tests.superset_test_config_sqllab_backend_persist
+        CYPRESS_GROUP: 'Backend Persist'
+        CYPRESS_PATH: 'cypress/integration/sqllab/*'
+      run: |
+        killall python # exit the running Flask app
+        flask run -p 8081 --no-debugger &
+        sleep 3 # wait for the Flask app to start
+        cd ${{ github.workspace }}/superset-frontend/cypress-base/
+        npm run cypress -- run \
+          --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}"
+        # --record --group "${{ env.CYPRESS_GROUP }}" --ci-build-id ${{ github.event_name }}-${{ github.run_id }}