Merge branch 'master' into feat/dashboard_dataset_access

* master: (53 commits)
  test: Adds tests to the UndoRedoKeyListeners component (#13919)
  chore: Adds dataMask reducer to reducerIndex (#13951)
  test: Tests audit for the Dashboard FilterBar (#13916)
  fix: unable to apply logging format (#14074)
  refactor: Bootstrap to AntD - Slider (#13989)
  chore(spa refactor): refactoring dashboard to use api's instead of bootstrapdata (#13306)
  fix(listview): update listview feature flag (#13906)
  Add docs for configuring Docker Compose setup (#13961)
  feat: invalid password error message (Postgres) (#14038)
  fix: flacky test in test_update_dataset_item_w_override_columns (#14082)
  feat: Implement Celery SoftTimeLimit handling (#13740)
  feat: only send alert error emails to owners of the alert (#13862)
  feat: add descriptions to report emails (#13827)
  Make chart exclude itself from cross filtering (#14046)
  fix: fix bug when remove chart not  removing it's related cross filter data (#14081)
  feat(native-filters): Add default first value to select filter (#13726)
  feat: Make async query JWT cookie domain configurable (#14007)
  fix: add exception to catch session not having JWT (#14036)
  Use consistent chart value (#14031)
  fix: Use superset generic db to catch external_metadata queries (#13974)
  ...

.github/workflows/ephemeral-env.yml

@@ -11,6 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       slash-command: ${{ steps.eval-body.outputs.result }}
+      feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
 
     steps:
     - name: Debug
@@ -28,6 +29,22 @@ jobs:
           const result = pattern.exec(context.payload.comment.body)
           return result === null ? 'noop' : result[1]
 
+    - name: Eval comment body for feature flags
+      uses: actions/github-script@v3
+      id: eval-feature-flags
+      with:
+        script: |
+          const pattern = /FEATURE_(\w+)=(\w+)/g;
+          let results = [];
+          [...context.payload.comment.body.matchAll(pattern)].forEach(match => {
+            const config = {
+              name: `SUPERSET_FEATURE_${match[1]}`,
+              value: match[2],
+            };
+            results.push(config);
+          });
+          return results;
+
     - name: Limit to committers
       if: >
         steps.eval-body.outputs.result != 'noop' &&
@@ -100,6 +117,10 @@ jobs:
         container-name: superset-ci
         image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}
 
+    - name: Update env vars in the Amazon ECS task definition
+      run: |
+        cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral_env_comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
+
     - name: Describe ECS service
       id: describe-services
       run: |

.github/workflows/superset-websocket.yml

@@ -0,0 +1,33 @@
+name: WebSocket server
+on:
+  push:
+    paths:
+      - "superset-websocket/**"
+  pull_request:
+    paths:
+      - "superset-websocket/**"
+
+jobs:
+  app-checks:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - name: Install dependencies
+        working-directory: ./superset-websocket
+        run: npm install
+      - name: lint
+        working-directory: ./superset-websocket
+        run: npm run lint
+      - name: prettier
+        working-directory: ./superset-websocket
+        run: npm run prettier-check
+      - name: unit tests
+        working-directory: ./superset-websocket
+        run: npm run test
+      - name: build
+        working-directory: ./superset-websocket
+        run: npm run build

CONTRIBUTING.md

@@ -251,9 +251,16 @@ Finally, never submit a PR that will put master branch in broken state. If the P
 
 #### Test Environments
 
-- Members of the Apache GitHub org can launch an ephemeral test environment directly on a pull request by creating a comment containing (only) the command `/testenv up`
+- Members of the Apache GitHub org can launch an ephemeral test environment directly on a pull request by creating a comment containing (only) the command `/testenv up`.
+  - Note that org membership must be public in order for this validation to function properly.
+- Feature flags may be set for a test environment by specifying the flag name (prefixed with `FEATURE_`) and value after the command.
+  - Format: `/testenv up FEATURE_<feature flag name>=true|false`
+  - Example: `/testenv up FEATURE_DASHBOARD_NATIVE_FILTERS=true`
+  - Multiple feature flags may be set in single command, separated by whitespace
 - A comment will be created by the workflow script with the address and login information for the ephemeral environment.
 - Test environments may be created once the Docker build CI workflow for the PR has completed successfully.
+- Test environments do not currently update automatically when new commits are added to a pull request.
+- Test environments do not currently support async workers, though this is planned.
 - Running test environments will be shutdown upon closing the pull request.
 
 #### Merging
@@ -435,7 +442,7 @@ superset db upgrade
 # Create default roles and permissions
 superset init
 
-# Load some data to play with
+# Load some data to play with (you must create an Admin user with the username `admin` for this command to work)
 superset load-examples
 
 # Start the Flask dev web server from inside your virtualenv.
@@ -1271,6 +1278,7 @@ The following configuration settings are available for async queries (see config
 - `GLOBAL_ASYNC_QUERIES_REDIS_STREAM_LIMIT_FIREHOSE` - the maximum number of events for all users (FIFO eviction)
 - `GLOBAL_ASYNC_QUERIES_JWT_COOKIE_NAME` - the async query feature uses a [JWT](https://tools.ietf.org/html/rfc7519) cookie for authentication, this setting is the cookie's name
 - `GLOBAL_ASYNC_QUERIES_JWT_COOKIE_SECURE` - JWT cookie secure option
+- `GLOBAL_ASYNC_QUERIES_JWT_COOKIE_DOMAIN` - JWT cookie domain option ([see docs for set_cookie](https://tedboy.github.io/flask/interface_api.response_object.html#flask.Response.set_cookie))
 - `GLOBAL_ASYNC_QUERIES_JWT_SECRET` - JWT's use a secret key to sign and validate the contents. This value should be at least 32 bytes and have sufficient randomness for proper security
 - `GLOBAL_ASYNC_QUERIES_TRANSPORT` - currently the only available option is (HTTP) `polling`, but support for a WebSocket will be added in future versions
 - `GLOBAL_ASYNC_QUERIES_POLLING_DELAY` - the time (in ms) between polling requests

UPDATING.md

@@ -25,6 +25,8 @@ assists people when migrating to a new version.
 ## Next
 - [13772](https://github.com/apache/superset/pull/13772): Row level security (RLS) is now enabled by default. To activate the feature, please run `superset init` to expose the RLS menus to Admin users.
 
+- [13980](https://github.com/apache/superset/pull/13980): Data health checks no longer use the metadata database as an interim cache. Though non-breaking, deployments which implement complex logic should likely memoize the callback function. Refer to documentation in the confg.py file for more detail.
+
 ### Breaking Changes
 ### Potential Downtime
 ### Deprecations

docs/.nvmrc

@@ -0,0 +1 @@
+v12

docs/installation.rst

@@ -1133,6 +1133,7 @@ The following configuration settings are available for async queries (see config
 - ``GLOBAL_ASYNC_QUERIES_REDIS_STREAM_LIMIT_FIREHOSE`` - the maximum number of events for all users (FIFO eviction)
 - ``GLOBAL_ASYNC_QUERIES_JWT_COOKIE_NAME`` - the async query feature uses a `JWT <https://tools.ietf.org/html/rfc7519>`_ cookie for authentication, this setting is the cookie's name
 - ``GLOBAL_ASYNC_QUERIES_JWT_COOKIE_SECURE`` - JWT cookie secure option
+- ``GLOBAL_ASYNC_QUERIES_JWT_COOKIE_DOMAIN`` - JWT cookie domain option (`see docs for set_cookie <https://tedboy.github.io/flask/interface_api.response_object.html#flask.Response.set_cookie>`
 - ``GLOBAL_ASYNC_QUERIES_JWT_SECRET`` - JWT's use a secret key to sign and validate the contents. This value should be at least 32 bytes and have sufficient randomness for proper security
 - ``GLOBAL_ASYNC_QUERIES_TRANSPORT`` - currently the only available option is (HTTP) `polling`, but support for a WebSocket will be added in future versions
 - ``GLOBAL_ASYNC_QUERIES_POLLING_DELAY`` - the time (in ms) between polling requests
@@ -1570,7 +1571,7 @@ You can enable or disable features with flag from ``superset_config.py``:
 
 .. code-block:: python
 
-     DEFAULT_FEATURE_FLAGS = {
+     FEATURE_FLAGS = {
          'CLIENT_CACHE': False,
          'ENABLE_EXPLORE_JSON_CSRF_PROTECTION': False,
          'PRESTO_EXPAND_DATA': False,

docs/sqllab.rst

@@ -155,7 +155,7 @@ can optionally specify a custom formatter. Eg:
         return [{"Cost": f"US$ {cost:.2f}"}]
 
 
-    DEFAULT_FEATURE_FLAGS = {
+    FEATURE_FLAGS = {
         "ESTIMATE_QUERY_COST": True,
         "QUERY_COST_FORMATTERS_BY_ENGINE": {"presto": presto_query_cost_formatter},
     }

docs/src/pages/docs/Miscellaneous/index.mdx

@@ -6,89 +6,57 @@ index: 1
 version: 1
 ---
 
-## Country Map Tools
+## The Country Map Visualization
 
-This tool is used in slices for visualization number or string by region, province or department of
-your countries. So, if you want to use tools, you need ISO 3166-2 code of region, province or
-department.
+The Country Map visualization allows you to plot lightweight choropleth maps of
+your countries by province, states, or other subdivision types. It does not rely
+on any third-party map services but would require you to provide the
+[ISO-3166-2](https://en.wikipedia.org/wiki/ISO_3166-2) codes of your country's
+top-level subdivisions. Comparing to a province or state's full names, the ISO
+code is less ambiguous and is unique to all regions in the world.
 
-ISO 3166-2 is part of the ISO 3166 standard published by the International Organization for
-Standardization (ISO), and defines codes for identifying the principal subdivisions (e.g., provinces
-or states) of all countries coded in ISO 3166-1
+## Included Maps
 
-The purpose of ISO 3166-2 is to establish an international standard of short and unique alphanumeric
-codes to represent the relevant administrative divisions and dependent territories of all countries
-in a more convenient and less ambiguous form than their full names. Each complete ISO 3166-2 code
-consists of two parts, separated by a hyphen:
-
-The first part is the ISO 3166-1 alpha-2 code of the country; The second part is a string of up to
-three alphanumeric characters, which is usually obtained from national sources and stems from coding
-systems already in use in the country concerned, but may also be developed by the ISO itself.
-
-We can apply these concepts to specify the state of Texas in the country of United States:
-
-```
-US-TX
-```
-
-### Included Codes
-
-The ISO codes for the following countries are included in Superset:
+The Country Maps visualization already ships with the maps for the following countries:
 
 - Belgium
+- Brazil
+- Bulgaria
+- Canada
 - China
 - Egypt
 - France
 - Germany
+- India
+- Iran
+- Italy
 - Japan
+- Korea
 - Liechtenstein
 - Morocco
+- Myanmar
+- Netherlands
+- Portugal
 - Russia
 - Singapore
 - Spain
 - Switzerland
-- United Kingdom
+- Syria
+- Thailand
+- Timorleste
+- UK
 - Ukraine
+- Uruguay
 - USA
+- Zambia
 
-### Adding New Countries
-
-To add a new country in country map tools, you need to follow the following steps:
+## Adding a New Country
 
-- You need shapefiles which contain data of your map. You can get this file on this site:
-  https://www.diva-gis.org/gdata
-- You need to add ISO 3166-2 with column name ISO for all record in your file. It’s important
-  because it’s a norm for mapping your data with geojson file
-- You need to convert shapefile to geojson file. This action can make with ogr2ogr tools:
-  https://www.gdal.org/ogr2ogr.html
-- Put your geojson file in next folder : superset-frontend/src/visualizations/CountryMap/countries
-  with the next name : nameofyourcountries.geojson
-- You can to reduce size of geojson file on this site: https://mapshaper.org/
-- Go in file `superset-frontend/src/explore/controls.jsx`
-- Add your country in component ‘select_country’. Here's an example:
+To add a new country to the list, you'd have to edit files in
+[@superset-ui/legacy-plugin-chart-country-map](https://github.com/apache-superset/superset-ui/tree/master/plugins/legacy-plugin-chart-country-map).
 
-```
-    type: 'SelectControl',
-    label: 'Country Name Type',
-    default: 'France',
-    choices: [
-    'Belgium',
-    'Brazil',
-    'China',
-    'Egypt',
-    'France',
-    'Germany',
-    'Italy',
-    'Japan',
-    'Korea',
-    'Morocco',
-    'Netherlands',
-    'Russia',
-    'Singapore',
-    'Spain',
-    'Uk',
-    'Usa',
-    ].map(s => [s, s]),
-    description: 'The name of country that Superset should display',
-},
-```
+1. Generate a new GeoJSON file for your country following the guide in [this Jupyter notebook](https://github.com/apache-superset/superset-ui/blob/master/plugins/legacy-plugin-chart-country-map/scripts/Country%20Map%20GeoJSON%20Generator.ipynb).
+2. Edit the countries list in [legacy-plugin-chart-country-map/src/countries.js](https://github.com/apache-superset/superset-ui/blob/master/plugins/legacy-plugin-chart-country-map/src/countries.js).
+3. Ping one of the Superset committers to get the `@superset-ui/legacy-plugin-chart-country-map` package published, or
+   publish it under another name yourself.
+4. Update npm dependencies in `superset-frontend/package.json` to install the updated plugin package.

docs/src/pages/docs/Miscellaneous/issue_codes.mdx

@@ -131,3 +131,21 @@ Superset encountered an unexpected error.
 
 Someething unexpected happened in the Superset backend. Please reach out
 to your administrator.
+
+## Issue 1012
+
+```
+The username provided when connecting to a database is not valid.
+```
+
+The user provided a username that doesn't exist in the database. Please check
+that the username is typed correctly and exists in the database.
+
+## Issue 1013
+
+```
+The password provided when connecting to a database is not valid.
+```
+
+The user provided a password that is incorrect. Please check that the
+password is typed correctly.

docs/src/pages/docs/installation/configuring.mdx

@@ -185,7 +185,7 @@ functionalities in Superset, but will be only affected by a subset of users.
 You can enable or disable features with flag from `superset_config.py`:
 
 ```python
-DEFAULT_FEATURE_FLAGS = {
+FEATURE_FLAGS = {
     'CLIENT_CACHE': False,
     'ENABLE_EXPLORE_JSON_CSRF_PROTECTION': False,
     'PRESTO_EXPAND_DATA': False,

docs/src/pages/docs/installation/index.mdx

@@ -32,12 +32,13 @@ part of the base Docker installation on Linux, once you have a working engine, f
 
 **Windows**
 
-Superset is not officially supported on Windows unfortunately. The best option for Windows users to
+Superset is not officially supported on Windows unfortunately. One option for Windows users to
 try out Superset locally is to install an Ubuntu Desktop VM via
 [VirtualBox](https://www.virtualbox.org/) and proceed with the Docker on Linux instructions inside
 of that VM. We recommend assigning at least 8GB of RAM to the virtual machine as well as
 provisioning a hard drive of at least 40GB, so that there will be enough space for both the OS and
-all of the required dependencies.
+all of the required dependencies. Docker Desktop [recently added support for Windows Subsystem for Linux (WSL) 2](https://docs.docker.com/docker-for-windows/wsl/), which may be another option.
+
 
 ### 2. Clone Superset's Github repository
 
@@ -59,12 +60,6 @@ Navigate to the folder you created in step 1:
 $ cd superset
 ```
 
-We recommend that you check out and run the code from the last tagged release:
-
-```bash
-$ git checkout latest
-```
-
 Then, run the following command:
 
 ```bash
@@ -74,14 +69,20 @@ $ docker-compose -f docker-compose-non-dev.yml up
 You should see a wall of logging output from the containers being launched on your machine. Once
 this output slows, you should have a running instance of Superset on your local machine!
 
-**Note:** this will bring up superset in a non-dev mode, changes to the codebase will not be reflected.
-If you would like to run superset in dev mode, simply replace the previous command with:
+**Note:** This will bring up superset in a non-dev mode, changes to the codebase will not be reflected.
+If you would like to run superset in dev mode to test local changes, simply replace the previous command with: `docker-compose up`,
+and wait for the `superset_node` container to finish building the assets.
 
-```bash
-$ docker-compose up
-```
+#### Configuring Docker Compose
+
+The following is for users who want to configure how Superset starts up in Docker Compose; otherwise, you can skip to the next section.
+
+You can configure the Docker Compose settings for dev and non-dev mode with `docker/.env` and `docker/.env-non-dev` respectively. These environment files set the environment for most containers in the Docker Compose setup, and some variables affect multiple containers and others only single ones.
+
+One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init` container will load example data and visualizations into the database and Superset. Thiese examples are quite helpful for most people, but probably unnecessary for experienced users. The loading process can sometimes take a few minutes and a good amount of CPU, so you may want to disable it on a resource-constrained device.
+
+**Note:** Users often want to connect to other databases from Superset. Currently, the easiest way to do this is to modify the `docker-compose-non-dev.yml` file and add your database as a service that the other services depend on (via `x-superset-depends-on`). Others have attempted to set `network_mode: host` on the Superset services, but these generally break the installation, because the configuration requires use of the Docker Compose DNS resolver for the service names. If you have a good solution for this, let us know!
 
-and wait for the `superset_node` container to finish building the assets.
 
 ### 4. Log in to Superset
 
@@ -99,6 +100,3 @@ username: admin
 ```bash
 password: admin
 ```
-
-Congrats! You have successfully installed Superset! Click 'Next' to learn how to connect a database
-driver.

helm/superset/templates/NOTES.txt

@@ -31,7 +31,6 @@
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "superset.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
   echo http://$SERVICE_IP:{{ .Values.service.port }}
 {{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "superset.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   echo "Visit http://127.0.0.1:8088 to use your application"
-  kubectl port-forward $POD_NAME 8088:80
+  kubectl port-forward service/superset 8088:8088 --namespace {{ .Release.Namespace }}
 {{- end }}

helm/superset/templates/deployment-beat.yaml

@@ -24,6 +24,10 @@ metadata:
     chart: {{ template "superset.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
+{{- if .Values.supersetCeleryBeat.deploymentAnnotations }}
+  annotations:
+    {{ toYaml .Values.supersetCeleryBeat.deploymentAnnotations | nindent 4 }}
+{{- end }}
 spec:
   # This must be a singleton
   replicas: 1
@@ -44,6 +48,9 @@ spec:
         # Optionally force the thing to reload
         force-reload: {{ randAlphaNum 5 | quote }}
         {{ end }}
+      {{- if .Values.supersetCeleryBeat.podAnnotations }}
+        {{ toYaml .Values.supersetCeleryBeat.podAnnotations | nindent 8 }}
+      {{- end }}
       labels:
         app: {{ template "superset.name" . }}-celerybeat
         release: {{ .Release.Name }}