diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b727b729d1d5..46dcc017d171d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -927,7 +927,7 @@ To do this, you'll need to: but perfect for testing (stores cache in `/tmp`) ```python - from werkzeug.contrib.cache import FileSystemCache + from cachelib.file import FileSystemCache RESULTS_BACKEND = FileSystemCache('/tmp/sqllab') ``` diff --git a/UPDATING.md b/UPDATING.md index 046585214ef9b..60f29ebc706f8 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -23,6 +23,8 @@ assists people when migrating to a new version. ## Next +* [9786](https://github.com/apache/incubator-superset/pull/9786): with the upgrade of `werkzeug` from version `0.16.0` to `1.0.1`, the `werkzeug.contrib.cache` module has been moved to a standalone package [cachelib](https://pypi.org/project/cachelib/). For example, to import the `RedisCache` class, please use the following import: `from cachelib.redis import RedisCache`. + * [9572](https://github.com/apache/incubator-superset/pull/9572): a change which by defau;t means that the Jinja `current_user_id`, `current_username`, and `url_param` context calls no longer need to be wrapped via `cache_key_wrapper` in order to be included in the cache key. The `cache_key_wrapper` function should only be required for Jinja add-ons. * [8867](https://github.com/apache/incubator-superset/pull/8867): a change which adds the `tmp_schema_name` column to the `query` table which requires locking the table. Given the `query` table is heavily used performance may be degraded during the migration. Scheduled downtime may be advised. diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 2eb7d68a1ee1c..a2521df77f2b6 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -25,7 +25,7 @@ import logging import os -from werkzeug.contrib.cache import FileSystemCache +from cachelib.file import FileSystemCache logger = logging.getLogger() diff --git a/docs/installation.rst b/docs/installation.rst index 881d2e673b50c..90ddb319cabe3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -562,7 +562,7 @@ The connection string for PostgreSQL looks like this :: postgresql+psycopg2://{username}:{password}@{host}:{port}/{database} -Additional may be configured via the ``extra`` field under ``engine_params``. +Additional may be configured via the ``extra`` field under ``engine_params``. If you would like to enable mutual SSL here is a sample configuration: .. code-block:: json @@ -577,7 +577,7 @@ If you would like to enable mutual SSL here is a sample configuration: } } -If the key ``sslrootcert`` is present the server's certificate will be verified to be signed by the same Certificate Authority (CA). +If the key ``sslrootcert`` is present the server's certificate will be verified to be signed by the same Certificate Authority (CA). If you would like to enable mutual SSL here is a sample configuration: @@ -1056,7 +1056,7 @@ have the same configuration. celery beat --app=superset.tasks.celery_app:app To setup a result backend, you need to pass an instance of a derivative -of ``werkzeug.contrib.cache.BaseCache`` to the ``RESULTS_BACKEND`` +of ``from cachelib.base.BaseCache`` to the ``RESULTS_BACKEND`` configuration key in your ``superset_config.py``. It's possible to use Memcached, Redis, S3 (https://pypi.python.org/pypi/s3werkzeugcache), memory or the file system (in a single server-type setup or for testing), @@ -1072,7 +1072,7 @@ look something like: RESULTS_BACKEND = S3Cache(S3_CACHE_BUCKET, S3_CACHE_KEY_PREFIX) # On Redis - from werkzeug.contrib.cache import RedisCache + from cachelib.redis import RedisCache RESULTS_BACKEND = RedisCache( host='localhost', port=6379, key_prefix='superset_results') diff --git a/requirements-dev.txt b/requirements-dev.txt index a408b03ea0368..a8284018c0e15 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,20 +17,20 @@ black==19.10b0 coverage==4.5.3 flask-cors==3.0.7 -flask-testing==0.7.1 +flask-testing==0.8.0 ipdb==0.12 isort==4.3.21 mypy==0.770 nose==1.3.7 -pip-tools==4.5.1 +pip-tools==5.1.2 pre-commit==1.17.0 psycopg2-binary==2.7.5 pycodestyle==2.5.0 -pydruid==0.5.7 -pyhive==0.6.1 +pydruid==0.5.9 +pyhive==0.6.2 pylint==1.9.2 -redis==3.2.1 -requests==2.22.0 +redis==3.5.1 +requests==2.23.0 statsd==3.3.0 tox==3.11.1 pillow==7.0.0 diff --git a/requirements.txt b/requirements.txt index 21d267fc46dfb..b79d2a7c1c00c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,58 +4,60 @@ # # pip-compile --output-file=requirements.txt setup.py # -alembic==1.3.2 # via flask-migrate +alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu apispec[yaml]==1.3.3 # via flask-appbuilder attrs==19.3.0 # via jsonschema babel==2.8.0 # via flask-babel backoff==1.10.0 # via apache-superset (setup.py) -billiard==3.6.1.0 # via celery -bleach==3.1.0 # via apache-superset (setup.py) -celery==4.4.0 # via apache-superset (setup.py) -cffi==1.13.2 # via cryptography -click==7.1.1 # via apache-superset (setup.py), flask, flask-appbuilder +billiard==3.6.3.0 # via celery +bleach==3.1.5 # via apache-superset (setup.py) +brotli==1.0.7 # via flask-compress +cachelib==0.1 # via apache-superset (setup.py) +celery==4.4.2 # via apache-superset (setup.py) +cffi==1.14.0 # via cryptography +click==7.1.2 # via apache-superset (setup.py), flask, flask-appbuilder colorama==0.4.3 # via apache-superset (setup.py), flask-appbuilder contextlib2==0.6.0.post1 # via apache-superset (setup.py) croniter==0.3.31 # via apache-superset (setup.py) -cryptography==2.8 # via apache-superset (setup.py) +cryptography==2.9.2 # via apache-superset (setup.py) dataclasses==0.6 # via apache-superset (setup.py) -decorator==4.4.1 # via retry +decorator==4.4.2 # via retry defusedxml==0.6.0 # via python3-openid dnspython==1.16.0 # via email-validator -email-validator==1.0.5 # via flask-appbuilder +email-validator==1.1.0 # via flask-appbuilder flask-appbuilder==2.3.4 # via apache-superset (setup.py) flask-babel==1.0.0 # via flask-appbuilder flask-caching==1.8.0 # via apache-superset (setup.py) -flask-compress==1.4.0 # via apache-superset (setup.py) +flask-compress==1.5.0 # via apache-superset (setup.py) flask-jwt-extended==3.24.1 # via flask-appbuilder flask-login==0.4.1 # via flask-appbuilder -flask-migrate==2.5.2 # via apache-superset (setup.py) +flask-migrate==2.5.3 # via apache-superset (setup.py) flask-openid==1.2.5 # via flask-appbuilder flask-sqlalchemy==2.4.1 # via flask-appbuilder, flask-migrate flask-talisman==0.7.0 # via apache-superset (setup.py) -flask-wtf==0.14.2 # via apache-superset (setup.py), flask-appbuilder -flask==1.1.1 # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf +flask-wtf==0.14.3 # via apache-superset (setup.py), flask-appbuilder +flask==1.1.2 # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf geographiclib==1.50 # via geopy -geopy==1.21.0 # via apache-superset (setup.py) +geopy==1.22.0 # via apache-superset (setup.py) gunicorn==20.0.4 # via apache-superset (setup.py) -humanize==0.5.1 # via apache-superset (setup.py) +humanize==2.4.0 # via apache-superset (setup.py) idna==2.9 # via email-validator -importlib-metadata==1.4.0 # via jsonschema, kombu +importlib-metadata==1.6.0 # via jsonschema, kombu, markdown isodate==0.6.0 # via apache-superset (setup.py) -itsdangerous==1.1.0 # via flask -jinja2==2.10.3 # via flask, flask-babel +itsdangerous==1.1.0 # via flask, flask-wtf +jinja2==2.11.2 # via flask, flask-babel jsonschema==3.2.0 # via flask-appbuilder -kombu==4.6.7 # via celery -mako==1.1.1 # via alembic -markdown==3.1.1 # via apache-superset (setup.py) -markupsafe==1.1.1 # via jinja2, mako +kombu==4.6.8 # via celery +mako==1.1.2 # via alembic +markdown==3.2.2 # via apache-superset (setup.py) +markupsafe==1.1.1 # via jinja2, mako, wtforms marshmallow-enum==1.5.1 # via flask-appbuilder -marshmallow-sqlalchemy==0.21.0 # via flask-appbuilder -marshmallow==2.19.5 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy -more-itertools==8.1.0 # via zipp -msgpack==0.6.2 # via apache-superset (setup.py) -numpy==1.18.1 # via pandas, pyarrow +marshmallow-sqlalchemy==0.23.0 # via flask-appbuilder +marshmallow==2.21.0 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy +msgpack==1.0.0 # via apache-superset (setup.py) +numpy==1.18.4 # via pandas, pyarrow +packaging==20.3 # via bleach pandas==1.0.3 # via apache-superset (setup.py) parsedatetime==2.5 # via apache-superset (setup.py) pathlib2==2.3.5 # via apache-superset (setup.py) @@ -63,30 +65,31 @@ polyline==1.4.0 # via apache-superset (setup.py) prison==0.1.3 # via flask-appbuilder py==1.8.1 # via retry pyarrow==0.17.0 # via apache-superset (setup.py) -pycparser==2.19 # via cffi +pycparser==2.20 # via cffi pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended -pyrsistent==0.15.7 # via jsonschema +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema python-dateutil==2.8.1 # via alembic, apache-superset (setup.py), croniter, flask-appbuilder, pandas -python-dotenv==0.10.5 # via apache-superset (setup.py) +python-dotenv==0.13.0 # via apache-superset (setup.py) python-editor==1.0.4 # via alembic python-geohash==0.8.5 # via apache-superset (setup.py) python3-openid==3.1.0 # via flask-openid -pytz==2019.3 # via babel, celery, flask-babel, pandas -pyyaml==5.3 # via apache-superset (setup.py), apispec +pytz==2020.1 # via babel, celery, flask-babel, pandas +pyyaml==5.3.1 # via apache-superset (setup.py), apispec retry==0.9.2 # via apache-superset (setup.py) selenium==3.141.0 # via apache-superset (setup.py) simplejson==3.17.0 # via apache-superset (setup.py) -six==1.14.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json -sqlalchemy-utils==0.36.1 # via apache-superset (setup.py), flask-appbuilder +six==1.14.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json +sqlalchemy-utils==0.36.4 # via apache-superset (setup.py), flask-appbuilder sqlalchemy==1.3.16 # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -sqlparse==0.3.0 # via apache-superset (setup.py) -urllib3==1.25.8 # via selenium +sqlparse==0.3.1 # via apache-superset (setup.py) +urllib3==1.25.9 # via selenium vine==1.3.0 # via amqp, celery webencodings==0.5.1 # via bleach -werkzeug==0.16.0 # via flask, flask-jwt-extended +werkzeug==1.0.1 # via flask, flask-jwt-extended wtforms-json==0.3.3 # via apache-superset (setup.py) -wtforms==2.2.1 # via flask-wtf, wtforms-json -zipp==2.0.0 # via importlib-metadata +wtforms==2.3.1 # via flask-wtf, wtforms-json +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.cfg b/setup.cfg index 28a3ab3189351..e5717964201c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,7 +45,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,bleach,celery,click,colorama,contextlib2,croniter,cryptography,dataclasses,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parsedatetime,pathlib2,polyline,prison,pyarrow,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,croniter,cryptography,dataclasses,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parsedatetime,pathlib2,polyline,prison,pyarrow,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false diff --git a/setup.py b/setup.py index 3e9d1612a5d27..acf67ed95bb52 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ def get_git_sha(): install_requires=[ "backoff>=1.8.0", "bleach>=3.0.2, <4.0.0", + "cachelib>=0.1,<0.2", "celery>=4.3.0, <5.0.0, !=4.4.1", "click<8", "colorama", @@ -88,7 +89,7 @@ def get_git_sha(): "humanize", "isodate", "markdown>=3.0", - "msgpack>=0.6.1, <0.7.0", + "msgpack>=1.0.0, <1.1", "pandas>=1.0.3, <1.1", "parsedatetime", "pathlib2", @@ -102,7 +103,10 @@ def get_git_sha(): "selenium>=3.141.0", "simplejson>=3.15.0", "sqlalchemy>=1.3.16, <2.0", - "sqlalchemy-utils>=0.33.2", + # Breaking change in sqlalchemy-utils==0.36.6, upgrading will probably + # require a migration on EncryptedType columns. For more information, see + # https://github.com/kvesteri/sqlalchemy-utils/issues/444 + "sqlalchemy-utils>=0.33.2,<0.36.5", "sqlparse>=0.3.0, <0.4", "wtforms-json", ], @@ -130,5 +134,5 @@ def get_git_sha(): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", ], - tests_require=["flask-testing==0.7.1"], + tests_require=["flask-testing==0.8.0"], ) diff --git a/superset/config.py b/superset/config.py index 19de94a0561ce..42cd703e7c442 100644 --- a/superset/config.py +++ b/superset/config.py @@ -568,7 +568,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods Callable[["Database", "models.User", str, str], str] ] = None -# An instantiated derivative of werkzeug.contrib.cache.BaseCache +# An instantiated derivative of cachelib.base.BaseCache # if enabled, it can be used to store the results of long-running queries # in SQL Lab by using the "Run Async" button/feature RESULTS_BACKEND = None diff --git a/tests/sql_parse_tests.py b/tests/sql_parse_tests.py index 1722d6305744a..b7cc760ca6134 100644 --- a/tests/sql_parse_tests.py +++ b/tests/sql_parse_tests.py @@ -187,8 +187,7 @@ def test_select_if(self): # SHOW TABLES ((FROM | IN) qualifiedName)? (LIKE pattern=STRING)? def test_show_tables(self): query = "SHOW TABLES FROM s1 like '%order%'" - # TODO: figure out what should code do here - self.assertEqual({Table("s1")}, self.extract_tables(query)) + self.assertEqual(set(), self.extract_tables(query)) # SHOW COLUMNS (FROM | IN) qualifiedName def test_show_columns(self): diff --git a/tests/superset_test_config_thumbnails.py b/tests/superset_test_config_thumbnails.py index 5a88a045e99b1..86f1de9bd0892 100644 --- a/tests/superset_test_config_thumbnails.py +++ b/tests/superset_test_config_thumbnails.py @@ -17,8 +17,8 @@ # type: ignore from copy import copy +from cachelib.redis import RedisCache from flask import Flask -from werkzeug.contrib.cache import RedisCache from superset.config import * # type: ignore diff --git a/tests/thumbnails_tests.py b/tests/thumbnails_tests.py index 305f314f8d651..d2b4c5bae955c 100644 --- a/tests/thumbnails_tests.py +++ b/tests/thumbnails_tests.py @@ -42,7 +42,7 @@ class CeleryStartMixin: @classmethod def setUpClass(cls): with app.app_context(): - from werkzeug.contrib.cache import RedisCache + from cachelib.redis import RedisCache class CeleryConfig(object): BROKER_URL = "redis://localhost"