From c86345fe1599cb9bfdbffd32fd502ae2a503a82f Mon Sep 17 00:00:00 2001 From: Craig Date: Fri, 29 Oct 2021 13:03:01 -0700 Subject: [PATCH 1/4] Bumping pyarrow / pandas --- requirements/base.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index c8b9c69b01257..3192eeda61111 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -168,7 +168,7 @@ packaging==21.0 # via # bleach # deprecation -pandas==1.2.5 +pandas==1.3.4 # via apache-superset parsedatetime==2.6 # via apache-superset @@ -178,7 +178,7 @@ polyline==1.4.0 # via apache-superset prison==0.2.1 # via flask-appbuilder -pyarrow==4.0.1 +pyarrow==5.0.0 # via apache-superset pycparser==2.20 # via cffi diff --git a/setup.py b/setup.py index 72bc4adc2b9e2..0a049aa27e6fd 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def get_git_sha() -> str: "isodate", "markdown>=3.0", "msgpack>=1.0.0, <1.1", - "pandas>=1.2.2, <1.3", + "pandas>=1.3.0, <1.4", "parsedatetime", "pgsanity", "polyline", @@ -98,7 +98,7 @@ def get_git_sha() -> str: "python-dateutil", "python-dotenv", "python-geohash", - "pyarrow>=4.0.1, <4.1", + "pyarrow>=5.0.0, <6.0", "pyyaml>=5.4", "PyJWT>=1.7.1, <2", "redis", From 1fa7b3041c60f52ca600692e9da86322bbdf0830 Mon Sep 17 00:00:00 2001 From: Craig Date: Fri, 29 Oct 2021 13:35:59 -0700 Subject: [PATCH 2/4] Appeasing the pylint gods --- requirements/development.in | 4 +- requirements/development.txt | 6 +-- requirements/testing.txt | 73 --------------------------- superset/config.py | 4 +- superset/db_engine_specs/bigquery.py | 2 +- superset/db_engine_specs/druid.py | 6 ++- superset/db_engine_specs/snowflake.py | 2 +- superset/sql_lab.py | 2 +- superset/utils/core.py | 2 +- 9 files changed, 15 insertions(+), 86 deletions(-) diff --git a/requirements/development.in b/requirements/development.in index d53e0509f54c8..a0dae01f050b8 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -17,11 +17,11 @@ # under the License. -r base.in flask-cors>=2.0.0 -mysqlclient==1.4.2.post1 +mysqlclient==2.0.3 pillow>=8.3.1,<9 pydruid>=0.6.1,<0.7 pyhive[hive]>=0.6.1 -psycopg2-binary==2.8.5 +psycopg2-binary==2.9.1 tableschema thrift>=0.11.0,<1.0.0 progress>=1.5,<2 diff --git a/requirements/development.txt b/requirements/development.txt index 3a62a283865a7..dc993897636e7 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,4 @@ -# SHA1:dbd3e93a11a36fc6b18d6194ac96ba29bd0ad2a8 +# SHA1:a2fe77c9b8bffc8c8f3de4df6709c8be957c2f87 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -36,7 +36,7 @@ jsonlines==2.0.0 # via tabulator linear-tsv==1.1.0 # via tabulator -mysqlclient==1.4.2.post1 +mysqlclient==2.0.3 # via -r requirements/development.in openpyxl==3.0.7 # via tabulator @@ -44,7 +44,7 @@ pillow==8.3.1 # via -r requirements/development.in progress==1.6 # via -r requirements/development.in -psycopg2-binary==2.8.5 +psycopg2-binary==2.9.1 # via -r requirements/development.in pure-sasl==0.6.2 # via thrift-sasl diff --git a/requirements/testing.txt b/requirements/testing.txt index 8c542e0056d26..d89d2e255fa5c 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -17,8 +17,6 @@ astroid==2.6.6 # via pylint backcall==0.2.0 # via ipython -cachetools==4.2.4 - # via google-auth coverage==5.5 # via pytest-cov decorator==5.0.9 @@ -31,43 +29,6 @@ flask-testing==0.8.1 # via -r requirements/testing.in freezegun==1.1.0 # via -r requirements/testing.in -google-api-core[grpc]==2.1.0 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-core - # pybigquery -google-auth==2.2.1 - # via - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # pandas-gbq - # pybigquery - # pydata-google-auth -google-auth-oauthlib==0.4.6 - # via - # pandas-gbq - # pydata-google-auth -google-cloud-bigquery[bqstorage,pandas]==2.28.0 - # via - # apache-superset - # pandas-gbq - # pybigquery -google-cloud-bigquery-storage==2.9.1 - # via google-cloud-bigquery -google-cloud-core==2.1.0 - # via google-cloud-bigquery -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==2.0.3 - # via google-cloud-bigquery -googleapis-common-protos==1.53.0 - # via google-api-core -grpcio==1.41.0 - # via - # google-api-core - # google-cloud-bigquery iniconfig==1.1.1 # via pytest ipdb==0.13.9 @@ -84,22 +45,14 @@ jedi==0.18.0 # via ipython lazy-object-proxy==1.6.0 # via astroid -libcst==0.3.21 - # via google-cloud-bigquery-storage matplotlib-inline==0.1.2 # via ipython mccabe==0.6.1 # via pylint -mypy-extensions==0.4.3 - # via typing-inspect -oauthlib==3.1.1 - # via requests-oauthlib openapi-schema-validator==0.1.5 # via openapi-spec-validator openapi-spec-validator==0.3.1 # via -r requirements/testing.in -pandas-gbq==0.15.0 - # via apache-superset parameterized==0.8.1 # via -r requirements/testing.in parso==0.8.2 @@ -110,28 +63,8 @@ pickleshare==0.7.5 # via ipython prompt-toolkit==3.0.19 # via ipython -proto-plus==1.19.2 - # via - # google-cloud-bigquery - # google-cloud-bigquery-storage -protobuf==3.18.1 - # via - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # proto-plus ptyprocess==0.7.0 # via pexpect -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pybigquery==0.10.2 - # via apache-superset -pydata-google-auth==1.2.0 - # via pandas-gbq pyfakefs==4.5.0 # via -r requirements/testing.in pygments==2.9.0 @@ -151,18 +84,12 @@ pytest-cov==2.12.1 # via -r requirements/testing.in pytest-mock==3.6.1 # via -r requirements/testing.in -requests-oauthlib==1.3.0 - # via google-auth-oauthlib -rsa==4.7.2 - # via google-auth statsd==3.3.0 # via -r requirements/testing.in traitlets==5.0.5 # via # ipython # matplotlib-inline -typing-inspect==0.7.1 - # via libcst wcwidth==0.2.5 # via prompt-toolkit websocket-client==1.2.0 diff --git a/superset/config.py b/superset/config.py index e7cae127e3e9d..779a54211e72b 100644 --- a/superset/config.py +++ b/superset/config.py @@ -37,7 +37,7 @@ from dateutil import tz from flask import Blueprint from flask_appbuilder.security.manager import AUTH_DB -from pandas.io.parsers import STR_NA_VALUES +from pandas._libs.parsers import STR_NA_VALUES # pylint: disable=no-name-in-module from typing_extensions import Literal from werkzeug.local import LocalProxy @@ -1301,7 +1301,7 @@ def SQL_QUERY_MUTATOR( # pylint: disable=invalid-name,unused-argument elif importlib.util.find_spec("superset_config") and not is_test(): try: import superset_config # pylint: disable=import-error - from superset_config import * # type: ignore # pylint: disable=import-error,wildcard-import + from superset_config import * # type: ignore # pylint: disable=import-error,wildcard-import,unused-wildcard-import print(f"Loaded your LOCAL configuration at [{superset_config.__file__}]") except Exception: diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index c8785c4b549e8..c2b92a01ceeea 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -384,7 +384,7 @@ def get_parameters_from_uri( @classmethod def get_dbapi_exception_mapping(cls) -> Dict[Type[Exception], Type[Exception]]: - # pylint: disable=import-error,import-outside-toplevel + # pylint: disable=import-outside-toplevel from google.auth.exceptions import DefaultCredentialsError return {DefaultCredentialsError: SupersetDBAPIDisconnectionError} diff --git a/superset/db_engine_specs/druid.py b/superset/db_engine_specs/druid.py index 089723bc75434..8f9cd005dd803 100644 --- a/superset/db_engine_specs/druid.py +++ b/superset/db_engine_specs/druid.py @@ -57,10 +57,12 @@ class DruidEngineSpec(BaseEngineSpec): "P3M": "TIME_FLOOR(CAST({col} AS TIMESTAMP), 'P3M')", "P1Y": "TIME_FLOOR(CAST({col} AS TIMESTAMP), 'P1Y')", "P1W/1970-01-03T00:00:00Z": ( - "TIME_SHIFT(TIME_FLOOR(TIME_SHIFT(CAST({col} AS TIMESTAMP), 'P1D', 1), 'P1W'), 'P1D', 5)" # pylint: disable=line-too-long + "TIME_SHIFT(TIME_FLOOR(TIME_SHIFT(CAST({col} AS TIMESTAMP), " + "'P1D', 1), 'P1W'), 'P1D', 5)" ), "1969-12-28T00:00:00Z/P1W": ( - "TIME_SHIFT(TIME_FLOOR(TIME_SHIFT(CAST({col} AS TIMESTAMP), 'P1D', 1), 'P1W'), 'P1D', -1)" # pylint: disable=line-too-long + "TIME_SHIFT(TIME_FLOOR(TIME_SHIFT(CAST({col} AS TIMESTAMP), " + "'P1D', 1), 'P1W'), 'P1D', -1)" ), } diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index 749a61f174fd4..0310f61d99e1b 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -231,7 +231,7 @@ def get_parameters_from_uri( @classmethod def validate_parameters( - cls, parameters: SnowflakeParametersType # pylint: disable=unused-argument + cls, parameters: SnowflakeParametersType ) -> List[SupersetError]: errors: List[SupersetError] = [] required = { diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 9efebfb767ab7..27b188355a985 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -176,7 +176,7 @@ def get_sql_results( # pylint: disable=too-many-arguments return handle_query_error(ex, query, session) -def execute_sql_statement( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements +def execute_sql_statement( # pylint: disable=too-many-arguments,too-many-locals sql_statement: str, query: Query, user_name: Optional[str], diff --git a/superset/utils/core.py b/superset/utils/core.py index 81c4343cd5372..bb4dfdaf2e799 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -16,7 +16,7 @@ # under the License. """Utility functions used across Superset""" # pylint: disable=too-many-lines -import _thread # pylint: disable=C0411 +import _thread import collections import decimal import errno From 28d7bb9b5b594caf8ee812f0df097b13ef242ccb Mon Sep 17 00:00:00 2001 From: Craig Date: Fri, 29 Oct 2021 13:38:08 -0700 Subject: [PATCH 3/4] Adjusting UPDATING.md --- UPDATING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/UPDATING.md b/UPDATING.md index 3e2d5da22604e..82d9f3a35f7c3 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -26,6 +26,7 @@ assists people when migrating to a new version. ### Breaking Changes +- [17290](https://github.com/apache/superset/pull/17290): Bumps pandas to `1.3.4` and pyarrow to `5.0.0` - [16660](https://github.com/apache/incubator-superset/pull/16660): The `columns` Jinja parameter has been renamed `table_columns` to make the `columns` query object parameter available in the Jinja context. - [16711](https://github.com/apache/incubator-superset/pull/16711): The `url_param` Jinja function will now by default escape the result. For instance, the value `O'Brien` will now be changed to `O''Brien`. To disable this behavior, call `url_param` with `escape_result` set to `False`: `url_param("my_key", "my default", escape_result=False)`. From 5d2f7a3ae34cf703cf3d91a24d3f1ff4969431ef Mon Sep 17 00:00:00 2001 From: Craig Date: Fri, 29 Oct 2021 16:45:58 -0700 Subject: [PATCH 4/4] Buping testing requirements --- requirements/testing.in | 3 ++ requirements/testing.txt | 86 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/requirements/testing.in b/requirements/testing.in index 5dbb3194787f0..c172de6f26a00 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -19,13 +19,16 @@ docker flask-testing freezegun +google-cloud-bigquery ipdb # pinning ipython as pip-compile-multi was bringing higher version # of the ipython that was not found in CI ipython openapi-spec-validator openpyxl +pandas_gbq parameterized +pybigquery pyfakefs pyhive[presto]>=0.6.3 pylint==2.9.6 diff --git a/requirements/testing.txt b/requirements/testing.txt index d89d2e255fa5c..ef84e6973c4ea 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,4 +1,4 @@ -# SHA1:a36e63b551290f1060a819fe4f1f50bc6200403c +# SHA1:4aabffca9a6688f2911d6f8697495e7045a529d0 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -17,6 +17,8 @@ astroid==2.6.6 # via pylint backcall==0.2.0 # via ipython +cachetools==4.2.4 + # via google-auth coverage==5.5 # via pytest-cov decorator==5.0.9 @@ -29,6 +31,49 @@ flask-testing==0.8.1 # via -r requirements/testing.in freezegun==1.1.0 # via -r requirements/testing.in +google-api-core[grpc]==2.2.1 + # via + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-core + # pybigquery +google-auth==2.2.1 + # via + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # pandas-gbq + # pybigquery + # pydata-google-auth +google-auth-oauthlib==0.4.6 + # via + # pandas-gbq + # pydata-google-auth +google-cloud-bigquery[bqstorage,pandas]==2.29.0 + # via + # -r requirements/testing.in + # apache-superset + # pandas-gbq + # pybigquery +google-cloud-bigquery-storage==2.9.1 + # via google-cloud-bigquery +google-cloud-core==2.1.0 + # via google-cloud-bigquery +google-crc32c==1.3.0 + # via google-resumable-media +google-resumable-media==2.1.0 + # via google-cloud-bigquery +googleapis-common-protos==1.53.0 + # via + # google-api-core + # grpcio-status +grpcio==1.41.1 + # via + # google-api-core + # google-cloud-bigquery + # grpcio-status +grpcio-status==1.41.1 + # via google-api-core iniconfig==1.1.1 # via pytest ipdb==0.13.9 @@ -45,14 +90,24 @@ jedi==0.18.0 # via ipython lazy-object-proxy==1.6.0 # via astroid +libcst==0.3.21 + # via google-cloud-bigquery-storage matplotlib-inline==0.1.2 # via ipython mccabe==0.6.1 # via pylint +mypy-extensions==0.4.3 + # via typing-inspect +oauthlib==3.1.1 + # via requests-oauthlib openapi-schema-validator==0.1.5 # via openapi-spec-validator openapi-spec-validator==0.3.1 # via -r requirements/testing.in +pandas-gbq==0.15.0 + # via + # -r requirements/testing.in + # apache-superset parameterized==0.8.1 # via -r requirements/testing.in parso==0.8.2 @@ -63,8 +118,31 @@ pickleshare==0.7.5 # via ipython prompt-toolkit==3.0.19 # via ipython +proto-plus==1.19.7 + # via + # google-cloud-bigquery + # google-cloud-bigquery-storage +protobuf==3.19.1 + # via + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpcio-status + # proto-plus ptyprocess==0.7.0 # via pexpect +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pybigquery==0.10.2 + # via + # -r requirements/testing.in + # apache-superset +pydata-google-auth==1.2.0 + # via pandas-gbq pyfakefs==4.5.0 # via -r requirements/testing.in pygments==2.9.0 @@ -84,12 +162,18 @@ pytest-cov==2.12.1 # via -r requirements/testing.in pytest-mock==3.6.1 # via -r requirements/testing.in +requests-oauthlib==1.3.0 + # via google-auth-oauthlib +rsa==4.7.2 + # via google-auth statsd==3.3.0 # via -r requirements/testing.in traitlets==5.0.5 # via # ipython # matplotlib-inline +typing-inspect==0.7.1 + # via libcst wcwidth==0.2.5 # via prompt-toolkit websocket-client==1.2.0