From 139cd495e1b825fc405f938b981fcffdd3394b00 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 19 May 2021 15:30:21 -0400 Subject: [PATCH 01/24] save bg form --- superset/db_engine_specs/bigquery.py | 64 ++++++++++++++++++++++++++++ tests/databases/api_tests.py | 41 ++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 9c52b44eca7b3..2e0cc8f98226d 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -19,9 +19,13 @@ from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING import pandas as pd +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin from flask_babel import gettext as __ +from marshmallow import fields, Schema from sqlalchemy import literal_column from sqlalchemy.sql.expression import ColumnClause +from typing_extensions import TypedDict from superset.db_engine_specs.base import BaseEngineSpec from superset.errors import SupersetErrorType @@ -38,6 +42,32 @@ + "permission in project (?P.+?)" ) +# TODO: Fill in descriptions of each field +class BigQueryParametersSchema(Schema): + type = fields.String(required=True, description=__("")) + project_id = fields.String(required=True, description=__("")) + private_key_id = fields.String(required=True, description=__("")) + private_key = fields.String(required=True, description=__("")) + client_email = fields.String(required=True, description=__("")) + client_id = fields.String(required=True, description=__("")) + auth_uri = fields.String(required=True, description=__("")) + token_uri = fields.String(required=True, description=__("")) + auth_provider_x509_cert_url = fields.String(required=True, description=__("")) + client_x509_cert_url = fields.String(required=True, description=__("")) + + +class BigQueryParametersType(TypedDict): + type: str + project_id: str + private_key_id: str + private_key: str + client_email: str + client_id: str + auth_uri: str + token_uri: str + auth_provider_x509_cert_url: str + client_x509_cert_url: str + class BigQueryEngineSpec(BaseEngineSpec): """Engine spec for Google's BigQuery @@ -48,6 +78,10 @@ class BigQueryEngineSpec(BaseEngineSpec): engine_name = "Google BigQuery" max_column_name_length = 128 + parameters_schema = BigQueryParametersSchema() + drivername = engine + sqlalchemy_uri_placeholder = "bigquery://{project_id}" + # BigQuery doesn't maintain context when running multiple statements in the # same cursor, so we need to run all statements at once run_multiple_statements_as_one = True @@ -282,3 +316,33 @@ def df_to_sql( to_gbq_kwargs[key] = to_sql_kwargs[key] pandas_gbq.to_gbq(df, **to_gbq_kwargs) + + @classmethod + def build_sqlalchemy_url(cls, parameters: BigQueryParametersType) -> str: + project_id = parameters.get("project_id") + return f"{cls.drivername}://{project_id}" + + @classmethod + def get_parameters_from_uri(cls, uri: str) -> Any: + # We might need to add a special case for bigquery since + # we are relying on the json credentials + return {"foo": "bar"} + + @classmethod + def parameters_json_schema(cls) -> Any: + """ + Return configuration parameters as OpenAPI. + """ + if not cls.parameters_schema: + return None + + print(cls.parameters_schema) + + spec = APISpec( + title="Database Parameters", + version="1.0.0", + openapi_version="3.0.2", + plugins=[MarshmallowPlugin()], + ) + spec.components.schema(cls.__name__, schema=cls.parameters_schema) + return spec.to_dict()["components"]["schemas"][cls.__name__] diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 7f53c579db3cc..c631c4364c4d4 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -36,6 +36,7 @@ from superset.connectors.sqla.models import SqlaTable from superset.db_engine_specs.mysql import MySQLEngineSpec from superset.db_engine_specs.postgres import PostgresEngineSpec +from superset.db_engine_specs.bigquery import BigQueryEngineSpec from superset.db_engine_specs.hana import HanaEngineSpec from superset.errors import SupersetError from superset.models.core import Database, ConfigurationMethod @@ -1373,6 +1374,7 @@ def test_available(self, app, get_available_engine_specs): app.config = {"PREFERRED_DATABASES": ["postgresql"]} get_available_engine_specs.return_value = [ PostgresEngineSpec, + BigQueryEngineSpec, HanaEngineSpec, ] @@ -1428,6 +1430,45 @@ def test_available(self, app, get_available_engine_specs): "preferred": True, "sqlalchemy_uri_placeholder": "postgresql+psycopg2://user:password@host:port/dbname[?key=value&key=value...]", }, + { + "engine": "bigquery", + "name": "Google BigQuery", + "parameters": { + "properties": { + "auth_provider_x509_cert_url": { + "description": "", + "type": "string", + }, + "auth_uri": {"description": "", "type": "string",}, + "client_email": {"description": "", "type": "string",}, + "client_id": {"description": "", "type": "string",}, + "client_x509_cert_url": { + "description": "", + "type": "string", + }, + "private_key": {"description": "", "type": "string",}, + "private_key_id": {"description": "", "type": "string",}, + "project_id": {"description": "", "type": "string",}, + "token_uri": {"description": "", "type": "string",}, + "type": {"description": "", "type": "string",}, + }, + "required": [ + "auth_provider_x509_cert_url", + "auth_uri", + "client_email", + "client_id", + "client_x509_cert_url", + "private_key", + "private_key_id", + "project_id", + "token_uri", + "type", + ], + "type": "object", + }, + "preferred": True, + "sqlalchemy_uri_placeholder": "bigquery://{project_id}", + }, {"engine": "hana", "name": "SAP HANA", "preferred": False}, ] } From f252d72b3d10f228f09dc5c61e2c2172beedecf0 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 19 May 2021 15:39:49 -0400 Subject: [PATCH 02/24] remove prints --- superset/db_engine_specs/bigquery.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 2e0cc8f98226d..8c21b8583d2fa 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -336,8 +336,6 @@ def parameters_json_schema(cls) -> Any: if not cls.parameters_schema: return None - print(cls.parameters_schema) - spec = APISpec( title="Database Parameters", version="1.0.0", From b77a3e59da4a150e2d909e0fcf7df55d6c44a9ed Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 13:38:26 -0400 Subject: [PATCH 03/24] add fe bigquery form --- .../src/common/components/index.tsx | 1 + .../data/database/DatabaseModal/index.tsx | 43 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/superset-frontend/src/common/components/index.tsx b/superset-frontend/src/common/components/index.tsx index bf17aac743291..c0cf44adb99e0 100644 --- a/superset-frontend/src/common/components/index.tsx +++ b/superset-frontend/src/common/components/index.tsx @@ -54,6 +54,7 @@ export { Tag, Tabs, Tooltip, + Upload, Input as AntdInput, } from 'antd'; export { Card as AntdCard } from 'antd'; diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx index 4547f8bc0d4b0..f4d8d1460c208 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx @@ -17,6 +17,7 @@ * under the License. */ import { t } from '@superset-ui/core'; +import { Upload, Button } from 'antd'; import React, { FunctionComponent, useEffect, @@ -25,7 +26,7 @@ import React, { Reducer, } from 'react'; import Tabs from 'src/components/Tabs'; -import { Alert } from 'src/common/components'; +import { Alert, Select, Upload, Button } from 'src/common/components'; import withToasts from 'src/messageToasts/enhancers/withToasts'; import { testDatabaseConnection, @@ -37,6 +38,7 @@ import ExtraOptions from './ExtraOptions'; import SqlAlchemyForm from './SqlAlchemyForm'; import { StyledBasicTab, + StyledJsonEditor, StyledModal, EditHeader, EditHeaderTitle, @@ -50,6 +52,8 @@ import { const DOCUMENTATION_LINK = 'https://superset.apache.org/docs/databases/installing-database-drivers'; +const { Option } = Select; + interface DatabaseModalProps { addDangerToast: (msg: string) => void; addSuccessToast: (msg: string) => void; @@ -148,10 +152,11 @@ const DatabaseModal: FunctionComponent = ({ Reducer | null, DBReducerActionType> >(dbReducer, null); const [tabKey, setTabKey] = useState(DEFAULT_TAB_KEY); + const [bigQueryOption, setBigQueryOption] = useState('upload'); const conf = useCommonConf(); const isEditMode = !!databaseId; - const useSqlAlchemyForm = true; // TODO: set up logic + const useSqlAlchemyForm = false; // TODO: set up logic const hasConnectedDb = false; // TODO: set up logic // Database fetch logic @@ -266,6 +271,39 @@ const DatabaseModal: FunctionComponent = ({ setTabKey(key); }; + const BigQueryForm = () => ( + <> + + {bigQueryOption === 'paste' ? ( +
+ { + console.log(json); + }} + width="100%" + height="160px" + /> +
+ ) : ( + + + + )} + + ); + return isEditMode || useSqlAlchemyForm ? ( = ({ >

TODO: db form

+
); From bb24664f456f32a2aaa06a54653a3ddbb0ffebb4 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 15:54:54 -0400 Subject: [PATCH 04/24] update api --- superset/databases/commands/validate.py | 3 +- superset/databases/schemas.py | 6 +++- superset/db_engine_specs/base.py | 6 +++- superset/db_engine_specs/bigquery.py | 43 +++++++++---------------- superset/models/core.py | 2 ++ tests/db_engine_specs/postgres_tests.py | 5 ++- 6 files changed, 34 insertions(+), 31 deletions(-) diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index e0621020a4fa9..beb028be3d0d8 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -83,7 +83,8 @@ def run(self) -> None: # try to connect sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( - self._properties["parameters"] # type: ignore + self._properties["parameters"], # type: ignore + self._properties["encrypted_extra"], ) if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri(): sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index a23659fc0fbf8..74b7af441ef1d 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -246,6 +246,8 @@ def build_sqlalchemy_uri( the constructed SQLAlchemy URI to be passed. """ parameters = data.pop("parameters", None) + encrypted_extra = data.get("encrypted_extra", None) + if parameters: if "engine" not in parameters: raise ValidationError( @@ -275,7 +277,9 @@ def build_sqlalchemy_uri( ] ) - data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri(parameters) + data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( + parameters, encrypted_extra + ) return data diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index fe0e434553bb7..97321b8b9a70a 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -1348,7 +1348,11 @@ class BasicParametersMixin: encryption_parameters: Dict[str, str] = {} @classmethod - def build_sqlalchemy_uri(cls, parameters: BasicParametersType) -> str: + def build_sqlalchemy_uri( + cls, + parameters: BasicParametersType, + encryted_extra: Optional[Dict[str, str]] = None, + ) -> str: query = parameters.get("query", {}) if parameters.get("encryption"): if not cls.encryption_parameters: diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 8c21b8583d2fa..b149467090ab7 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -29,6 +29,7 @@ from superset.db_engine_specs.base import BaseEngineSpec from superset.errors import SupersetErrorType +from superset.exceptions import SupersetGenericDBErrorException from superset.sql_parse import Table from superset.utils import core as utils from superset.utils.hashing import md5_sha_from_str @@ -44,29 +45,11 @@ # TODO: Fill in descriptions of each field class BigQueryParametersSchema(Schema): - type = fields.String(required=True, description=__("")) - project_id = fields.String(required=True, description=__("")) - private_key_id = fields.String(required=True, description=__("")) - private_key = fields.String(required=True, description=__("")) - client_email = fields.String(required=True, description=__("")) - client_id = fields.String(required=True, description=__("")) - auth_uri = fields.String(required=True, description=__("")) - token_uri = fields.String(required=True, description=__("")) - auth_provider_x509_cert_url = fields.String(required=True, description=__("")) - client_x509_cert_url = fields.String(required=True, description=__("")) + pass class BigQueryParametersType(TypedDict): - type: str - project_id: str - private_key_id: str - private_key: str - client_email: str - client_id: str - auth_uri: str - token_uri: str - auth_provider_x509_cert_url: str - client_x509_cert_url: str + pass class BigQueryEngineSpec(BaseEngineSpec): @@ -318,15 +301,21 @@ def df_to_sql( pandas_gbq.to_gbq(df, **to_gbq_kwargs) @classmethod - def build_sqlalchemy_url(cls, parameters: BigQueryParametersType) -> str: - project_id = parameters.get("project_id") - return f"{cls.drivername}://{project_id}" + def build_sqlalchemy_uri( + cls, _: BigQueryParametersType, encrypted_extra: Optional[Dict[str, str]] = None + ) -> str: + if encrypted_extra: + project_id = encrypted_extra.get("project_id") + return f"{cls.drivername}://{project_id}" + + raise SupersetGenericDBErrorException( + message="Big Query encrypted_extra is not available", + ) @classmethod - def get_parameters_from_uri(cls, uri: str) -> Any: - # We might need to add a special case for bigquery since - # we are relying on the json credentials - return {"foo": "bar"} + def get_parameters_from_uri(cls, _: str) -> Any: + # BigQuery doesn't have parameters + return None @classmethod def parameters_json_schema(cls) -> Any: diff --git a/superset/models/core.py b/superset/models/core.py index 1868af6c59e09..3889f41a84f9a 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -55,6 +55,7 @@ from superset import app, db_engine_specs, is_feature_enabled from superset.db_engine_specs.base import TimeGrain +from superset.db_engine_specs.bigquery import BigQueryEngineSpec from superset.extensions import cache_manager, encrypted_field_factory, security_manager from superset.models.helpers import AuditMixinNullable, ImportExportMixin from superset.models.tags import FavStarUpdater @@ -241,6 +242,7 @@ def backend(self) -> str: def parameters(self) -> Dict[str, Any]: # Build parameters if db_engine_spec is a subclass of BasicParametersMixin parameters = {"engine": self.backend} + if hasattr(self.db_engine_spec, "parameters_schema") and hasattr( self.db_engine_spec, "get_parameters_from_uri" ): diff --git a/tests/db_engine_specs/postgres_tests.py b/tests/db_engine_specs/postgres_tests.py index 6f6fa5451e39d..135621b5e5c9b 100644 --- a/tests/db_engine_specs/postgres_tests.py +++ b/tests/db_engine_specs/postgres_tests.py @@ -432,7 +432,10 @@ def test_base_parameters_mixin(): "query": {"foo": "bar"}, "encryption": True, } - sqlalchemy_uri = PostgresEngineSpec.build_sqlalchemy_uri(parameters) + encrypted_extra = None + sqlalchemy_uri = PostgresEngineSpec.build_sqlalchemy_uri( + parameters, encrypted_extra + ) assert sqlalchemy_uri == ( "postgresql+psycopg2://username:password@localhost:5432/dbname?" "foo=bar&sslmode=verify-ca" From 4a1683e2b74a5b1b2267912bd5e985caeab7a4b8 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 18:07:32 -0400 Subject: [PATCH 05/24] overriding credentials info json --- superset/databases/api.py | 8 +++++--- superset/db_engine_specs/bigquery.py | 11 +++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/superset/databases/api.py b/superset/databases/api.py index 3f506e0ed5649..b5b16ffd432b7 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -909,11 +909,13 @@ def available(self) -> Response: "preferred": engine_spec.engine in preferred_databases, } - if issubclass(engine_spec, BasicParametersMixin): - payload["parameters"] = engine_spec.parameters_json_schema() + if hasattr(engine_spec, "parameters_json_schema") or hasattr( + engine_spec, "sqlalchemy_uri_placeholder" + ): + payload["parameters"] = engine_spec.parameters_json_schema() # type: ignore payload[ "sqlalchemy_uri_placeholder" - ] = engine_spec.sqlalchemy_uri_placeholder + ] = engine_spec.sqlalchemy_uri_placeholder # type: ignore available_databases.append(payload) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index b149467090ab7..9ec8c08db93f5 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -43,9 +43,9 @@ + "permission in project (?P.+?)" ) -# TODO: Fill in descriptions of each field + class BigQueryParametersSchema(Schema): - pass + credentials_info = fields.String(description="credentials.json file for BigQuery") class BigQueryParametersType(TypedDict): @@ -328,8 +328,11 @@ def parameters_json_schema(cls) -> Any: spec = APISpec( title="Database Parameters", version="1.0.0", - openapi_version="3.0.2", + openapi_version="3.0.0", plugins=[MarshmallowPlugin()], ) spec.components.schema(cls.__name__, schema=cls.parameters_schema) - return spec.to_dict()["components"]["schemas"][cls.__name__] + + schemas = spec.to_dict()["components"]["schemas"][cls.__name__] + schemas["properties"]["credentials_info"]["type"] = "encryption_extra" + return schemas From 1fc08d47fb8c4c9d2a62a60a3855fb3e08c50be2 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 21:35:36 -0400 Subject: [PATCH 06/24] add custom field to big query schema --- superset/db_engine_specs/bigquery.py | 29 ++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 9ec8c08db93f5..8199a4836b7d6 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -21,6 +21,7 @@ import pandas as pd from apispec import APISpec from apispec.ext.marshmallow import MarshmallowPlugin +from apispec.ext.marshmallow.openapi import OpenAPIConverter from flask_babel import gettext as __ from marshmallow import fields, Schema from sqlalchemy import literal_column @@ -43,9 +44,25 @@ + "permission in project (?P.+?)" ) +ma_plugin = MarshmallowPlugin() + + +class EncryptedField(fields.String): + pass + class BigQueryParametersSchema(Schema): - credentials_info = fields.String(description="credentials.json file for BigQuery") + credentials_info = EncryptedField(description="credentials.json file for BigQuery") + + +def encrypted_field_properties( + self: OpenAPIConverter, field: EncryptedField +) -> Dict[str, Any]: + ret = {} + if isinstance(field, EncryptedField): + if self.openapi_version.major > 2: + ret["x-encrypted"] = True + return ret class BigQueryParametersType(TypedDict): @@ -329,10 +346,10 @@ def parameters_json_schema(cls) -> Any: title="Database Parameters", version="1.0.0", openapi_version="3.0.0", - plugins=[MarshmallowPlugin()], + plugins=[ma_plugin], ) - spec.components.schema(cls.__name__, schema=cls.parameters_schema) - schemas = spec.to_dict()["components"]["schemas"][cls.__name__] - schemas["properties"]["credentials_info"]["type"] = "encryption_extra" - return schemas + ma_plugin.init_spec(spec) + ma_plugin.converter.add_attribute_function(encrypted_field_properties) + spec.components.schema(cls.__name__, schema=cls.parameters_schema) + return spec.to_dict()["components"]["schemas"][cls.__name__] From b8640a1f38690eca5b6bf3c30abf150221ce6f29 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 21:48:41 -0400 Subject: [PATCH 07/24] tis works --- superset/db_engine_specs/bigquery.py | 5 +---- tests/databases/api_tests.py | 33 +++++----------------------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 8199a4836b7d6..18da59d40f33f 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -21,7 +21,6 @@ import pandas as pd from apispec import APISpec from apispec.ext.marshmallow import MarshmallowPlugin -from apispec.ext.marshmallow.openapi import OpenAPIConverter from flask_babel import gettext as __ from marshmallow import fields, Schema from sqlalchemy import literal_column @@ -55,9 +54,7 @@ class BigQueryParametersSchema(Schema): credentials_info = EncryptedField(description="credentials.json file for BigQuery") -def encrypted_field_properties( - self: OpenAPIConverter, field: EncryptedField -) -> Dict[str, Any]: +def encrypted_field_properties(self, field, **kwargs): ret = {} if isinstance(field, EncryptedField): if self.openapi_version.major > 2: diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index c631c4364c4d4..2676f5ee116e4 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -1435,38 +1435,15 @@ def test_available(self, app, get_available_engine_specs): "name": "Google BigQuery", "parameters": { "properties": { - "auth_provider_x509_cert_url": { - "description": "", + "credentials_info": { + "description": "credentials.json file for BigQuery", "type": "string", - }, - "auth_uri": {"description": "", "type": "string",}, - "client_email": {"description": "", "type": "string",}, - "client_id": {"description": "", "type": "string",}, - "client_x509_cert_url": { - "description": "", - "type": "string", - }, - "private_key": {"description": "", "type": "string",}, - "private_key_id": {"description": "", "type": "string",}, - "project_id": {"description": "", "type": "string",}, - "token_uri": {"description": "", "type": "string",}, - "type": {"description": "", "type": "string",}, + "x-encrypted": True, + } }, - "required": [ - "auth_provider_x509_cert_url", - "auth_uri", - "client_email", - "client_id", - "client_x509_cert_url", - "private_key", - "private_key_id", - "project_id", - "token_uri", - "type", - ], "type": "object", }, - "preferred": True, + "preferred": False, "sqlalchemy_uri_placeholder": "bigquery://{project_id}", }, {"engine": "hana", "name": "SAP HANA", "preferred": False}, From 8ca9b803b3f80c9cf030454d4fc2e03abf8623d5 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 21:50:29 -0400 Subject: [PATCH 08/24] update --- superset/db_engine_specs/bigquery.py | 2 +- tests/databases/api_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 18da59d40f33f..243b87b13373f 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -58,7 +58,7 @@ def encrypted_field_properties(self, field, **kwargs): ret = {} if isinstance(field, EncryptedField): if self.openapi_version.major > 2: - ret["x-encrypted"] = True + ret["x-encrypted-extra"] = True return ret diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 2676f5ee116e4..bed4c7be6a921 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -1438,7 +1438,7 @@ def test_available(self, app, get_available_engine_specs): "credentials_info": { "description": "credentials.json file for BigQuery", "type": "string", - "x-encrypted": True, + "x-encrypted-extra": True, } }, "type": "object", From 8adfe4c5eb79f3c2e8a4a254b962ac84a6ddb994 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Thu, 20 May 2021 21:53:50 -0400 Subject: [PATCH 09/24] remove fe code --- .../src/common/components/index.tsx | 1 - .../data/database/DatabaseModal/index.tsx | 43 +------------------ 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/superset-frontend/src/common/components/index.tsx b/superset-frontend/src/common/components/index.tsx index c0cf44adb99e0..bf17aac743291 100644 --- a/superset-frontend/src/common/components/index.tsx +++ b/superset-frontend/src/common/components/index.tsx @@ -54,7 +54,6 @@ export { Tag, Tabs, Tooltip, - Upload, Input as AntdInput, } from 'antd'; export { Card as AntdCard } from 'antd'; diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx index f4d8d1460c208..4547f8bc0d4b0 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx @@ -17,7 +17,6 @@ * under the License. */ import { t } from '@superset-ui/core'; -import { Upload, Button } from 'antd'; import React, { FunctionComponent, useEffect, @@ -26,7 +25,7 @@ import React, { Reducer, } from 'react'; import Tabs from 'src/components/Tabs'; -import { Alert, Select, Upload, Button } from 'src/common/components'; +import { Alert } from 'src/common/components'; import withToasts from 'src/messageToasts/enhancers/withToasts'; import { testDatabaseConnection, @@ -38,7 +37,6 @@ import ExtraOptions from './ExtraOptions'; import SqlAlchemyForm from './SqlAlchemyForm'; import { StyledBasicTab, - StyledJsonEditor, StyledModal, EditHeader, EditHeaderTitle, @@ -52,8 +50,6 @@ import { const DOCUMENTATION_LINK = 'https://superset.apache.org/docs/databases/installing-database-drivers'; -const { Option } = Select; - interface DatabaseModalProps { addDangerToast: (msg: string) => void; addSuccessToast: (msg: string) => void; @@ -152,11 +148,10 @@ const DatabaseModal: FunctionComponent = ({ Reducer | null, DBReducerActionType> >(dbReducer, null); const [tabKey, setTabKey] = useState(DEFAULT_TAB_KEY); - const [bigQueryOption, setBigQueryOption] = useState('upload'); const conf = useCommonConf(); const isEditMode = !!databaseId; - const useSqlAlchemyForm = false; // TODO: set up logic + const useSqlAlchemyForm = true; // TODO: set up logic const hasConnectedDb = false; // TODO: set up logic // Database fetch logic @@ -271,39 +266,6 @@ const DatabaseModal: FunctionComponent = ({ setTabKey(key); }; - const BigQueryForm = () => ( - <> - - {bigQueryOption === 'paste' ? ( -
- { - console.log(json); - }} - width="100%" - height="160px" - /> -
- ) : ( - - - - )} - - ); - return isEditMode || useSqlAlchemyForm ? ( = ({ >

TODO: db form

-
); From 7d2b7d9a5a35c6247ec3589aec0ac49aca61d21d Mon Sep 17 00:00:00 2001 From: hughhhh Date: Fri, 21 May 2021 10:09:51 -0400 Subject: [PATCH 10/24] update encrypted_extra reference --- superset/databases/commands/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index beb028be3d0d8..7e7af32bf3a53 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -84,7 +84,7 @@ def run(self) -> None: # try to connect sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( self._properties["parameters"], # type: ignore - self._properties["encrypted_extra"], + self._properties.get("encrypted_extra", "{}"), ) if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri(): sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted From 9bddea70c4d179f2e6b432c400ec8e94978fc4f0 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Fri, 21 May 2021 11:10:52 -0400 Subject: [PATCH 11/24] fix linting --- superset/db_engine_specs/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 243b87b13373f..b5a40a2c20c86 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -54,7 +54,7 @@ class BigQueryParametersSchema(Schema): credentials_info = EncryptedField(description="credentials.json file for BigQuery") -def encrypted_field_properties(self, field, **kwargs): +def encrypted_field_properties(self, field: Any, **_) -> Dict[str, Any]: # type: ignore ret = {} if isinstance(field, EncryptedField): if self.openapi_version.major > 2: From cdc6580f04998fd7fbb7ad9c357b70aaaa610279 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Fri, 21 May 2021 12:08:21 -0400 Subject: [PATCH 12/24] fix linting --- superset/databases/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/superset/databases/api.py b/superset/databases/api.py index b5b16ffd432b7..6ecd838ef96a8 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -66,7 +66,6 @@ ) from superset.databases.utils import get_table_metadata from superset.db_engine_specs import get_available_engine_specs -from superset.db_engine_specs.base import BasicParametersMixin from superset.exceptions import InvalidPayloadFormatError, InvalidPayloadSchemaError from superset.extensions import security_manager from superset.models.core import Database @@ -912,7 +911,9 @@ def available(self) -> Response: if hasattr(engine_spec, "parameters_json_schema") or hasattr( engine_spec, "sqlalchemy_uri_placeholder" ): - payload["parameters"] = engine_spec.parameters_json_schema() # type: ignore + payload[ + "parameters" + ] = engine_spec.parameters_json_schema() # type: ignore payload[ "sqlalchemy_uri_placeholder" ] = engine_spec.sqlalchemy_uri_placeholder # type: ignore From 05067527cbcdb478e6019e481c317e7b3b398772 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Fri, 21 May 2021 15:56:55 -0400 Subject: [PATCH 13/24] fix linting againa --- superset/models/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/superset/models/core.py b/superset/models/core.py index 3889f41a84f9a..540974f9de375 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -55,7 +55,6 @@ from superset import app, db_engine_specs, is_feature_enabled from superset.db_engine_specs.base import TimeGrain -from superset.db_engine_specs.bigquery import BigQueryEngineSpec from superset.extensions import cache_manager, encrypted_field_factory, security_manager from superset.models.helpers import AuditMixinNullable, ImportExportMixin from superset.models.tags import FavStarUpdater From 401873dd6fd2ace9362c489b933e4b9537c20ac7 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Fri, 21 May 2021 17:32:54 -0400 Subject: [PATCH 14/24] Update superset/db_engine_specs/bigquery.py Co-authored-by: Beto Dealmeida --- superset/db_engine_specs/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index b5a40a2c20c86..6883d43090390 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -63,7 +63,7 @@ def encrypted_field_properties(self, field: Any, **_) -> Dict[str, Any]: # type class BigQueryParametersType(TypedDict): - pass + credentials_info: Dict[str, Any] class BigQueryEngineSpec(BaseEngineSpec): From 77ce5312398316e15b9b5c4fbd746ee6bf3dd2c9 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sat, 22 May 2021 16:17:02 -0400 Subject: [PATCH 15/24] refactor --- superset/databases/commands/validate.py | 9 ++++++++- superset/databases/schemas.py | 22 ++++++++++++++++++++-- superset/db_engine_specs/bigquery.py | 13 +------------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index 7e7af32bf3a53..08cc10ef91aa7 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import json from contextlib import closing from typing import Any, Dict, Optional @@ -81,10 +82,16 @@ def run(self) -> None: if errors: raise InvalidParametersError(errors) + serialized_encrypted_extra = self._properties.get("encrypted_extra", "{}") + try: + encrypted_extra = json.loads(serialized_encrypted_extra) + except json.decoder.JSONDecodeError: + encrypted_extra = {} + # try to connect sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( self._properties["parameters"], # type: ignore - self._properties.get("encrypted_extra", "{}"), + encrypted_extra, ) if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri(): sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index 74b7af441ef1d..0381b3c9dbb75 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -246,7 +246,11 @@ def build_sqlalchemy_uri( the constructed SQLAlchemy URI to be passed. """ parameters = data.pop("parameters", None) - encrypted_extra = data.get("encrypted_extra", None) + serialized_encrypted_extra = data.get("encrypted_extra", "{}") + try: + encrypted_extra = json.loads(serialized_encrypted_extra) + except json.decoder.JSONDecodeError: + encrypted_extra = {} if parameters: if "engine" not in parameters: @@ -277,9 +281,11 @@ def build_sqlalchemy_uri( ] ) - data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( + if hasattr(engine_spec, "build_sqlalchemy_uri"): + data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( # type: ignore parameters, encrypted_extra ) + return data @@ -556,3 +562,15 @@ def validate_password(self, data: Dict[str, Any], **kwargs: Any) -> None: password = make_url(uri).password if password == PASSWORD_MASK and data.get("password") is None: raise ValidationError("Must provide a password for the database") + + +class EncryptedField(fields.String): + pass + + +def encrypted_field_properties(self, field: Any, **_) -> Dict[str, Any]: # type: ignore + ret = {} + if isinstance(field, EncryptedField): + if self.openapi_version.major > 2: + ret["x-encrypted-extra"] = True + return ret diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 6883d43090390..92f139bc20f91 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -27,6 +27,7 @@ from sqlalchemy.sql.expression import ColumnClause from typing_extensions import TypedDict +from superset.databases.schemas import encrypted_field_properties, EncryptedField from superset.db_engine_specs.base import BaseEngineSpec from superset.errors import SupersetErrorType from superset.exceptions import SupersetGenericDBErrorException @@ -46,22 +47,10 @@ ma_plugin = MarshmallowPlugin() -class EncryptedField(fields.String): - pass - - class BigQueryParametersSchema(Schema): credentials_info = EncryptedField(description="credentials.json file for BigQuery") -def encrypted_field_properties(self, field: Any, **_) -> Dict[str, Any]: # type: ignore - ret = {} - if isinstance(field, EncryptedField): - if self.openapi_version.major > 2: - ret["x-encrypted-extra"] = True - return ret - - class BigQueryParametersType(TypedDict): credentials_info: Dict[str, Any] From 5a604b45f13e7f162993d01411ed2e52eb9b52ae Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sat, 22 May 2021 16:32:13 -0400 Subject: [PATCH 16/24] rewrite get parameters from uri --- superset/db_engine_specs/bigquery.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 92f139bc20f91..d470fd82005c2 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -316,9 +316,16 @@ def build_sqlalchemy_uri( ) @classmethod - def get_parameters_from_uri(cls, _: str) -> Any: + def get_parameters_from_uri( + cls, _: str, encrypted_extra: Optional[Dict[str, str]] = None + ) -> Any: # BigQuery doesn't have parameters - return None + if encrypted_extra: + return encrypted_extra + + raise SupersetGenericDBErrorException( + message="Big Query encrypted_extra is not available", + ) @classmethod def parameters_json_schema(cls) -> Any: From bb66ffb0ba3011a70274425353e808ef88d97b8f Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sat, 22 May 2021 16:34:52 -0400 Subject: [PATCH 17/24] remove unused import --- superset/db_engine_specs/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index d470fd82005c2..fa8bd6c954efe 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -22,7 +22,7 @@ from apispec import APISpec from apispec.ext.marshmallow import MarshmallowPlugin from flask_babel import gettext as __ -from marshmallow import fields, Schema +from marshmallow import Schema from sqlalchemy import literal_column from sqlalchemy.sql.expression import ColumnClause from typing_extensions import TypedDict From b4e9a029423525f8467300fc830b3ba6f6ae476d Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sat, 22 May 2021 17:10:48 -0400 Subject: [PATCH 18/24] yerp --- superset/databases/commands/validate.py | 2 +- superset/databases/schemas.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index 08cc10ef91aa7..93e32ef71fb42 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -99,7 +99,7 @@ def run(self) -> None: server_cert=self._properties.get("server_cert", ""), extra=self._properties.get("extra", "{}"), impersonate_user=self._properties.get("impersonate_user", False), - encrypted_extra=self._properties.get("encrypted_extra", "{}"), + encrypted_extra=serialized_encrypted_extra, ) database.set_sqlalchemy_uri(sqlalchemy_uri) database.db_engine_spec.mutate_db_for_connection_test(database) diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index 41991266d30b1..fa63544266d68 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -270,18 +270,12 @@ def build_sqlalchemy_uri( [_('Engine "%(engine)s" is not a valid engine.', engine=engine,)] ) engine_spec = engine_specs[engine] + if hasattr(engine_spec, "build_sqlalchemy_uri"): - data[ - "sqlalchemy_uri" - ] = engine_spec.build_sqlalchemy_uri( # type: ignore - parameters + data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( # type: ignore + parameters, encrypted_extra ) - if hasattr(engine_spec, "build_sqlalchemy_uri"): - data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( # type: ignore - parameters, encrypted_extra - ) - return data From d9d737938479c4f1fd29e0052701ccb04b5f6926 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sat, 22 May 2021 19:17:46 -0400 Subject: [PATCH 19/24] linting --- superset/databases/schemas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index fa63544266d68..ff3054979ebba 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -272,7 +272,9 @@ def build_sqlalchemy_uri( engine_spec = engine_specs[engine] if hasattr(engine_spec, "build_sqlalchemy_uri"): - data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( # type: ignore + data[ + "sqlalchemy_uri" + ] = engine_spec.build_sqlalchemy_uri( # type: ignore parameters, encrypted_extra ) From eaecbb67a66c652e4c4dd2ec72cad3bb6decbb31 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Sun, 23 May 2021 09:36:30 -0400 Subject: [PATCH 20/24] Update superset/db_engine_specs/bigquery.py Co-authored-by: Beto Dealmeida --- superset/db_engine_specs/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index fa8bd6c954efe..4e21ad1890103 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -324,7 +324,7 @@ def get_parameters_from_uri( return encrypted_extra raise SupersetGenericDBErrorException( - message="Big Query encrypted_extra is not available", + message="Big Query encrypted_extra is not available.", ) @classmethod From a5cd2b7c8388f5f961f6e82e387ee2b23a342e5d Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Sun, 23 May 2021 09:36:49 -0400 Subject: [PATCH 21/24] Update superset/db_engine_specs/bigquery.py Co-authored-by: Beto Dealmeida --- superset/db_engine_specs/bigquery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 4e21ad1890103..673f7ce2e9e70 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -48,7 +48,9 @@ class BigQueryParametersSchema(Schema): - credentials_info = EncryptedField(description="credentials.json file for BigQuery") + credentials_info = EncryptedField( + description="Contents of BigQuery JSON credentials.", + ) class BigQueryParametersType(TypedDict): From 361e7554733b85f607edfa060ef532cce920311a Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Sun, 23 May 2021 09:36:55 -0400 Subject: [PATCH 22/24] Update superset/databases/api.py Co-authored-by: Beto Dealmeida --- superset/databases/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/databases/api.py b/superset/databases/api.py index 6ecd838ef96a8..0b0d29718d3c6 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -908,7 +908,7 @@ def available(self) -> Response: "preferred": engine_spec.engine in preferred_databases, } - if hasattr(engine_spec, "parameters_json_schema") or hasattr( + if hasattr(engine_spec, "parameters_json_schema") and hasattr( engine_spec, "sqlalchemy_uri_placeholder" ): payload[ From 65822ed71328f20255770178740d93c67d556bab Mon Sep 17 00:00:00 2001 From: hughhhh Date: Sun, 23 May 2021 10:18:52 -0400 Subject: [PATCH 23/24] fix test --- tests/databases/api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index bed4c7be6a921..1868f95d60fae 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -1436,7 +1436,7 @@ def test_available(self, app, get_available_engine_specs): "parameters": { "properties": { "credentials_info": { - "description": "credentials.json file for BigQuery", + "description": "Contents of BigQuery JSON credentials.", "type": "string", "x-encrypted-extra": True, } From 42e72473a4855a1ae8e1a42ac88d268943514206 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Sun, 23 May 2021 10:32:39 -0400 Subject: [PATCH 24/24] Update superset/db_engine_specs/bigquery.py Co-authored-by: Beto Dealmeida --- superset/db_engine_specs/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index 673f7ce2e9e70..a7ce77c580f6e 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -314,7 +314,7 @@ def build_sqlalchemy_uri( return f"{cls.drivername}://{project_id}" raise SupersetGenericDBErrorException( - message="Big Query encrypted_extra is not available", + message="Big Query encrypted_extra is not available.", ) @classmethod