From 19dc5dfbbf495c1e0cff1f46b647c3a25c91ff08 Mon Sep 17 00:00:00 2001 From: justinpark Date: Mon, 4 Mar 2024 14:52:18 -0800 Subject: [PATCH 1/9] fix: Missing sql_editor_id index --- ...7a20ff8d8_add_query_sql_editor_id_index.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py diff --git a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py new file mode 100644 index 0000000000000..04bc21bfb63a0 --- /dev/null +++ b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""add_query_sql_editor_id_index + +Revision ID: efb7a20ff8d8 +Revises: be1b217cd8cd +Create Date: 2024-03-04 14:48:16.998927 + +""" + +# revision identifiers, used by Alembic. +revision = 'efb7a20ff8d8' +down_revision = 'be1b217cd8cd' + +from alembic import op +import sqlalchemy as sa + +def upgrade(): + op.create_index( + op.f("ix_query_sql_editor_id"), "query", ["sql_editor_id"], unique=False + ) + + +def downgrade(): + op.drop_index(op.f("ix_query_sql_editor_id"), table_name="query") From ab74305a55f3572d52cb944e80dce8c5622dd659 Mon Sep 17 00:00:00 2001 From: justinpark Date: Mon, 4 Mar 2024 15:16:47 -0800 Subject: [PATCH 2/9] update index tupled and UPDATING.md --- UPDATING.md | 1 + ...8_efb7a20ff8d8_add_query_sql_editor_id_index.py | 14 +++++++++----- superset/models/sql_lab.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 5af1c5c501b57..4db84310ce575 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -80,6 +80,7 @@ assists people when migrating to a new version. ### Potential Downtime - [26416](https://github.com/apache/superset/pull/26416): Adds two database indexes to the `report_execution_log` table and one database index to the `report_recipient` to improve performance. Scheduled downtime may be required for large deployments. +- [27392](https://github.com/apache/superset/pull/27392): adds sql_editor_id index to query to improve performance, this may cause downtime on large deployments. ## 3.1.0 diff --git a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py index 04bc21bfb63a0..2362e5aa89632 100644 --- a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py +++ b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py @@ -23,17 +23,21 @@ """ # revision identifiers, used by Alembic. -revision = 'efb7a20ff8d8' -down_revision = 'be1b217cd8cd' +revision = "efb7a20ff8d8" +down_revision = "be1b217cd8cd" -from alembic import op import sqlalchemy as sa +from alembic import op + def upgrade(): op.create_index( - op.f("ix_query_sql_editor_id"), "query", ["sql_editor_id"], unique=False + op.f("ix_query_user_id_sql_editor_id"), + "query", + ["user_id", "sql_editor_id"], + unique=False, ) def downgrade(): - op.drop_index(op.f("ix_query_sql_editor_id"), table_name="query") + op.drop_index(op.f("ix_query_user_id_sql_editor_id"), table_name="query") diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 40a5132c556c6..31443b4bb13e6 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -107,7 +107,7 @@ class Query( user_id = Column(Integer, ForeignKey("ab_user.id"), nullable=True) status = Column(String(16), default=QueryStatus.PENDING) tab_name = Column(String(256)) - sql_editor_id = Column(String(256)) + sql_editor_id = Column(String(256), index=True) schema = Column(String(256)) catalog = Column(String(256), nullable=True, default=None) sql = Column(MediumText()) From 6796d36df4b49244d0c6ef95a49796370361b950 Mon Sep 17 00:00:00 2001 From: justinpark Date: Wed, 6 Mar 2024 14:28:49 -0800 Subject: [PATCH 3/9] use Index to config combo index --- superset/models/sql_lab.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 31443b4bb13e6..808d25c55e584 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -107,7 +107,7 @@ class Query( user_id = Column(Integer, ForeignKey("ab_user.id"), nullable=True) status = Column(String(16), default=QueryStatus.PENDING) tab_name = Column(String(256)) - sql_editor_id = Column(String(256), index=True) + sql_editor_id = Column(String(256)) schema = Column(String(256)) catalog = Column(String(256), nullable=True, default=None) sql = Column(MediumText()) @@ -150,7 +150,10 @@ class Query( ) user = relationship(security_manager.user_model, foreign_keys=[user_id]) - __table_args__ = (sqla.Index("ti_user_id_changed_on", user_id, changed_on),) + __table_args__ = ( + sqla.Index("ti_user_id_changed_on", user_id, changed_on), + sqla.Index("ix_query_user_id_sql_editor_id", user_id, sql_editor_id), + ) def get_template_processor(self, **kwargs: Any) -> BaseTemplateProcessor: return get_template_processor(query=self, database=self.database, **kwargs) From 2a4efb1a597027eb8d1cf4d4fb1515c9962f9958 Mon Sep 17 00:00:00 2001 From: "Michael S. Molina" Date: Thu, 7 Mar 2024 11:04:28 -0300 Subject: [PATCH 4/9] Checks for index existence --- superset/migrations/shared/utils.py | 17 ++++++++++++++ ...7a20ff8d8_add_query_sql_editor_id_index.py | 22 ++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/superset/migrations/shared/utils.py b/superset/migrations/shared/utils.py index 44d4c3924bba6..db20140db940b 100644 --- a/superset/migrations/shared/utils.py +++ b/superset/migrations/shared/utils.py @@ -51,6 +51,23 @@ def table_has_column(table: str, column: str) -> bool: return False +def table_has_index(table: str, index: str) -> bool: + """ + Checks if an index exists in a given table. + + :param table: A table name + :param index: A index name + :returns: True if the index exists in the table + """ + + insp = inspect(op.get_context().bind) + + try: + return any(ind["name"] == index for ind in insp.get_indexes(table)) + except NoSuchTableError: + return False + + uuid_by_dialect = { MySQLDialect: "UNHEX(REPLACE(CONVERT(UUID() using utf8mb4), '-', ''))", PGDialect: "uuid_in(md5(random()::text || clock_timestamp()::text)::cstring)", diff --git a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py index 2362e5aa89632..82f4f29cc5891 100644 --- a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py +++ b/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py @@ -26,18 +26,24 @@ revision = "efb7a20ff8d8" down_revision = "be1b217cd8cd" -import sqlalchemy as sa from alembic import op +from superset.migrations.shared.utils import table_has_index + +table = "query" +index = "ix_query_user_id_sql_editor_id" + def upgrade(): - op.create_index( - op.f("ix_query_user_id_sql_editor_id"), - "query", - ["user_id", "sql_editor_id"], - unique=False, - ) + if not table_has_index(table, index): + op.create_index( + op.f(index), + table, + ["user_id", "sql_editor_id"], + unique=False, + ) def downgrade(): - op.drop_index(op.f("ix_query_user_id_sql_editor_id"), table_name="query") + if table_has_index(table, index): + op.drop_index(op.f(index), table_name=table) From 3fb35e31a5f9b2a7ff005f521c580c51a2c83064 Mon Sep 17 00:00:00 2001 From: "JUST.in DO IT" Date: Thu, 7 Mar 2024 10:00:12 -0800 Subject: [PATCH 5/9] Update UPDATING.md Co-authored-by: Michael S. Molina <70410625+michael-s-molina@users.noreply.github.com> --- UPDATING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UPDATING.md b/UPDATING.md index 4db84310ce575..c9a23ca74f963 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -80,7 +80,7 @@ assists people when migrating to a new version. ### Potential Downtime - [26416](https://github.com/apache/superset/pull/26416): Adds two database indexes to the `report_execution_log` table and one database index to the `report_recipient` to improve performance. Scheduled downtime may be required for large deployments. -- [27392](https://github.com/apache/superset/pull/27392): adds sql_editor_id index to query to improve performance, this may cause downtime on large deployments. +- [27392](https://github.com/apache/superset/pull/27392): Adds an index to `query.sql_editor_id` to improve performance. This may cause downtime on large deployments. ## 3.1.0 From 1d191187b7ee52280e8be88f8a9de52657fccebd Mon Sep 17 00:00:00 2001 From: justinpark Date: Fri, 8 Mar 2024 16:01:53 -0800 Subject: [PATCH 6/9] change to single column index --- ..._3dfd0e78650e_add_query_sql_editor_id_index.py} | 14 +++++++------- superset/models/sql_lab.py | 7 ++----- 2 files changed, 9 insertions(+), 12 deletions(-) rename superset/migrations/versions/{2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py => 2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py} (84%) diff --git a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py similarity index 84% rename from superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py rename to superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py index 82f4f29cc5891..b9c74fab94cd9 100644 --- a/superset/migrations/versions/2024-03-04_14-48_efb7a20ff8d8_add_query_sql_editor_id_index.py +++ b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py @@ -16,22 +16,22 @@ # under the License. """add_query_sql_editor_id_index -Revision ID: efb7a20ff8d8 -Revises: be1b217cd8cd -Create Date: 2024-03-04 14:48:16.998927 +Revision ID: 3dfd0e78650e +Revises: 5f57af97bc3f +Create Date: 2024-05-02 13:40:23.126659 """ # revision identifiers, used by Alembic. -revision = "efb7a20ff8d8" -down_revision = "be1b217cd8cd" +revision = '3dfd0e78650e' +down_revision = '5f57af97bc3f' from alembic import op from superset.migrations.shared.utils import table_has_index table = "query" -index = "ix_query_user_id_sql_editor_id" +index = "ix_sql_editor_id" def upgrade(): @@ -39,7 +39,7 @@ def upgrade(): op.create_index( op.f(index), table, - ["user_id", "sql_editor_id"], + ["sql_editor_id"], unique=False, ) diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 808d25c55e584..31443b4bb13e6 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -107,7 +107,7 @@ class Query( user_id = Column(Integer, ForeignKey("ab_user.id"), nullable=True) status = Column(String(16), default=QueryStatus.PENDING) tab_name = Column(String(256)) - sql_editor_id = Column(String(256)) + sql_editor_id = Column(String(256), index=True) schema = Column(String(256)) catalog = Column(String(256), nullable=True, default=None) sql = Column(MediumText()) @@ -150,10 +150,7 @@ class Query( ) user = relationship(security_manager.user_model, foreign_keys=[user_id]) - __table_args__ = ( - sqla.Index("ti_user_id_changed_on", user_id, changed_on), - sqla.Index("ix_query_user_id_sql_editor_id", user_id, sql_editor_id), - ) + __table_args__ = (sqla.Index("ti_user_id_changed_on", user_id, changed_on),) def get_template_processor(self, **kwargs: Any) -> BaseTemplateProcessor: return get_template_processor(query=self, database=self.database, **kwargs) From 62216a80b68283067b9f75ce464d46a9ef8517eb Mon Sep 17 00:00:00 2001 From: justinpark Date: Thu, 2 May 2024 14:04:42 -0700 Subject: [PATCH 7/9] precommit --- ...-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py index b9c74fab94cd9..ba42b77b28fdb 100644 --- a/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py +++ b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py @@ -23,8 +23,8 @@ """ # revision identifiers, used by Alembic. -revision = '3dfd0e78650e' -down_revision = '5f57af97bc3f' +revision = "3dfd0e78650e" +down_revision = "5f57af97bc3f" from alembic import op From 520f1452c474b2e32d0b4ce0835981e1d1194d31 Mon Sep 17 00:00:00 2001 From: justinpark Date: Thu, 2 May 2024 16:16:58 -0700 Subject: [PATCH 8/9] noqa --- ...-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py index ba42b77b28fdb..d4eacaf5a2661 100644 --- a/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py +++ b/superset/migrations/versions/2024-05-02_13-40_3dfd0e78650e_add_query_sql_editor_id_index.py @@ -26,9 +26,9 @@ revision = "3dfd0e78650e" down_revision = "5f57af97bc3f" -from alembic import op +from alembic import op # noqa: E402 -from superset.migrations.shared.utils import table_has_index +from superset.migrations.shared.utils import table_has_index # noqa: E402 table = "query" index = "ix_sql_editor_id" From 3e2c382825b6f1206895a1867293ab347c969760 Mon Sep 17 00:00:00 2001 From: justinpark Date: Fri, 3 May 2024 09:34:24 -0700 Subject: [PATCH 9/9] move potential downtime section to next --- UPDATING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/UPDATING.md b/UPDATING.md index c9a23ca74f963..909cd9f6d9bed 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -48,6 +48,10 @@ assists people when migrating to a new version. listing all databases in CRUD-view and dropdown and didn't provide access to data as it seemed the name would imply. +### Potential Downtime + +- [27392](https://github.com/apache/superset/pull/27392): Adds an index to `query.sql_editor_id` to improve performance. This may cause downtime on large deployments. + ## 4.0.0 - [27119](https://github.com/apache/superset/pull/27119): Updates various database columns to use the `MediumText` type, potentially requiring a table lock on MySQL dbs or taking some time to complete on large deployments. @@ -80,7 +84,6 @@ assists people when migrating to a new version. ### Potential Downtime - [26416](https://github.com/apache/superset/pull/26416): Adds two database indexes to the `report_execution_log` table and one database index to the `report_recipient` to improve performance. Scheduled downtime may be required for large deployments. -- [27392](https://github.com/apache/superset/pull/27392): Adds an index to `query.sql_editor_id` to improve performance. This may cause downtime on large deployments. ## 3.1.0