From 63662c3f18409b17e0ee4c5ef034b36cc26b6f4c Mon Sep 17 00:00:00 2001 From: Kush Date: Tue, 14 Dec 2021 10:47:09 +0530 Subject: [PATCH 1/2] Add dag_id index for job table - helps in speeding up delete operation required to query jobs by dag_id Signed-off-by: Kush --- airflow/jobs/base_job.py | 1 + ...df053233_adding_index_for_dag_id_in_job.py | 45 +++++++++++++++++++ docs/apache-airflow/migrations-ref.rst | 5 ++- 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py diff --git a/airflow/jobs/base_job.py b/airflow/jobs/base_job.py index 275e755e2551a..46ccacbf04adb 100644 --- a/airflow/jobs/base_job.py +++ b/airflow/jobs/base_job.py @@ -71,6 +71,7 @@ class BaseJob(Base, LoggingMixin): __table_args__ = ( Index('job_type_heart', job_type, latest_heartbeat), Index('idx_job_state_heartbeat', state, latest_heartbeat), + Index('idx_job_dag_id', dag_id), ) task_instances_enqueued = relationship( diff --git a/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py new file mode 100644 index 0000000000000..4fb4d98da7727 --- /dev/null +++ b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""adding index for dag_id in job + +Revision ID: 587bdf053233 +Revises: 5e3ec427fdd3 +Create Date: 2021-12-14 10:20:12.482940 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '587bdf053233' +down_revision = '5e3ec427fdd3' +branch_labels = None +depends_on = None + + +def upgrade(): + """Apply adding index for dag_id in job""" + op.create_index('idx_job_dag_id', 'job', ['dag_id'], unique=False) + + +def downgrade(): + """Unapply adding index for dag_id in job""" + op.drop_index('idx_job_dag_id', table_name='job') diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index e73f29e59b6b5..5c9e24175d943 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -23,8 +23,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``f9da662e7089`` (head) | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values ``log_filename_template`` | -| | | | and ``task_log_prefix_template``. | +| ``587bdf053233`` (head) | ``f9da662e7089`` | ``2.3.0`` | Add index for ``dag_id`` column in ``job`` table. | ++--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ +| ``f9da662e7089`` | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values ``log_filename_template`` | +--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ | ``786e3737b18f`` | ``5e3ec427fdd3`` | ``2.3.0`` | Add ``timetable_description`` column to DagModel for UI. | +--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ From 4335b248aab4ba0f1f035dbb59ac3f63d1d98454 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 30 Dec 2021 01:11:21 +0000 Subject: [PATCH 2/2] Update 587bdf053233_adding_index_for_dag_id_in_job.py --- .../versions/587bdf053233_adding_index_for_dag_id_in_job.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py index 4fb4d98da7727..3532fe9e8df14 100644 --- a/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py +++ b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py @@ -19,18 +19,16 @@ """adding index for dag_id in job Revision ID: 587bdf053233 -Revises: 5e3ec427fdd3 +Revises: f9da662e7089 Create Date: 2021-12-14 10:20:12.482940 """ -import sqlalchemy as sa from alembic import op - # revision identifiers, used by Alembic. revision = '587bdf053233' -down_revision = '5e3ec427fdd3' +down_revision = 'f9da662e7089' branch_labels = None depends_on = None