Update conf column in dag_run table type from bytea to JSON

airflow/migrations/versions/0052_3_0_0_remove_pickled_data_from_dagrun_table.py

@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+remove pickled data from dagrun table.
+
+Revision ID: e39a26ac59f6
+Revises: 038dc8bc6284
+Create Date: 2024-12-01 08:33:15.425141
+
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e39a26ac59f6"
+down_revision = "038dc8bc6284"
+branch_labels = None
+depends_on = None
+airflow_version = "3.0.0"
+
+
+def upgrade():
+    """Apply remove pickled data from dagrun table."""
+    conn = op.get_bind()
+
+    # Update the dag_run.conf column value to NULL
+    conn.execute(text("UPDATE dag_run set conf=null WHERE conf IS NOT NULL"))
+
+    conf_type = sa.JSON().with_variant(postgresql.JSONB, "postgresql")
+
+    with op.batch_alter_table("dag_run", schema=None) as batch_op:
+        # Drop the existing column for SQLite (due to its limitations)
+        batch_op.drop_column("conf")
+        # Add the new column with the correct type for the all dialect
+        batch_op.add_column(sa.Column("conf", conf_type, nullable=True))
+
+
+def downgrade():
+    """Unapply Remove pickled data from dagrun table."""
+    conn = op.get_bind()
+    conn.execute(text("UPDATE dag_run set conf=null WHERE conf IS NOT NULL"))
+
+    # Update the dag_run.conf column value to NULL to avoid issues during the type change
+    conn.execute(text("UPDATE dag_run set conf=null WHERE conf IS NOT NULL"))
+
+    conf_type = sa.LargeBinary().with_variant(postgresql.BYTEA, "postgresql")
+
+    # Apply the same logic for all dialects, including SQLite
+    with op.batch_alter_table("dag_run", schema=None) as batch_op:
+        # Drop the existing column for SQLite (due to its limitations)
+        batch_op.drop_column("conf")
+        batch_op.add_column(sa.Column("conf", conf_type, nullable=True))

airflow/models/dagrun.py

@@ -25,14 +25,14 @@
 
 import re2
 from sqlalchemy import (
+    JSON,
     Boolean,
     Column,
     Enum,
     ForeignKey,
     ForeignKeyConstraint,
     Index,
     Integer,
-    PickleType,
     PrimaryKeyConstraint,
     String,
     Text,
@@ -44,6 +44,7 @@
     text,
     update,
 )
+from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import declared_attr, joinedload, relationship, synonym, validates
@@ -137,7 +138,7 @@ class DagRun(Base, LoggingMixin):
     triggered_by = Column(
         Enum(DagRunTriggeredByType, native_enum=False, length=50)
     )  # Airflow component that triggered the run.
-    conf = Column(PickleType)
+    conf = Column(JSON().with_variant(postgresql.JSONB, "postgresql"))
     # These two must be either both NULL or both datetime.
     data_interval_start = Column(UtcDateTime)
     data_interval_end = Column(UtcDateTime)

airflow/utils/db.py

@@ -94,7 +94,7 @@ class MappedClassProtocol(Protocol):
     "2.9.2": "686269002441",
     "2.10.0": "22ed7efa9da2",
     "2.10.3": "5f2621c13b39",
-    "3.0.0": "038dc8bc6284",
+    "3.0.0": "e39a26ac59f6",
 }
 
 

docs/apache-airflow/img/airflow_erd.sha256

@@ -1 +1 @@
-8f2fd91375c546b297490e701dc3853d7ba53c7cd1422ed7f7e57b9ac86f6eca
+3c7e69f4098d4078941590dd3a21b78473e8586683595a592dafc08decf4d534

docs/apache-airflow/migrations-ref.rst

@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | Revision ID             | Revises ID       | Airflow Version   | Description                                                  |
 +=========================+==================+===================+==============================================================+
-| ``038dc8bc6284`` (head) | ``e229247a6cb1`` | ``3.0.0``         | update trigger_timeout column in task_instance table to UTC. |
+| ``e39a26ac59f6`` (head) | ``038dc8bc6284`` | ``3.0.0``         | remove pickled data from dagrun table.                       |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``038dc8bc6284``        | ``e229247a6cb1`` | ``3.0.0``         | update trigger_timeout column in task_instance table to UTC. |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | ``e229247a6cb1``        | ``eed27faa34e3`` | ``3.0.0``         | Add DagBundleModel.                                          |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+

newsfragments/44533.significant.rst

@@ -0,0 +1,21 @@
+Update conf column in dag_run table type from byte ( that store a python pickle ) to JSON
+
+.. Provide additional contextual information
+
+Column conf of the table dag_run is using the type byte ( and storing a python pickle ) on the database , since airflow only support postgres 12+ and mysql 8+ , we updated it to  json type .
+
+.. Check the type of change that applies to this change
+
+* Types of change
+
+  * [ ] DAG changes
+  * [ ] Config changes
+  * [ ] API changes
+  * [ ] CLI changes
+  * [x] Behaviour changes
+  * [ ] Plugin changes
+  * [ ] Dependency change
+
+.. List the migration rules needed for this change (see https://github.com/apache/airflow/issues/41641)
+
+* Migrations rules needed