openlineage: add method to common.compat to not force hooks to try/ex…

…cept every 2.10 hook lineage call

Signed-off-by: Maciej Obuchowski <obuchowski.maciej@gmail.com>

.pre-commit-config.yaml

@@ -675,6 +675,15 @@ repos:
  (?x)
  ^airflow/providers/.*\.py$
  exclude: ^.*/.*_vendor/
+ - id: check-get-lineage-collector-providers
+ language: pygrep
+ name: Check providers import hook lineage code from compat
+ description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of
+ airflow.lineage.hook.
+ entry: "airflow\\.lineage\\.hook"
+ pass_filenames: true
+ files: ^airflow/providers/.*\.py$
+ exclude: ^airflow/providers/common/compat/.*\.py$
  - id: check-decorated-operator-implements-custom-name
  name: Check @task decorator implements custom_operator_name
  language: python

tests/providers/common/compat/test_empty.py → airflow/providers/common/compat/lineage/__init__.py

@@ -14,8 +14,3 @@
 # KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
-
-
-def test_empty():
- assert True

airflow/providers/common/compat/lineage/hook.py

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+
+def get_hook_lineage_collector():
+ # HookLineageCollector added in 2.10
+ try:
+ from airflow.lineage.hook import get_hook_lineage_collector
+
+ return get_hook_lineage_collector()
+ except ImportError:
+
+ class NoOpCollector:
+ """
+ NoOpCollector is a hook lineage collector that does nothing.
+
+ It is used when you want to disable lineage collection.
+ """
+
+ def add_input_dataset(self, *_):
+ pass
+
+ def add_output_dataset(self, *_):
+ pass
+
+ return NoOpCollector()

airflow/providers/openlineage/sqlparser.py

@@ -160,7 +160,6 @@ def parse_table_schemas(
  "database": database or database_info.database,
  "use_flat_cross_db_query": database_info.use_flat_cross_db_query,
  }
- self.log.info("PRE getting schemas for input and output tables")
  return get_table_schemas(
  hook,
  namespace,

contributing-docs/08_static_code_checks.rst

@@ -174,6 +174,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-for-inclusive-language | Check for language that we do not accept as community | |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
+| check-get-lineage-collector-providers | Check providers import hook lineage code from compat | |
++-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-google-re2-as-dependency | Check google-re2 is declared as dependency when needed | |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-hatch-build-order | Check order of dependencies in hatch_build.py | |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-7b121e6337aeb2242ab88b8f51ae1907
+9381c6120248c8e22bd10d9f882ef667

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -52,6 +52,7 @@
  "check-extras-order",
  "check-fab-migrations",
  "check-for-inclusive-language",
+ "check-get-lineage-collector-providers",
  "check-google-re2-as-dependency",
  "check-hatch-build-order",
  "check-hooks-apply",

tests/providers/common/compat/lineage/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/providers/common/compat/lineage/test_hook.py

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
+
+
+def test_that_compat_does_not_raise():
+ # On compat tests this goes into ImportError code path
+ assert get_hook_lineage_collector() is not None