docs: Add doc page with providers deprecations (#37075)

airflow/providers/amazon/aws/hooks/base_aws.py

@@ -1022,7 +1022,7 @@ def _parse_s3_config(config_file_name: str, config_format: str | None = "boto",
 
 @deprecated(
     reason=(
-        "airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory "
+        "`airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory` "
         "has been deprecated and will be removed in future"
     ),
     category=AirflowProviderDeprecationWarning,
@@ -1116,7 +1116,7 @@ def create_session(self, deferrable: bool = False) -> AioSession:
 
 @deprecated(
     reason=(
-        "airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook "
+        "`airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook` "
         "has been deprecated and will be removed in future"
     ),
     category=AirflowProviderDeprecationWarning,

airflow/providers/amazon/aws/hooks/redshift_cluster.py

@@ -197,7 +197,7 @@ def get_cluster_snapshot_status(self, snapshot_identifier: str):
 
 @deprecated(
     reason=(
-        "airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook "
+        "`airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook` "
         "has been deprecated and will be removed in future"
     ),
     category=AirflowProviderDeprecationWarning,

docs/README.rst

@@ -258,6 +258,19 @@ warning to report the wrong line in the file for your missing white space. If yo
 and the line number it reports is wrong, this is a known error. You can find the missing blank space by searching for the syntax you used to make your
 list, code block, or other whitespace-sensitive markup element.
 
+
+spelling error with class or method name
+****************************************
+When a spelling error occurs that has a class/function/method name as incorrectly spelled,
+instead of whitelisting it in docs/spelling_wordlist.txt you should make sure that
+this name is quoted with backticks "`" - this should exclude it from spellchecking process.
+
+In this example error, You should change the line with error so that the whole path is inside backticks "`".
+.. code-block:: text
+
+    Incorrect Spelling: 'BaseAsyncSessionFactory'
+    Line with Error: ' airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory has been deprecated'
+
 Support
 =======
 

docs/apache-airflow-providers/core-extensions/deprecations.rst

@@ -0,0 +1,31 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+Deprecations
+==========================================
+
+This is a summary of deprecated objects in latest versions of Apache Airflow Providers Packages.
+
+.. note::
+   This is an experimental page that may not contain all deprecated entities.
+   At the moment we only show deprecations for classes, functions, methods, properties etc.
+   Support for an argument deprecation or an argument value deprecation will be added in the future.
+
+.. airflow-deprecations::
+   :tags: None
+   :header-separator: "

docs/exts/docs_build/errors.py

@@ -77,8 +77,9 @@ def display_errors_summary(build_errors: dict[str, list[DocBuildError]]) -> None
                 console.print(
                     f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)} ({error.line_no})"
                 )
-                console.print()
-                console.print(prepare_code_snippet(error.file_path, error.line_no))
+                if os.path.isfile(error.file_path):
+                    console.print()
+                    console.print(prepare_code_snippet(error.file_path, error.line_no))
             elif error.file_path:
                 console.print(f"File path: {error.file_path}")
     console.print()

docs/exts/docs_build/spelling_checks.py

@@ -154,7 +154,9 @@ def display_spelling_error_summary(spelling_errors: dict[str, list[SpellingError
 
     console.print("=" * 100)
     console.print()
-    msg = """
+    msg = """[green]
+If there are spelling errors related to class or function name, make sure
+those names are quoted with backticks '`' - this should exclude it from spellcheck process.
 If there are spelling errors in the summary above, and the spelling is
 correct, add the spelling to docs/spelling_wordlist.txt or use the
 spelling directive.
@@ -177,11 +179,16 @@ def _display_error(error: SpellingError):
     if error.file_path:
         console.print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)}")
         if error.spelling:
-            console.print(f"Incorrect Spelling: '{error.spelling}'")
+            console.print(f"[red]Incorrect Spelling: '{error.spelling}'")
         if error.suggestion:
             console.print(f"Suggested Spelling: '{error.suggestion}'")
         if error.context_line:
             console.print(f"Line with Error: '{error.context_line}'")
-        if error.file_path and not error.file_path.endswith("<unknown>") and error.line_no:
+        if (
+            error.file_path
+            and not error.file_path.endswith("<unknown>")
+            and error.line_no
+            and os.path.isfile(error.file_path)
+        ):
             console.print(f"Line Number: {error.line_no}")
             console.print(prepare_code_snippet(error.file_path, error.line_no))

docs/exts/operators_and_hooks_ref.py

@@ -221,6 +221,111 @@ def _render_deferrable_operator_content(*, header_separator: str):
     )
 
 
+def _get_decorator_details(decorator):
+    def get_full_name(node):
+        if isinstance(node, ast.Attribute):
+            return f"{get_full_name(node.value)}.{node.attr}"
+        elif isinstance(node, ast.Name):
+            return node.id
+        else:
+            return ast.dump(node)
+
+    def eval_node(node):
+        try:
+            return ast.literal_eval(node)
+        except ValueError:
+            return ast.dump(node)
+
+    if isinstance(decorator, ast.Call):
+        name = get_full_name(decorator.func)
+        args = [eval_node(arg) for arg in decorator.args]
+        kwargs = {kw.arg: eval_node(kw.value) for kw in decorator.keywords if kw.arg != "category"}
+        return name, args, kwargs
+    elif isinstance(decorator, ast.Name):
+        return decorator.id, [], {}
+    elif isinstance(decorator, ast.Attribute):
+        return decorator.attr, [], {}
+    else:
+        return decorator, [], {}
+
+
+def _iter_module_for_deprecations(ast_node, file_path, class_name=None) -> list[dict[str, Any]]:
+    deprecations = []
+    decorators_of_deprecation = {"deprecated"}
+
+    def analyze_decorators(node, _file_path, object_type, _class_name=None):
+        for decorator in node.decorator_list:
+            if str(_class_name).startswith("_") or str(node.name).startswith("_"):
+                continue
+            decorator_name, decorator_args, decorator_kwargs = _get_decorator_details(decorator)
+
+            instructions = decorator_kwargs.get("reason", "No instructions were provided.")
+            if len(decorator_args) == 1 and isinstance(decorator_args[0], str) and not instructions:
+                instructions = decorator_args[0]
+
+            if decorator_name in (
+                "staticmethod",
+                "classmethod",
+                "property",
+                "abstractmethod",
+                "cached_property",
+            ):
+                object_type = decorator_name
+
+            if decorator_name in decorators_of_deprecation:
+                object_name = f"{_class_name}.{node.name}" if _class_name else node.name
+                object_path = os.path.join(_file_path, object_name).replace("/", ".").lstrip(".")
+                deprecations.append(
+                    {
+                        "object_path": object_path,
+                        "object_name": object_name,
+                        "object_type": object_type,
+                        "instructions": instructions,
+                    }
+                )
+
+    for child in ast.iter_child_nodes(ast_node):
+        if isinstance(child, ast.ClassDef):
+            analyze_decorators(child, file_path, object_type="class")
+            deprecations.extend(_iter_module_for_deprecations(child, file_path, class_name=child.name))
+        elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            analyze_decorators(
+                child, file_path, _class_name=class_name, object_type="method" if class_name else "function"
+            )
+        else:
+            deprecations.extend(_iter_module_for_deprecations(child, file_path, class_name=class_name))
+
+    return deprecations
+
+
+def _render_deprecations_content(*, header_separator: str):
+    providers = []
+    for provider_yaml_path in get_provider_yaml_paths():
+        provider_parent_path = Path(provider_yaml_path).parent
+        provider_info: dict[str, Any] = {"name": "", "deprecations": []}
+        for root, _, file_names in os.walk(provider_parent_path):
+            file_names = [f for f in file_names if f.endswith(".py") and f != "__init__.py"]
+            for file_name in file_names:
+                file_path = f"{os.path.relpath(root)}/{file_name}"
+                with open(file_path) as file:
+                    ast_obj = ast.parse(file.read())
+                provider_info["deprecations"].extend(_iter_module_for_deprecations(ast_obj, file_path[:-3]))
+
+        if provider_info["deprecations"]:
+            provider_info["deprecations"] = sorted(
+                provider_info["deprecations"], key=lambda p: p["object_path"]
+            )
+            provider_yaml_content = yaml.safe_load(Path(provider_yaml_path).read_text())
+            provider_info["name"] = provider_yaml_content["package-name"]
+            providers.append(provider_info)
+
+    return _render_template(
+        "deprecations.rst.jinja2",
+        providers=sorted(providers, key=lambda p: p["name"]),
+        header_separator=header_separator,
+    )
+
+
 class BaseJinjaReferenceDirective(Directive):
     """The base directive for OperatorsHooksReferenceDirective and TransfersReferenceDirective"""
 
@@ -399,6 +504,15 @@ def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAU
         )
 
 
+class DeprecationsDirective(BaseJinjaReferenceDirective):
+    """Generate list of deprecated entities"""
+
+    def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR):
+        return _render_deprecations_content(
+            header_separator=header_separator,
+        )
+
+
 def setup(app):
     """Setup plugin"""
     app.add_directive("operators-hooks-ref", OperatorsHooksReferenceDirective)
@@ -412,6 +526,7 @@ def setup(app):
     app.add_directive("airflow-notifications", NotificationsDirective)
     app.add_directive("airflow-executors", ExecutorsDirective)
     app.add_directive("airflow-deferrable-operators", DeferrableOperatorDirective)
+    app.add_directive("airflow-deprecations", DeprecationsDirective)
 
     return {"parallel_read_safe": True, "parallel_write_safe": True}
 

docs/exts/templates/deprecations.rst.jinja2

@@ -0,0 +1,37 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+#}
+{%- for provider in providers %}
+{{ provider["name"] }}
+{{ header_separator * (provider['name']|length) }}
+
+{% for deprecation in provider['deprecations'] %}
+
+{% set prefix = ':attr:' %}
+{% if deprecation["object_type"] == 'class' %}
+  {% set prefix = ':class:' %}
+{% elif deprecation["object_type"] == 'function' %}
+  {% set prefix = ':func:' %}
+{% elif 'method' in deprecation["object_type"] %}
+  {% set prefix = ':meth:' %}
+{% endif %}
+
+- {{ prefix }}`{{ deprecation["object_name"] }} <{{ deprecation["object_path"] }}>` - {{ deprecation["instructions"] }}
+{% endfor %}
+
+{% endfor %}