diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py index 5f1e075cc31ab..ff2f33dbe8205 100644 --- a/airflow/providers/amazon/aws/hooks/base_aws.py +++ b/airflow/providers/amazon/aws/hooks/base_aws.py @@ -1022,7 +1022,7 @@ def _parse_s3_config(config_file_name: str, config_format: str | None = "boto", @deprecated( reason=( - "airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory " + "`airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory` " "has been deprecated and will be removed in future" ), category=AirflowProviderDeprecationWarning, @@ -1116,7 +1116,7 @@ def create_session(self, deferrable: bool = False) -> AioSession: @deprecated( reason=( - "airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook " + "`airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook` " "has been deprecated and will be removed in future" ), category=AirflowProviderDeprecationWarning, diff --git a/airflow/providers/amazon/aws/hooks/redshift_cluster.py b/airflow/providers/amazon/aws/hooks/redshift_cluster.py index 286299661c8ea..6168278a599fa 100644 --- a/airflow/providers/amazon/aws/hooks/redshift_cluster.py +++ b/airflow/providers/amazon/aws/hooks/redshift_cluster.py @@ -197,7 +197,7 @@ def get_cluster_snapshot_status(self, snapshot_identifier: str): @deprecated( reason=( - "airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook " + "`airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook` " "has been deprecated and will be removed in future" ), category=AirflowProviderDeprecationWarning, diff --git a/docs/README.rst b/docs/README.rst index d5cddec4864f5..9cb436c0f2fcc 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -258,6 +258,19 @@ warning to report the wrong line in the file for your missing white space. If yo and the line number it reports is wrong, this is a known error. You can find the missing blank space by searching for the syntax you used to make your list, code block, or other whitespace-sensitive markup element. + +spelling error with class or method name +**************************************** +When a spelling error occurs that has a class/function/method name as incorrectly spelled, +instead of whitelisting it in docs/spelling_wordlist.txt you should make sure that +this name is quoted with backticks "`" - this should exclude it from spellchecking process. + +In this example error, You should change the line with error so that the whole path is inside backticks "`". +.. code-block:: text + + Incorrect Spelling: 'BaseAsyncSessionFactory' + Line with Error: ' airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory has been deprecated' + Support ======= diff --git a/docs/apache-airflow-providers/core-extensions/deprecations.rst b/docs/apache-airflow-providers/core-extensions/deprecations.rst new file mode 100644 index 0000000000000..177ea77ab94c6 --- /dev/null +++ b/docs/apache-airflow-providers/core-extensions/deprecations.rst @@ -0,0 +1,31 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Deprecations +========================================== + +This is a summary of deprecated objects in latest versions of Apache Airflow Providers Packages. + +.. note:: + This is an experimental page that may not contain all deprecated entities. + At the moment we only show deprecations for classes, functions, methods, properties etc. + Support for an argument deprecation or an argument value deprecation will be added in the future. + +.. airflow-deprecations:: + :tags: None + :header-separator: " diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py index 0abfc8fe573f3..79a403924ba2a 100644 --- a/docs/exts/docs_build/errors.py +++ b/docs/exts/docs_build/errors.py @@ -77,8 +77,9 @@ def display_errors_summary(build_errors: dict[str, list[DocBuildError]]) -> None console.print( f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)} ({error.line_no})" ) - console.print() - console.print(prepare_code_snippet(error.file_path, error.line_no)) + if os.path.isfile(error.file_path): + console.print() + console.print(prepare_code_snippet(error.file_path, error.line_no)) elif error.file_path: console.print(f"File path: {error.file_path}") console.print() diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py index 2e3f7d409f741..a1d56b77461c0 100644 --- a/docs/exts/docs_build/spelling_checks.py +++ b/docs/exts/docs_build/spelling_checks.py @@ -154,7 +154,9 @@ def display_spelling_error_summary(spelling_errors: dict[str, list[SpellingError console.print("=" * 100) console.print() - msg = """ + msg = """[green] +If there are spelling errors related to class or function name, make sure +those names are quoted with backticks '`' - this should exclude it from spellcheck process. If there are spelling errors in the summary above, and the spelling is correct, add the spelling to docs/spelling_wordlist.txt or use the spelling directive. @@ -177,11 +179,16 @@ def _display_error(error: SpellingError): if error.file_path: console.print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)}") if error.spelling: - console.print(f"Incorrect Spelling: '{error.spelling}'") + console.print(f"[red]Incorrect Spelling: '{error.spelling}'") if error.suggestion: console.print(f"Suggested Spelling: '{error.suggestion}'") if error.context_line: console.print(f"Line with Error: '{error.context_line}'") - if error.file_path and not error.file_path.endswith("") and error.line_no: + if ( + error.file_path + and not error.file_path.endswith("") + and error.line_no + and os.path.isfile(error.file_path) + ): console.print(f"Line Number: {error.line_no}") console.print(prepare_code_snippet(error.file_path, error.line_no)) diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index 272c8224a11d1..413cda59f4637 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -221,6 +221,111 @@ def _render_deferrable_operator_content(*, header_separator: str): ) +def _get_decorator_details(decorator): + def get_full_name(node): + if isinstance(node, ast.Attribute): + return f"{get_full_name(node.value)}.{node.attr}" + elif isinstance(node, ast.Name): + return node.id + else: + return ast.dump(node) + + def eval_node(node): + try: + return ast.literal_eval(node) + except ValueError: + return ast.dump(node) + + if isinstance(decorator, ast.Call): + name = get_full_name(decorator.func) + args = [eval_node(arg) for arg in decorator.args] + kwargs = {kw.arg: eval_node(kw.value) for kw in decorator.keywords if kw.arg != "category"} + return name, args, kwargs + elif isinstance(decorator, ast.Name): + return decorator.id, [], {} + elif isinstance(decorator, ast.Attribute): + return decorator.attr, [], {} + else: + return decorator, [], {} + + +def _iter_module_for_deprecations(ast_node, file_path, class_name=None) -> list[dict[str, Any]]: + deprecations = [] + decorators_of_deprecation = {"deprecated"} + + def analyze_decorators(node, _file_path, object_type, _class_name=None): + for decorator in node.decorator_list: + if str(_class_name).startswith("_") or str(node.name).startswith("_"): + continue + decorator_name, decorator_args, decorator_kwargs = _get_decorator_details(decorator) + + instructions = decorator_kwargs.get("reason", "No instructions were provided.") + if len(decorator_args) == 1 and isinstance(decorator_args[0], str) and not instructions: + instructions = decorator_args[0] + + if decorator_name in ( + "staticmethod", + "classmethod", + "property", + "abstractmethod", + "cached_property", + ): + object_type = decorator_name + + if decorator_name in decorators_of_deprecation: + object_name = f"{_class_name}.{node.name}" if _class_name else node.name + object_path = os.path.join(_file_path, object_name).replace("/", ".").lstrip(".") + deprecations.append( + { + "object_path": object_path, + "object_name": object_name, + "object_type": object_type, + "instructions": instructions, + } + ) + + for child in ast.iter_child_nodes(ast_node): + if isinstance(child, ast.ClassDef): + analyze_decorators(child, file_path, object_type="class") + deprecations.extend(_iter_module_for_deprecations(child, file_path, class_name=child.name)) + elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + analyze_decorators( + child, file_path, _class_name=class_name, object_type="method" if class_name else "function" + ) + else: + deprecations.extend(_iter_module_for_deprecations(child, file_path, class_name=class_name)) + + return deprecations + + +def _render_deprecations_content(*, header_separator: str): + providers = [] + for provider_yaml_path in get_provider_yaml_paths(): + provider_parent_path = Path(provider_yaml_path).parent + provider_info: dict[str, Any] = {"name": "", "deprecations": []} + for root, _, file_names in os.walk(provider_parent_path): + file_names = [f for f in file_names if f.endswith(".py") and f != "__init__.py"] + for file_name in file_names: + file_path = f"{os.path.relpath(root)}/{file_name}" + with open(file_path) as file: + ast_obj = ast.parse(file.read()) + provider_info["deprecations"].extend(_iter_module_for_deprecations(ast_obj, file_path[:-3])) + + if provider_info["deprecations"]: + provider_info["deprecations"] = sorted( + provider_info["deprecations"], key=lambda p: p["object_path"] + ) + provider_yaml_content = yaml.safe_load(Path(provider_yaml_path).read_text()) + provider_info["name"] = provider_yaml_content["package-name"] + providers.append(provider_info) + + return _render_template( + "deprecations.rst.jinja2", + providers=sorted(providers, key=lambda p: p["name"]), + header_separator=header_separator, + ) + + class BaseJinjaReferenceDirective(Directive): """The base directive for OperatorsHooksReferenceDirective and TransfersReferenceDirective""" @@ -399,6 +504,15 @@ def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAU ) +class DeprecationsDirective(BaseJinjaReferenceDirective): + """Generate list of deprecated entities""" + + def render_content(self, *, tags: set[str] | None, header_separator: str = DEFAULT_HEADER_SEPARATOR): + return _render_deprecations_content( + header_separator=header_separator, + ) + + def setup(app): """Setup plugin""" app.add_directive("operators-hooks-ref", OperatorsHooksReferenceDirective) @@ -412,6 +526,7 @@ def setup(app): app.add_directive("airflow-notifications", NotificationsDirective) app.add_directive("airflow-executors", ExecutorsDirective) app.add_directive("airflow-deferrable-operators", DeferrableOperatorDirective) + app.add_directive("airflow-deprecations", DeprecationsDirective) return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/exts/templates/deprecations.rst.jinja2 b/docs/exts/templates/deprecations.rst.jinja2 new file mode 100644 index 0000000000000..f134a58ce9361 --- /dev/null +++ b/docs/exts/templates/deprecations.rst.jinja2 @@ -0,0 +1,37 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} +{%- for provider in providers %} +{{ provider["name"] }} +{{ header_separator * (provider['name']|length) }} + +{% for deprecation in provider['deprecations'] %} + +{% set prefix = ':attr:' %} +{% if deprecation["object_type"] == 'class' %} + {% set prefix = ':class:' %} +{% elif deprecation["object_type"] == 'function' %} + {% set prefix = ':func:' %} +{% elif 'method' in deprecation["object_type"] %} + {% set prefix = ':meth:' %} +{% endif %} + +- {{ prefix }}`{{ deprecation["object_name"] }} <{{ deprecation["object_path"] }}>` - {{ deprecation["instructions"] }} +{% endfor %} + +{% endfor %}