diff --git a/airflow/api/auth/backend/kerberos_auth.py b/airflow/api/auth/backend/kerberos_auth.py index 509b7c6b0a7ea..5306ca6d1ee74 100644 --- a/airflow/api/auth/backend/kerberos_auth.py +++ b/airflow/api/auth/backend/kerberos_auth.py @@ -96,10 +96,7 @@ def init_app(app): def _unauthorized(): - """ - Indicate that authorization is required. - :return: - """ + """Indicate that authorization is required.""" return Response("Unauthorized", 401, {"WWW-Authenticate": "Negotiate"}) diff --git a/airflow/api/client/api_client.py b/airflow/api/client/api_client.py index 222c0f142d263..1fdbc7027b3f7 100644 --- a/airflow/api/client/api_client.py +++ b/airflow/api/client/api_client.py @@ -79,6 +79,7 @@ def delete_pool(self, name): def get_lineage(self, dag_id: str, execution_date: str): """ Return the lineage information for the dag on this execution date. + :param dag_id: :param execution_date: :return: diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index f821a04a3ff48..c6a6949ab48ad 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -269,6 +269,7 @@ def _save_dot_to_file(dot: Dot, filename: str) -> None: def dag_state(args, session: Session = NEW_SESSION) -> None: """ Returns the state (and conf if exists) of a DagRun at the command line. + >>> airflow dags state tutorial 2015-01-01T00:00:00.000000 running >>> airflow dags state a_dag_with_conf_passed 2015-01-01T00:00:00.000000 @@ -290,6 +291,7 @@ def dag_state(args, session: Session = NEW_SESSION) -> None: def dag_next_execution(args) -> None: """ Returns the next execution datetime of a DAG at the command line. + >>> airflow dags next-execution tutorial 2018-08-31 10:38:00 """ @@ -350,7 +352,7 @@ def dag_list_dags(args) -> None: @suppress_logs_and_warning @provide_session def dag_details(args, session=NEW_SESSION): - """Get DAG details given a DAG id""" + """Get DAG details given a DAG id.""" dag = DagModel.get_dagmodel(args.dag_id, session=session) if not dag: raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table") diff --git a/airflow/cli/commands/internal_api_command.py b/airflow/cli/commands/internal_api_command.py index c78ac09e28064..3d8f205bd1e7c 100644 --- a/airflow/cli/commands/internal_api_command.py +++ b/airflow/cli/commands/internal_api_command.py @@ -201,7 +201,7 @@ def monitor_gunicorn(gunicorn_master_pid: int): def create_app(config=None, testing=False): - """Create a new instance of Airflow Internal API app""" + """Create a new instance of Airflow Internal API app.""" flask_app = Flask(__name__) flask_app.config["APP_NAME"] = "Airflow Internal API" @@ -255,7 +255,7 @@ def create_app(config=None, testing=False): def cached_app(config=None, testing=False): - """Return cached instance of Airflow Internal API app""" + """Return cached instance of Airflow Internal API app.""" global app if not app: app = create_app(config=config, testing=testing) diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index 3873a55a66c38..aab8bb10dc790 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -467,6 +467,7 @@ def task_failed_deps(args) -> None: def task_state(args) -> None: """ Returns the state of a TaskInstance at the command line. + >>> airflow tasks state tutorial sleep 2015-01-01 success """ diff --git a/airflow/cli/commands/triggerer_command.py b/airflow/cli/commands/triggerer_command.py index 7bfd77bf8ee2f..2da482187726c 100644 --- a/airflow/cli/commands/triggerer_command.py +++ b/airflow/cli/commands/triggerer_command.py @@ -37,7 +37,7 @@ @contextmanager def _serve_logs(skip_serve_logs: bool = False) -> Generator[None, None, None]: - """Starts serve_logs sub-process""" + """Starts serve_logs sub-process.""" sub_proc = None if skip_serve_logs is False: port = conf.getint("logging", "trigger_log_server_port", fallback=8794) diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py index 78493b8b5574b..daa8370680fef 100644 --- a/airflow/dag_processing/manager.py +++ b/airflow/dag_processing/manager.py @@ -519,7 +519,7 @@ def deactivate_stale_dags( ): """ Detects DAGs which are no longer present in files. - Deactivate them and remove them in the serialized_dag table + Deactivate them and remove them in the serialized_dag table. """ to_deactivate = set() query = session.query(DagModel.dag_id, DagModel.fileloc, DagModel.last_parsed_time).filter( @@ -891,7 +891,7 @@ def _log_file_processing_stats(self, known_file_paths): def get_pid(self, file_path) -> int | None: """ Retrieve the PID of the process processing the given file or None if the file is not being processed. - :param file_path: the path to the file that's being processed + :param file_path: the path to the file that's being processed. """ if file_path in self._processors: return self._processors[file_path].pid @@ -950,7 +950,7 @@ def get_start_time(self, file_path) -> datetime | None: Retrieve the last start time for processing a specific path. :param file_path: the path to the file that's being processed :return: the start time of the process that's processing the - specified file or None if the file is not currently being processed + specified file or None if the file is not currently being processed. """ if file_path in self._processors: return self._processors[file_path].start_time @@ -959,7 +959,7 @@ def get_start_time(self, file_path) -> datetime | None: def get_run_count(self, file_path) -> int: """ The number of times the given file has been parsed. - :param file_path: the path to the file that's being processed + :param file_path: the path to the file that's being processed. """ stat = self._file_stats.get(file_path) return stat.run_count if stat else 0 @@ -1233,7 +1233,7 @@ def _kill_timed_out_processors(self): self._processors.pop(proc) def _add_paths_to_queue(self, file_paths_to_enqueue: list[str], add_at_front: bool): - """Adds stuff to the back or front of the file queue, unless it's already present""" + """Adds stuff to the back or front of the file queue, unless it's already present.""" new_file_paths = list(p for p in file_paths_to_enqueue if p not in self._file_path_queue) if add_at_front: self._file_path_queue.extendleft(new_file_paths) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 613750028a8fc..9cdb8c418a1b1 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -208,7 +208,7 @@ def __init__(self, *args, **kwargs): class DagInvalidTriggerRule(AirflowException): - """Raise when a dag has 'fail_stop' enabled yet has a non-default trigger rule""" + """Raise when a dag has 'fail_stop' enabled yet has a non-default trigger rule.""" @classmethod def check(cls, dag: DAG | None, trigger_rule: str): diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py index f78fa1b19333b..9599adabdfaa4 100644 --- a/airflow/executors/base_executor.py +++ b/airflow/executors/base_executor.py @@ -78,7 +78,7 @@ class RunningRetryAttemptType: @property def elapsed(self): - """Seconds since first attempt""" + """Seconds since first attempt.""" return (pendulum.now("UTC") - self.first_attempt_time).total_seconds() def can_try_again(self): diff --git a/airflow/executors/celery_kubernetes_executor.py b/airflow/executors/celery_kubernetes_executor.py index a4ab1bb804cd7..77dc14753fbbe 100644 --- a/airflow/executors/celery_kubernetes_executor.py +++ b/airflow/executors/celery_kubernetes_executor.py @@ -149,7 +149,7 @@ def queue_task_instance( ) def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]: - """Fetch task log from Kubernetes executor""" + """Fetch task log from Kubernetes executor.""" if ti.queue == self.kubernetes_executor.kubernetes_queue: return self.kubernetes_executor.get_task_log(ti=ti, try_number=try_number) return [], [] diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py index be0fcd00d6375..470df3101a3b8 100644 --- a/airflow/executors/kubernetes_executor.py +++ b/airflow/executors/kubernetes_executor.py @@ -401,7 +401,7 @@ def delete_pod(self, pod_name: str, namespace: str) -> None: raise def patch_pod_executor_done(self, *, pod_name: str, namespace: str): - """Add a "done" annotation to ensure we don't continually adopt pods""" + """Add a "done" annotation to ensure we don't continually adopt pods.""" self.log.debug("Patching pod %s in namespace %s to mark it as done", pod_name, namespace) try: self.kube_client.patch_namespaced_pod( @@ -1004,7 +1004,7 @@ def _flush_result_queue(self) -> None: break def end(self) -> None: - """Called when the executor shuts down""" + """Called when the executor shuts down.""" if TYPE_CHECKING: assert self.task_queue assert self.result_queue diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py index df11d49ae421b..715bdb42ae56a 100644 --- a/airflow/executors/local_executor.py +++ b/airflow/executors/local_executor.py @@ -389,10 +389,7 @@ def sync(self) -> None: self.impl.sync() def end(self) -> None: - """ - Ends the executor. - :return: - """ + """Ends the executor.""" if TYPE_CHECKING: assert self.impl assert self.manager diff --git a/airflow/executors/local_kubernetes_executor.py b/airflow/executors/local_kubernetes_executor.py index 39ca8608bf08b..050132c1a0084 100644 --- a/airflow/executors/local_kubernetes_executor.py +++ b/airflow/executors/local_kubernetes_executor.py @@ -149,7 +149,7 @@ def queue_task_instance( ) def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]: - """Fetch task log from kubernetes executor""" + """Fetch task log from kubernetes executor.""" if ti.queue == self.kubernetes_executor.kubernetes_queue: return self.kubernetes_executor.get_task_log(ti=ti, try_number=try_number) return [], [] diff --git a/airflow/jobs/job.py b/airflow/jobs/job.py index 399fa39f25ebb..0451092bd7289 100644 --- a/airflow/jobs/job.py +++ b/airflow/jobs/job.py @@ -269,8 +269,10 @@ def run_job( job: Job | JobPydantic, execute_callable: Callable[[], int | None], session: Session = NEW_SESSION ) -> int | None: """ - Runs the job. The Job is always an ORM object and setting the state is happening within the - same DB session and the session is kept open throughout the whole execution + Runs the job. + + The Job is always an ORM object and setting the state is happening within the + same DB session and the session is kept open throughout the whole execution. :meta private: diff --git a/airflow/jobs/local_task_job_runner.py b/airflow/jobs/local_task_job_runner.py index 93046696e69d3..8ef03b8154271 100644 --- a/airflow/jobs/local_task_job_runner.py +++ b/airflow/jobs/local_task_job_runner.py @@ -117,7 +117,7 @@ def signal_handler(signum, frame): self.handle_task_exit(128 + signum) def segfault_signal_handler(signum, frame): - """Setting sigmentation violation signal handler""" + """Setting sigmentation violation signal handler.""" self.log.critical(SIGSEGV_MESSAGE) self.task_runner.terminate() self.handle_task_exit(128 + signum) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index e53eeb0f18a85..128a82d7eedb3 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -88,7 +88,7 @@ @dataclass class ConcurrencyMap: """ - Dataclass to represent concurrency maps + Dataclass to represent concurrency maps. It contains a map from (dag_id, task_id) to # of task instances, a map from (dag_id, task_id) to # of task instances in the given state list and a map from (dag_id, run_id, task_id) @@ -1122,7 +1122,7 @@ def _create_dagruns_for_dags(self, guard: CommitProhibitorGuard, session: Sessio def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -> None: """ Unconditionally create a DAG run for the given DAG, and update the dag_model's fields to control - if/when the next DAGRun should be created + if/when the next DAGRun should be created. """ # Bulk Fetch DagRuns with dag_id and execution_date same # as DagModel.dag_id and DagModel.next_dagrun @@ -1343,7 +1343,7 @@ def _schedule_all_dag_runs( dag_runs: Iterable[DagRun], session: Session, ) -> list[tuple[DagRun, DagCallbackRequest | None]]: - """Makes scheduling decisions for all `dag_runs`""" + """Makes scheduling decisions for all `dag_runs`.""" callback_tuples = [(run, self._schedule_dag_run(run, session=session)) for run in dag_runs] guard.commit() return callback_tuples @@ -1739,7 +1739,7 @@ def _set_orphaned(self, dataset: DatasetModel) -> int: def _orphan_unreferenced_datasets(self, session: Session = NEW_SESSION) -> None: """ Detects datasets that are no longer referenced in any DAG schedule parameters or task outlets and - sets the dataset is_orphaned flag to True + sets the dataset is_orphaned flag to True. """ orphaned_dataset_query = ( session.query(DatasetModel) diff --git a/airflow/jobs/triggerer_job_runner.py b/airflow/jobs/triggerer_job_runner.py index c417be1539340..f0295a12b97c0 100644 --- a/airflow/jobs/triggerer_job_runner.py +++ b/airflow/jobs/triggerer_job_runner.py @@ -299,7 +299,7 @@ def is_needed(cls, session) -> bool: def on_kill(self): """ Called when there is an external kill command (via the heartbeat - mechanism, for example) + mechanism, for example). """ self.trigger_runner.stop = True @@ -677,7 +677,7 @@ def update_triggers(self, requested_trigger_ids: set[int]): def set_trigger_logging_metadata(self, ti: TaskInstance, trigger_id, trigger): """ - Set up logging for triggers + Set up logging for triggers. We want to ensure that each trigger logs to its own file and that the log messages are not propagated to parent loggers. diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py index 0c0ec2eac76ed..156bcd0a75049 100644 --- a/airflow/kubernetes/pod_generator.py +++ b/airflow/kubernetes/pod_generator.py @@ -449,7 +449,7 @@ def build_selector_for_k8s_executor_pod( airflow_worker=None, ): """ - Generate selector for kubernetes executor pod + Generate selector for kubernetes executor pod. :meta private: """ @@ -481,7 +481,7 @@ def build_labels_for_k8s_executor_pod( run_id=None, ): """ - Generate labels for kubernetes executor pod + Generate labels for kubernetes executor pod. :meta private: """ diff --git a/airflow/metrics/validators.py b/airflow/metrics/validators.py index 9ed389877f50f..4201d5ddf4b7b 100644 --- a/airflow/metrics/validators.py +++ b/airflow/metrics/validators.py @@ -107,7 +107,7 @@ def __subclasshook__(cls, subclass: Callable[[str], str]) -> bool: @abc.abstractmethod def test(self, name: str) -> bool: - """Test if name is allowed""" + """Test if name is allowed.""" raise NotImplementedError diff --git a/airflow/models/__init__.py b/airflow/models/__init__.py index aa1e2c98ef4c6..45690b5ae0cb0 100644 --- a/airflow/models/__init__.py +++ b/airflow/models/__init__.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Airflow models""" +"""Airflow models.""" from __future__ import annotations # Do not add new models to this -- this is for compat only diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index c8584a9f4a567..42e1cc688e174 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -138,7 +138,7 @@ def inherits_from_empty_operator(self) -> bool: @property def dag_id(self) -> str: - """Returns dag id if it has one or an adhoc + owner""" + """Returns dag id if it has one or an adhoc + owner.""" dag = self.get_dag() if dag: return dag.dag_id @@ -243,7 +243,7 @@ def iter_mapped_dependants(self) -> Iterator[MappedOperator | MappedTaskGroup]: def iter_mapped_task_groups(self) -> Iterator[MappedTaskGroup]: """Return mapped task groups this task belongs to. - Groups are returned from the closest to the outmost. + Groups are returned from the innermost to the outmost. :meta private: """ @@ -254,7 +254,10 @@ def iter_mapped_task_groups(self) -> Iterator[MappedTaskGroup]: parent = parent.task_group def get_closest_mapped_task_group(self) -> MappedTaskGroup | None: - """:meta private:""" + """Get the mapped task group "closest" to this task in the DAG. + + :meta private: + """ return next(self.iter_mapped_task_groups(), None) def unmap(self, resolve: None | dict[str, Any] | tuple[Context, Session]) -> BaseOperator: @@ -296,7 +299,7 @@ def priority_weight_total(self) -> int: @cached_property def operator_extra_link_dict(self) -> dict[str, Any]: - """Returns dictionary of all extra links for the operator""" + """Returns dictionary of all extra links for the operator.""" op_extra_links_from_plugin: dict[str, Any] = {} from airflow import plugins_manager @@ -315,7 +318,7 @@ def operator_extra_link_dict(self) -> dict[str, Any]: @cached_property def global_operator_extra_link_dict(self) -> dict[str, Any]: - """Returns dictionary of all global extra links""" + """Returns dictionary of all global extra links.""" from airflow import plugins_manager plugins_manager.initialize_extra_operators_links_plugins() diff --git a/airflow/models/base.py b/airflow/models/base.py index fc8a9fab7ff69..575eb6e0b1015 100644 --- a/airflow/models/base.py +++ b/airflow/models/base.py @@ -52,7 +52,7 @@ def _get_schema(): def get_id_collation_args(): - """Get SQLAlchemy args to use for COLLATION""" + """Get SQLAlchemy args to use for COLLATION.""" collation = conf.get("database", "sql_engine_collation_for_ids", fallback=None) if collation: return {"collation": collation} diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 06379592d04b3..c4510cd43966b 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -1043,7 +1043,7 @@ def __gt__(self, other): def __lt__(self, other): """ Called for [Inlet] > [Operator] or [Operator] < [Inlet], so that if other is - an attr annotated object it is set as an inlet to this operator + an attr annotated object it is set as an inlet to this operator. """ if not isinstance(other, Iterable): other = [other] @@ -1069,19 +1069,25 @@ def __setattr__(self, key, value): self.set_xcomargs_dependencies() def add_inlets(self, inlets: Iterable[Any]): - """Sets inlets to this operator""" + """Sets inlets to this operator.""" self.inlets.extend(inlets) def add_outlets(self, outlets: Iterable[Any]): - """Defines the outlets of this operator""" + """Defines the outlets of this operator.""" self.outlets.extend(outlets) def get_inlet_defs(self): - """:meta private:""" + """Gets inlet definitions on this task. + + :meta private: + """ return self.inlets def get_outlet_defs(self): - """:meta private:""" + """Gets outlet definitions on this task. + + :meta private: + """ return self.outlets def get_dag(self) -> DAG | None: @@ -1089,7 +1095,7 @@ def get_dag(self) -> DAG | None: @property # type: ignore[override] def dag(self) -> DAG: # type: ignore[override] - """Returns the Operator's DAG if set, otherwise raises an error""" + """Returns the Operator's DAG if set, otherwise raises an error.""" if self._dag: return self._dag else: @@ -1141,7 +1147,7 @@ def has_dag(self): def prepare_for_execution(self) -> BaseOperator: """ Lock task for execution to disable custom action in __setattr__ and - returns a copy of the task + returns a copy of the task. """ other = copy.copy(self) other._lock_for_execution = True @@ -1407,12 +1413,12 @@ def operator_class(self) -> type[BaseOperator]: # type: ignore[override] @property def task_type(self) -> str: - """@property: type of the task""" + """@property: type of the task.""" return self.__class__.__name__ @property def operator_name(self) -> str: - """@property: use a more friendly display name for the operator, if set""" + """@property: use a more friendly display name for the operator, if set.""" try: return self.custom_operator_name # type: ignore except AttributeError: @@ -1430,7 +1436,7 @@ def leaves(self) -> list[BaseOperator]: @property def output(self) -> XComArg: - """Returns reference to XCom pushed by current operator""" + """Returns reference to XCom pushed by current operator.""" from airflow.models.xcom_arg import XComArg return XComArg(operator=self) @@ -1543,7 +1549,7 @@ def serialize_for_task_group(self) -> tuple[DagAttributeTypes, Any]: @property def inherits_from_empty_operator(self): - """Used to determine if an Operator is inherited from EmptyOperator""" + """Used to determine if an Operator is inherited from EmptyOperator.""" # This looks like `isinstance(self, EmptyOperator) would work, but this also # needs to cope when `self` is a Serialized instance of a EmptyOperator or one # of its subclasses (which don't inherit from anything but BaseOperator). @@ -1567,7 +1573,13 @@ def defer( raise TaskDeferred(trigger=trigger, method_name=method_name, kwargs=kwargs, timeout=timeout) def unmap(self, resolve: None | dict[str, Any] | tuple[Context, Session]) -> BaseOperator: - """:meta private:""" + """Get the "normal" operator from the current operator. + + Since a BaseOperator is not mapped to begin with, this simply returns + the original operator. + + :meta private: + """ return self diff --git a/airflow/models/connection.py b/airflow/models/connection.py index 162be8befd07b..a5653412093e9 100644 --- a/airflow/models/connection.py +++ b/airflow/models/connection.py @@ -204,7 +204,7 @@ def _parse_from_uri(self, uri: str): self.extra = json.dumps(query) def get_uri(self) -> str: - """Return connection in URI format""" + """Return connection in URI format.""" if self.conn_type and "_" in self.conn_type: self.log.warning( "Connection schemes (type: %s) shall not contain '_' according to RFC3986.", @@ -313,7 +313,7 @@ def extra(cls): return synonym("_extra", descriptor=property(cls.get_extra, cls.set_extra)) def rotate_fernet_key(self): - """Encrypts data with a new key. See: :ref:`security/fernet`""" + """Encrypts data with a new key. See: :ref:`security/fernet`.""" fernet = get_fernet() if self._password and self.is_encrypted: self._password = fernet.rotate(self._password.encode("utf-8")).decode() @@ -321,7 +321,7 @@ def rotate_fernet_key(self): self._extra = fernet.rotate(self._extra.encode("utf-8")).decode() def get_hook(self, *, hook_params=None): - """Return hook based on conn_type""" + """Return hook based on conn_type.""" from airflow.providers_manager import ProvidersManager hook = ProvidersManager().hooks.get(self.conn_type, None) diff --git a/airflow/models/crypto.py b/airflow/models/crypto.py index d1a2b06a60599..df072665fcbd7 100644 --- a/airflow/models/crypto.py +++ b/airflow/models/crypto.py @@ -27,13 +27,13 @@ class FernetProtocol(Protocol): - """This class is only used for TypeChecking (for IDEs, mypy, etc)""" + """This class is only used for TypeChecking (for IDEs, mypy, etc).""" def decrypt(self, b): - """Decrypt with Fernet""" + """Decrypt with Fernet.""" def encrypt(self, b): - """Encrypt with Fernet""" + """Encrypt with Fernet.""" class NullFernet: diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 9b227db8a5782..56e114310133a 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -214,7 +214,7 @@ def get_dataset_triggered_next_run_info( ) -> dict[str, dict[str, int | str]]: """ Given a list of dag_ids, get string representing how close any that are dataset triggered are - their next run, e.g. "1 of 2 datasets updated" + their next run, e.g. "1 of 2 datasets updated". """ from airflow.models.dataset import DagScheduleDatasetReference, DatasetDagRunQueue as DDRQ, DatasetModel @@ -1128,7 +1128,10 @@ def is_subdag(self) -> bool: @property def full_filepath(self) -> str: - """:meta private:""" + """Full file path to the DAG. + + :meta private: + """ warnings.warn( "DAG.full_filepath is deprecated in favour of fileloc", RemovedInAirflow3Warning, @@ -1230,7 +1233,10 @@ def task_group(self) -> TaskGroup: @property def filepath(self) -> str: - """:meta private:""" + """Relative file path to the DAG. + + :meta private: + """ warnings.warn( "filepath is deprecated, use relative_fileloc instead", RemovedInAirflow3Warning, @@ -1274,7 +1280,7 @@ def allow_future_exec_dates(self) -> bool: def get_concurrency_reached(self, session=NEW_SESSION) -> bool: """ Returns a boolean indicating whether the max_active_tasks limit for this DAG - has been reached + has been reached. """ TI = TaskInstance qry = session.query(func.count(TI.task_id)).filter( @@ -1295,12 +1301,12 @@ def concurrency_reached(self): @provide_session def get_is_active(self, session=NEW_SESSION) -> None: - """Returns a boolean indicating whether this DAG is active""" + """Returns a boolean indicating whether this DAG is active.""" return session.query(DagModel.is_active).filter(DagModel.dag_id == self.dag_id).scalar() @provide_session def get_is_paused(self, session=NEW_SESSION) -> None: - """Returns a boolean indicating whether this DAG is paused""" + """Returns a boolean indicating whether this DAG is paused.""" return session.query(DagModel.is_paused).filter(DagModel.dag_id == self.dag_id).scalar() @property @@ -1362,7 +1368,7 @@ def handle_callback(self, dagrun, success=True, reason=None, session=NEW_SESSION def get_active_runs(self): """ - Returns a list of dag run execution dates currently running + Returns a list of dag run execution dates currently running. :return: List of execution dates """ @@ -1377,7 +1383,7 @@ def get_active_runs(self): @provide_session def get_num_active_runs(self, external_trigger=None, only_running=True, session=NEW_SESSION): """ - Returns the number of active "running" dag runs + Returns the number of active "running" dag runs. :param external_trigger: True for externally triggered active dag runs :param session: @@ -1446,7 +1452,7 @@ def get_dagruns_between(self, start_date, end_date, session=NEW_SESSION): @provide_session def get_latest_execution_date(self, session: Session = NEW_SESSION) -> pendulum.DateTime | None: - """Returns the latest date for which at least one dag run exists""" + """Returns the latest date for which at least one dag run exists.""" return session.query(func.max(DagRun.execution_date)).filter(DagRun.dag_id == self.dag_id).scalar() @property @@ -1461,7 +1467,7 @@ def latest_execution_date(self): @property def subdags(self): - """Returns a list of the subdag objects associated to this DAG""" + """Returns a list of the subdag objects associated to this DAG.""" # Check SubDag for class but don't check class directly from airflow.operators.subdag import SubDagOperator @@ -1516,7 +1522,7 @@ def get_template_env(self, *, force_sandboxed: bool = False) -> jinja2.Environme def set_dependency(self, upstream_task_id, downstream_task_id): """ Simple utility method to set dependency between two tasks that - already have been added to the DAG using add_task() + already have been added to the DAG using add_task(). """ self.get_task(upstream_task_id).set_downstream(self.get_task(downstream_task_id)) @@ -2283,7 +2289,7 @@ def __deepcopy__(self, memo): return result def sub_dag(self, *args, **kwargs): - """This method is deprecated in favor of partial_subset""" + """This method is deprecated in favor of partial_subset.""" warnings.warn( "This method is deprecated and will be removed in a future version. Please use partial_subset", RemovedInAirflow3Warning, @@ -2475,7 +2481,7 @@ def task(self) -> TaskDecoratorCollection: def add_task(self, task: Operator) -> None: """ - Add a task to the DAG + Add a task to the DAG. :param task: the task you want to add """ @@ -2522,7 +2528,7 @@ def add_task(self, task: Operator) -> None: def add_tasks(self, tasks: Iterable[Operator]) -> None: """ - Add a list of tasks to the DAG + Add a list of tasks to the DAG. :param tasks: a lit of tasks you want to add """ @@ -2616,7 +2622,7 @@ def run( run_job(job=job, execute_callable=job_runner._execute) def cli(self): - """Exposes a CLI specific to this DAG""" + """Exposes a CLI specific to this DAG.""" check_cycle(self) from airflow.cli import cli_parser @@ -2650,7 +2656,7 @@ def add_logger_if_needed(ti: TaskInstance): of into a task file. Since this is a local test run, it is much better for the user to see logs in the command line, rather than needing to search for a log file. Args: - ti: The taskinstance that will receive a logger + ti: The taskinstance that will receive a logger. """ format = logging.Formatter("[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s") @@ -2821,7 +2827,7 @@ def bulk_sync_to_db( dags: Collection[DAG], session=NEW_SESSION, ): - """This method is deprecated in favor of bulk_write_to_db""" + """This method is deprecated in favor of bulk_write_to_db.""" warnings.warn( "This method is deprecated and will be removed in a future version. Please use bulk_write_to_db", RemovedInAirflow3Warning, @@ -3069,7 +3075,7 @@ def sync_to_db(self, processor_subdir: str | None = None, session=NEW_SESSION): self.bulk_write_to_db([self], processor_subdir=processor_subdir, session=session) def get_default_view(self): - """This is only there for backward compatible jinja2 templates""" + """This is only there for backward compatible jinja2 templates.""" if self.default_view is None: return conf.get("webserver", "dag_default_view").lower() else: @@ -3080,7 +3086,7 @@ def get_default_view(self): def deactivate_unknown_dags(active_dag_ids, session=NEW_SESSION): """ Given a list of known DAGs, deactivate any other DAGs that are - marked as active in the ORM + marked as active in the ORM. :param active_dag_ids: list of DAG IDs that are active :return: None @@ -3254,7 +3260,7 @@ def __repr__(self): class DagOwnerAttributes(Base): """ Table defining different owner attributes. For example, a link for an owner that will be passed as - a hyperlink to the DAGs view + a hyperlink to the DAGs view. """ __tablename__ = "dag_owner_attributes" @@ -3279,7 +3285,7 @@ def get_all(cls, session) -> dict[str, dict[str, str]]: class DagModel(Base): - """Table containing DAG properties""" + """Table containing DAG properties.""" __tablename__ = "dag" """ @@ -3436,7 +3442,7 @@ def get_is_paused(self, *, session: Session | None = None) -> bool: @provide_session def get_paused_dag_ids(dag_ids: list[str], session: Session = NEW_SESSION) -> set[str]: """ - Given a list of dag_ids, get a set of Paused Dag Ids + Given a list of dag_ids, get a set of Paused Dag Ids. :param dag_ids: List of Dag ids :param session: ORM Session @@ -3455,7 +3461,7 @@ def get_paused_dag_ids(dag_ids: list[str], session: Session = NEW_SESSION) -> se def get_default_view(self) -> str: """ Get the Default DAG View, returns the default config value if DagModel does not - have a value + have a value. """ # This is for backwards-compatibility with old dags that don't have None as default_view return self.default_view or conf.get_mandatory_value("webserver", "dag_default_view").lower() @@ -3585,7 +3591,7 @@ def calculate_dagrun_date_fields( most_recent_dag_run: None | datetime | DataInterval, ) -> None: """ - Calculate ``next_dagrun`` and `next_dagrun_create_after`` + Calculate ``next_dagrun`` and `next_dagrun_create_after``. :param dag: The DAG object :param most_recent_dag_run: DataInterval (or datetime) of most recent run of this dag, or none @@ -3834,6 +3840,7 @@ def _get_or_create_dagrun( """ Create a DAGRun, but only after clearing the previous instance of said dagrun to prevent collisions. This function is only meant for the `dag.test` function as a helper function. + :param dag: Dag to be used to find dagrun :param conf: configuration to pass to newly created dagrun :param start_date: start date of new dagrun, defaults to execution_date diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index 5df74c1f2b0e7..02055b5e94a79 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -61,7 +61,7 @@ class FileLoadStat(NamedTuple): - """Information about single file""" + """Information about single file.""" file: str duration: timedelta @@ -154,7 +154,7 @@ def size(self) -> int: @property def store_serialized_dags(self) -> bool: - """Whether to read dags from DB""" + """Whether to read dags from DB.""" warnings.warn( "The store_serialized_dags property has been deprecated. Use read_dags_from_db instead.", RemovedInAirflow3Warning, @@ -174,7 +174,7 @@ def dag_ids(self) -> list[str]: @provide_session def get_dag(self, dag_id, session: Session = None): """ - Gets the DAG out of the dictionary, and refreshes it if expired + Gets the DAG out of the dictionary, and refreshes it if expired. :param dag_id: DAG ID """ @@ -261,7 +261,7 @@ def get_dag(self, dag_id, session: Session = None): return self.dags.get(dag_id) def _add_dag_from_db(self, dag_id: str, session: Session): - """Add DAG to DagBag from DB""" + """Add DAG to DagBag from DB.""" from airflow.models.serialized_dag import SerializedDagModel row = SerializedDagModel.get(dag_id, session) @@ -590,7 +590,7 @@ def collect_dags_from_db(self): self.dags.update(subdags) def dagbag_report(self): - """Prints a report around DagBag loading stats""" + """Prints a report around DagBag loading stats.""" stats = self.dagbag_stats dag_folder = self.dag_folder duration = sum((o.duration for o in stats), timedelta()).total_seconds() @@ -688,7 +688,7 @@ def sync_to_db(self, processor_subdir: str | None = None, session: Session = NEW @classmethod @provide_session def _sync_perm_for_dag(cls, dag: DAG, session: Session = NEW_SESSION): - """Sync DAG specific permissions""" + """Sync DAG specific permissions.""" root_dag_id = dag.parent_dag.dag_id if dag.parent_dag else dag.dag_id cls.logger().debug("Syncing DAG permissions: %s to the DB", root_dag_id) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index fc403fda5f512..98810693cc1a1 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -77,7 +77,7 @@ class TISchedulingDecision(NamedTuple): - """Type of return for DagRun.task_instance_scheduling_decisions""" + """Type of return for DagRun.task_instance_scheduling_decisions.""" tis: list[TI] schedulable_tis: list[TI] @@ -99,7 +99,7 @@ def _creator_note(val): class DagRun(Base, LoggingMixin): """ DagRun describes an instance of a Dag. It can be created - by the scheduler (for regular runs) or by an external trigger + by the scheduler (for regular runs) or by an external trigger. """ __tablename__ = "dag_run" @@ -265,7 +265,7 @@ def state(self): @provide_session def refresh_from_db(self, session: Session = NEW_SESSION) -> None: """ - Reloads the current dagrun from the database + Reloads the current dagrun from the database. :param session: database session """ @@ -436,7 +436,7 @@ def find_duplicate( @staticmethod def generate_run_id(run_type: DagRunType, execution_date: datetime) -> str: - """Generate Run ID based on Run Type and Execution Date""" + """Generate Run ID based on Run Type and Execution Date.""" # _Ensure_ run_type is a DagRunType, not just a string from user code return DagRunType(run_type).generate_run_id(execution_date) @@ -446,7 +446,7 @@ def get_task_instances( state: Iterable[TaskInstanceState | None] | None = None, session: Session = NEW_SESSION, ) -> list[TI]: - """Returns the task instances for this dag run""" + """Returns the task instances for this dag run.""" tis = ( session.query(TI) .options(joinedload(TI.dag_run)) @@ -483,7 +483,7 @@ def get_task_instance( map_index: int = -1, ) -> TI | None: """ - Returns the task instance specified by task_id for this dag run + Returns the task instance specified by task_id for this dag run. :param task_id: the task id :param session: Sqlalchemy ORM Session @@ -509,7 +509,7 @@ def get_dag(self) -> DAG: def get_previous_dagrun( self, state: DagRunState | None = None, session: Session = NEW_SESSION ) -> DagRun | None: - """The previous DagRun, if there is one""" + """The previous DagRun, if there is one.""" filters = [ DagRun.dag_id == self.dag_id, DagRun.execution_date < self.execution_date, @@ -520,7 +520,7 @@ def get_previous_dagrun( @provide_session def get_previous_scheduled_dagrun(self, session: Session = NEW_SESSION) -> DagRun | None: - """The previous, SCHEDULED DagRun, if there is one""" + """The previous, SCHEDULED DagRun, if there is one.""" return ( session.query(DagRun) .filter( @@ -1120,7 +1120,7 @@ def _create_tasks( session: Session, ) -> CreatedTasks: """ - Create missing tasks -- and expand any MappedOperator that _only_ have literals as input + Create missing tasks -- and expand any MappedOperator that _only_ have literals as input. :param tasks: Tasks to create jobs for in the DAG run :param task_creator: Function to create task instances @@ -1231,7 +1231,7 @@ def _revise_map_indexes_if_mapped(self, task: Operator, *, session: Session) -> @staticmethod def get_run(session: Session, dag_id: str, execution_date: datetime) -> DagRun | None: """ - Get a single DAG Run + Get a single DAG Run. :meta private: :param session: Sqlalchemy ORM Session @@ -1262,7 +1262,7 @@ def is_backfill(self) -> bool: @classmethod @provide_session def get_latest_runs(cls, session: Session = NEW_SESSION) -> list[DagRun]: - """Returns the latest DagRun for each DAG""" + """Returns the latest DagRun for each DAG.""" subquery = ( session.query(cls.dag_id, func.max(cls.execution_date).label("execution_date")) .group_by(cls.dag_id) diff --git a/airflow/models/log.py b/airflow/models/log.py index 33fdeaee070fe..c3cbaeeabc3cf 100644 --- a/airflow/models/log.py +++ b/airflow/models/log.py @@ -25,7 +25,7 @@ class Log(Base): - """Used to actively log events to the database""" + """Used to actively log events to the database.""" __tablename__ = "log" diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py index 345329ef6d4a1..5b8c9455c66d3 100644 --- a/airflow/models/mappedoperator.py +++ b/airflow/models/mappedoperator.py @@ -541,7 +541,7 @@ def get_dag(self) -> DAG | None: @property def output(self) -> XComArg: - """Returns reference to XCom pushed by current operator""" + """Returns reference to XCom pushed by current operator.""" from airflow.models.xcom_arg import XComArg return XComArg(operator=self) diff --git a/airflow/models/param.py b/airflow/models/param.py index 80da2b858e5b5..d9a547b9ad867 100644 --- a/airflow/models/param.py +++ b/airflow/models/param.py @@ -135,7 +135,7 @@ def resolve(self, value: Any = NOTSET, suppress_exception: bool = False) -> Any: return final_val def dump(self) -> dict: - """Dump the Param as a dictionary""" + """Dump the Param as a dictionary.""" out_dict = {self.CLASS_IDENTIFIER: f"{self.__module__}.{self.__class__.__name__}"} out_dict.update(self.__dict__) return out_dict @@ -245,7 +245,7 @@ def __getitem__(self, key: str) -> Any: return param.resolve(suppress_exception=self.suppress_exception) def get_param(self, key: str) -> Param: - """Get the internal :class:`.Param` object for this key""" + """Get the internal :class:`.Param` object for this key.""" return self.__dict[key] def items(self): @@ -260,11 +260,11 @@ def update(self, *args, **kwargs) -> None: super().update(*args, **kwargs) def dump(self) -> dict[str, Any]: - """Dumps the ParamsDict object as a dictionary, while suppressing exceptions""" + """Dumps the ParamsDict object as a dictionary, while suppressing exceptions.""" return {k: v.resolve(suppress_exception=True) for k, v in self.items()} def validate(self) -> dict[str, Any]: - """Validates & returns all the Params object stored in the dictionary""" + """Validates & returns all the Params object stored in the dictionary.""" resolved_dict = {} try: for k, v in self.items(): diff --git a/airflow/models/pool.py b/airflow/models/pool.py index e7376c31eac1b..d1766d4a0a688 100644 --- a/airflow/models/pool.py +++ b/airflow/models/pool.py @@ -32,7 +32,7 @@ class PoolStats(TypedDict): - """Dictionary containing Pool Stats""" + """Dictionary containing Pool Stats.""" total: int running: int @@ -149,7 +149,7 @@ def slots_stats( session: Session = NEW_SESSION, ) -> dict[str, PoolStats]: """ - Get Pool stats (Number of Running, Queued, Open & Total tasks) + Get Pool stats (Number of Running, Queued, Open & Total tasks). If ``lock_rows`` is True, and the database engine in use supports the ``NOWAIT`` syntax, then a non-blocking lock will be attempted -- if the lock is not available then SQLAlchemy will throw an @@ -203,7 +203,7 @@ def slots_stats( def to_json(self) -> dict[str, Any]: """ - Get the Pool in a json structure + Get the Pool in a json structure. :return: the pool object in json format """ diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 7fd2c29edfa61..1de9807e948c1 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Save Rendered Template Fields""" +"""Save Rendered Template Fields.""" from __future__ import annotations import os @@ -40,7 +40,7 @@ class RenderedTaskInstanceFields(Base): - """Save Rendered Template Fields""" + """Save Rendered Template Fields.""" __tablename__ = "rendered_task_instance_fields" @@ -176,7 +176,7 @@ def get_k8s_pod_yaml(cls, ti: TaskInstance, session: Session = NEW_SESSION) -> d @provide_session def write(self, session: Session = None): - """Write instance to database + """Write instance to database. :param session: SqlAlchemy Session """ diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index d3f49c64d3950..d81df7773f2ee 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -211,7 +211,7 @@ def data(self) -> dict | None: @property def dag(self) -> SerializedDAG: - """The DAG deserialized from the ``data`` column""" + """The DAG deserialized from the ``data`` column.""" SerializedDAG._load_operator_extra_links = self.load_op_links if isinstance(self.data, dict): data = self.data @@ -226,7 +226,7 @@ def dag(self) -> SerializedDAG: def remove_dag(cls, dag_id: str, session: Session = NEW_SESSION) -> None: """Deletes a DAG with given dag_id. :param dag_id: dag_id to be deleted - :param session: ORM Session + :param session: ORM Session. """ session.execute(cls.__table__.delete().where(cls.dag_id == dag_id)) @@ -330,7 +330,7 @@ def bulk_sync_to_db( def get_last_updated_datetime(cls, dag_id: str, session: Session = NEW_SESSION) -> datetime | None: """ Get the date when the Serialized DAG associated to DAG was last updated - in serialized_dag table + in serialized_dag table. :param dag_id: DAG ID :param session: ORM Session @@ -341,7 +341,7 @@ def get_last_updated_datetime(cls, dag_id: str, session: Session = NEW_SESSION) @provide_session def get_max_last_updated_datetime(cls, session: Session = NEW_SESSION) -> datetime | None: """ - Get the maximum date when any DAG was last updated in serialized_dag table + Get the maximum date when any DAG was last updated in serialized_dag table. :param session: ORM Session """ @@ -381,7 +381,7 @@ def get_latest_version_hash_and_updated_datetime( @provide_session def get_dag_dependencies(cls, session: Session = NEW_SESSION) -> dict[str, list[DagDependency]]: """ - Get the dependencies between DAGs + Get the dependencies between DAGs. :param session: ORM Session """ diff --git a/airflow/models/skipmixin.py b/airflow/models/skipmixin.py index facc43bc8f2d7..4197a808c6acb 100644 --- a/airflow/models/skipmixin.py +++ b/airflow/models/skipmixin.py @@ -55,7 +55,7 @@ def _ensure_tasks(nodes: Iterable[DAGNode]) -> Sequence[Operator]: class SkipMixin(LoggingMixin): - """A Mixin to skip Tasks Instances""" + """A Mixin to skip Tasks Instances.""" def _set_state_to_skipped( self, diff --git a/airflow/models/taskfail.py b/airflow/models/taskfail.py index ead4cee000f28..7ae459ab8c433 100644 --- a/airflow/models/taskfail.py +++ b/airflow/models/taskfail.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Taskfail tracks the failed run durations of each task instance""" +"""Taskfail tracks the failed run durations of each task instance.""" from __future__ import annotations from sqlalchemy import Column, ForeignKeyConstraint, Index, Integer, text diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index d392c604f073f..bb7175a4a330e 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -551,7 +551,10 @@ def stats_tags(self) -> dict[str, str]: @staticmethod def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any]: - """:meta private:""" + """Insert mapping. + + :meta private: + """ return { "dag_id": task.dag_id, "task_id": task.task_id, @@ -572,7 +575,7 @@ def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any @reconstructor def init_on_load(self) -> None: - """Initialize the attributes that aren't stored in the DB""" + """Initialize the attributes that aren't stored in the DB.""" # correctly config the ti log self._log = logging.getLogger("airflow.task") self.test_mode = False # can be changed when calling 'run' @@ -752,7 +755,7 @@ def generate_command( @property def log_url(self) -> str: - """Log URL for TaskInstance""" + """Log URL for TaskInstance.""" iso = quote(self.execution_date.isoformat()) base_url = conf.get_mandatory_value("webserver", "BASE_URL") return ( @@ -765,7 +768,7 @@ def log_url(self) -> str: @property def mark_success_url(self) -> str: - """URL to mark TI success""" + """URL to mark TI success.""" base_url = conf.get_mandatory_value("webserver", "BASE_URL") return ( f"{base_url}/confirm" @@ -807,7 +810,7 @@ def error(self, session: Session = NEW_SESSION) -> None: @provide_session def refresh_from_db(self, session: Session = NEW_SESSION, lock_for_update: bool = False) -> None: """ - Refreshes the task instance from the database based on the primary key + Refreshes the task instance from the database based on the primary key. :param session: SQLAlchemy ORM Session :param lock_for_update: if True, indicates that the database should @@ -908,7 +911,7 @@ def clear_xcom_data(self, session: Session = NEW_SESSION) -> None: @property def key(self) -> TaskInstanceKey: - """Returns a tuple that identifies the task instance uniquely""" + """Returns a tuple that identifies the task instance uniquely.""" return TaskInstanceKey(self.dag_id, self.task_id, self.run_id, self.try_number, self.map_index) @provide_session @@ -1133,7 +1136,7 @@ def are_dependencies_met( @provide_session def get_failed_dep_statuses(self, dep_context: DepContext | None = None, session: Session = NEW_SESSION): - """Get failed Dependencies""" + """Get failed Dependencies.""" dep_context = dep_context or DepContext() for dep in dep_context.deps | self.task.deps: for dep_status in dep.get_dep_statuses(self, session, dep_context): @@ -1205,7 +1208,7 @@ def ready_for_retry(self) -> bool: @provide_session def get_dagrun(self, session: Session = NEW_SESSION) -> DagRun: """ - Returns the DagRun for this TaskInstance + Returns the DagRun for this TaskInstance. :param session: SQLAlchemy ORM Session :return: DagRun @@ -1245,7 +1248,7 @@ def check_and_change_state_before_execution( """ Checks dependencies and then sets state to RUNNING if they are met. Returns True if and only if state is set to RUNNING, which implies that task should be - executed, in preparation for _run_raw_task + executed, in preparation for _run_raw_task. :param verbose: whether to turn on more verbose logging :param ignore_all_deps: Ignore all of the non-critical dependencies, just runs @@ -1577,7 +1580,7 @@ def _register_dataset_changes(self, *, session: Session) -> None: ) def _execute_task_with_callbacks(self, context, test_mode=False): - """Prepare Task for Execution""" + """Prepare Task for Execution.""" from airflow.models.renderedtifields import RenderedTaskInstanceFields parent_pid = os.getpid() @@ -1646,7 +1649,7 @@ def _run_finished_callback( context: Context, callback_type: str, ) -> None: - """Run callback after task finishes""" + """Run callback after task finishes.""" if callbacks: callbacks = callbacks if isinstance(callbacks, list) else [callbacks] for callback in callbacks: @@ -1659,7 +1662,7 @@ def _run_finished_callback( ) def _execute_task(self, context, task_orig): - """Executes Task (optionally with a Timeout) and pushes Xcom results""" + """Executes Task (optionally with a Timeout) and pushes Xcom results.""" task_to_execute = self.task # If the task has been deferred and is being executed due to a trigger, # then we need to pick the right method to come back to, otherwise @@ -1752,7 +1755,7 @@ def _defer_task(self, session: Session, defer: TaskDeferred) -> None: self.trigger_timeout = self.start_date + execution_timeout def _run_execute_callback(self, context: Context, task: Operator) -> None: - """Functions that need to be run before a Task is executed""" + """Functions that need to be run before a Task is executed.""" callbacks = task.on_execute_callback if callbacks: callbacks = callbacks if isinstance(callbacks, list) else [callbacks] @@ -1777,7 +1780,7 @@ def run( pool: str | None = None, session: Session = NEW_SESSION, ) -> None: - """Run TaskInstance""" + """Run TaskInstance.""" res = self.check_and_change_state_before_execution( verbose=verbose, ignore_all_deps=ignore_all_deps, @@ -1799,7 +1802,7 @@ def run( ) def dry_run(self) -> None: - """Only Renders Templates for the TI""" + """Only Renders Templates for the TI.""" from airflow.models.baseoperator import BaseOperator self.task = self.task.prepare_for_execution() @@ -1862,7 +1865,7 @@ def _handle_reschedule( @staticmethod def get_truncated_error_traceback(error: BaseException, truncate_to: Callable) -> TracebackType | None: """ - Truncates the traceback of an exception to the first frame called from within a given function + Truncates the traceback of an exception to the first frame called from within a given function. :param error: exception to get traceback from :param truncate_to: Function to truncate TB to. Must have a ``__code__`` attribute @@ -1886,7 +1889,7 @@ def handle_failure( force_fail: bool = False, session: Session = NEW_SESSION, ) -> None: - """Handle Failure for the TaskInstance""" + """Handle Failure for the TaskInstance.""" if test_mode is None: test_mode = self.test_mode @@ -1976,7 +1979,7 @@ def handle_failure( session.flush() def is_eligible_to_retry(self): - """Is task instance is eligible for retry""" + """Is task instance is eligible for retry.""" if self.state == State.RESTARTING: # If a task is cleared when running, it goes into RESTARTING state and is always # eligible for retry @@ -1992,7 +1995,7 @@ def get_template_context( session: Session | None = None, ignore_param_exceptions: bool = True, ) -> Context: - """Return TI Context""" + """Return TI Context.""" # Do not use provide_session here -- it expunges everything on exit! if not session: session = settings.Session() @@ -2224,7 +2227,7 @@ def get_rendered_template_fields(self, session: Session = NEW_SESSION) -> None: @provide_session def get_rendered_k8s_spec(self, session: Session = NEW_SESSION): - """Fetch rendered template fields from DB""" + """Fetch rendered template fields from DB.""" from airflow.models.renderedtifields import RenderedTaskInstanceFields rendered_k8s_spec = RenderedTaskInstanceFields.get_k8s_pod_yaml(self, session=session) @@ -2236,7 +2239,7 @@ def get_rendered_k8s_spec(self, session: Session = NEW_SESSION): return rendered_k8s_spec def overwrite_params_with_dag_run_conf(self, params, dag_run): - """Overwrite Task Params with DagRun.conf""" + """Overwrite Task Params with DagRun.conf.""" if dag_run and dag_run.conf: self.log.debug("Updating task params (%s) with DagRun.conf (%s)", params, dag_run.conf) params.update(dag_run.conf) @@ -2261,7 +2264,7 @@ def render_templates(self, context: Context | None = None) -> Operator: return original_task def render_k8s_pod_yaml(self) -> dict | None: - """Render k8s pod yaml""" + """Render k8s pod yaml.""" from kubernetes.client.api_client import ApiClient from airflow.kubernetes.kube_config import KubeConfig @@ -2377,7 +2380,7 @@ def email_alert(self, exception, task: BaseOperator) -> None: send_email(task.email, subject, html_content_err) def set_duration(self) -> None: - """Set TI duration""" + """Set TI duration.""" if self.end_date and self.start_date: self.duration = (self.end_date - self.start_date).total_seconds() else: @@ -2540,7 +2543,7 @@ def xcom_pull( @provide_session def get_num_running_task_instances(self, session: Session, same_dagrun=False) -> int: - """Return Number of running TIs from the DB""" + """Return Number of running TIs from the DB.""" # .count() is inefficient num_running_task_instances_query = session.query(func.count()).filter( TaskInstance.dag_id == self.dag_id, @@ -2558,7 +2561,7 @@ def init_run_context(self, raw: bool = False) -> None: @staticmethod def filter_for_tis(tis: Iterable[TaskInstance | TaskInstanceKey]) -> BooleanClauseList | None: - """Returns SQLAlchemy filter to query selected task instances""" + """Returns SQLAlchemy filter to query selected task instances.""" # DictKeys type, (what we often pass here from the scheduler) is not directly indexable :( # Or it might be a generator, but we need to be able to iterate over it more than once tis = list(tis) @@ -2649,7 +2652,7 @@ def filter_for_tis(tis: Iterable[TaskInstance | TaskInstanceKey]) -> BooleanClau def ti_selector_condition(cls, vals: Collection[str | tuple[str, int]]) -> ColumnOperators: """ Build an SQLAlchemy filter for a list where each element can contain - whether a task_id, or a tuple of (task_id,map_index) + whether a task_id, or a tuple of (task_id,map_index). :meta private: """ @@ -2674,7 +2677,8 @@ def ti_selector_condition(cls, vals: Collection[str | tuple[str, int]]) -> Colum @provide_session def schedule_downstream_tasks(self, session: Session = NEW_SESSION, max_tis_per_query: int | None = None): """ - The mini-scheduler for scheduling downstream tasks of this task instance + The mini-scheduler for scheduling downstream tasks of this task instance. + :meta: private """ from sqlalchemy.exc import OperationalError diff --git a/airflow/models/taskinstancekey.py b/airflow/models/taskinstancekey.py index b34aaffd85550..47ed1d4d5422f 100644 --- a/airflow/models/taskinstancekey.py +++ b/airflow/models/taskinstancekey.py @@ -31,18 +31,18 @@ class TaskInstanceKey(NamedTuple): @property def primary(self) -> tuple[str, str, str, int]: - """Return task instance primary key part of the key""" + """Return task instance primary key part of the key.""" return self.dag_id, self.task_id, self.run_id, self.map_index @property def reduced(self) -> TaskInstanceKey: - """Remake the key by subtracting 1 from try number to match in memory information""" + """Remake the key by subtracting 1 from try number to match in memory information.""" return TaskInstanceKey( self.dag_id, self.task_id, self.run_id, max(1, self.try_number - 1), self.map_index ) def with_try_number(self, try_number: int) -> TaskInstanceKey: - """Returns TaskInstanceKey with provided ``try_number``""" + """Returns TaskInstanceKey with provided ``try_number``.""" return TaskInstanceKey(self.dag_id, self.task_id, self.run_id, try_number, self.map_index) @property diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py index d93879498be2c..f0af40e88be70 100644 --- a/airflow/models/taskmixin.py +++ b/airflow/models/taskmixin.py @@ -78,12 +78,12 @@ def update_relative( """ def __lshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): - """Implements Task << Task""" + """Implements Task << Task.""" self.set_upstream(other) return other def __rshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): - """Implements Task >> Task""" + """Implements Task >> Task.""" self.set_downstream(other) return other @@ -99,7 +99,10 @@ def __rlshift__(self, other: DependencyMixin | Sequence[DependencyMixin]): class TaskMixin(DependencyMixin): - """:meta private:""" + """Mixin to provide task-related things. + + :meta private: + """ def __init_subclass__(cls) -> None: warnings.warn( @@ -143,7 +146,7 @@ def has_dag(self) -> bool: @property def dag_id(self) -> str: - """Returns dag id if it has one or an adhoc/meaningless ID""" + """Returns dag id if it has one or an adhoc/meaningless ID.""" if self.dag: return self.dag.dag_id return "_in_memory_dag_" @@ -206,7 +209,7 @@ def _set_relatives( self.dag = dag def add_only_new(obj, item_set: set[str], item: str) -> None: - """Adds only new items to item set""" + """Adds only new items to item set.""" if item in item_set: self.log.warning("Dependency %s, %s already registered for DAG: %s", obj, item, dag.dag_id) else: @@ -245,14 +248,14 @@ def set_upstream( @property def downstream_list(self) -> Iterable[Operator]: - """List of nodes directly downstream""" + """List of nodes directly downstream.""" if not self.dag: raise AirflowException(f"Operator {self} has not been assigned to a DAG yet") return [self.dag.get_task(tid) for tid in self.downstream_task_ids] @property def upstream_list(self) -> Iterable[Operator]: - """List of nodes directly upstream""" + """List of nodes directly upstream.""" if not self.dag: raise AirflowException(f"Operator {self} has not been assigned to a DAG yet") return [self.dag.get_task(tid) for tid in self.upstream_task_ids] diff --git a/airflow/models/trigger.py b/airflow/models/trigger.py index c69ac89ff2e23..a2791f0c27d6d 100644 --- a/airflow/models/trigger.py +++ b/airflow/models/trigger.py @@ -95,7 +95,7 @@ def from_object(cls, trigger: BaseTrigger) -> Trigger: def bulk_fetch(cls, ids: Iterable[int], session: Session = NEW_SESSION) -> dict[int, Trigger]: """ Fetches all the Triggers by ID and returns a dict mapping - ID -> Trigger instance + ID -> Trigger instance. """ query = ( session.query(cls) @@ -114,7 +114,7 @@ def bulk_fetch(cls, ids: Iterable[int], session: Session = NEW_SESSION) -> dict[ def clean_unused(cls, session: Session = NEW_SESSION) -> None: """ Deletes all triggers that have no tasks/DAGs dependent on them - (triggers have a one-to-many relationship to both) + (triggers have a one-to-many relationship to both). """ # Update all task instances with trigger IDs that are not DEFERRED to remove them for attempt in run_with_db_retries(): diff --git a/airflow/models/variable.py b/airflow/models/variable.py index 5e5a42887c0cf..265959255769d 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -68,7 +68,7 @@ def __repr__(self): return f"{self.key} : {self._val}" def get_val(self): - """Get Airflow Variable from Metadata DB and decode it using the Fernet Key""" + """Get Airflow Variable from Metadata DB and decode it using the Fernet Key.""" from cryptography.fernet import InvalidToken as InvalidFernetToken if self._val is not None and self.is_encrypted: @@ -93,7 +93,7 @@ def set_val(self, value): @declared_attr def val(cls): - """Get Airflow Variable from Metadata DB and decode it using the Fernet Key""" + """Get Airflow Variable from Metadata DB and decode it using the Fernet Key.""" return synonym("_val", descriptor=property(cls.get_val, cls.set_val)) @classmethod @@ -128,7 +128,7 @@ def get( deserialize_json: bool = False, ) -> Any: """ - Gets a value for an Airflow Variable Key + Gets a value for an Airflow Variable Key. :param key: Variable Key :param default_var: Default value of the Variable if the Variable doesn't exist @@ -190,7 +190,7 @@ def update( session: Session = None, ): """ - Updates a given Airflow Variable with the Provided value + Updates a given Airflow Variable with the Provided value. :param key: Variable Key :param value: Value to set for the Variable @@ -213,7 +213,7 @@ def update( @internal_api_call def delete(key: str, session: Session = None) -> int: """ - Delete an Airflow Variable for a given key + Delete an Airflow Variable for a given key. :param key: Variable Key :param session: SQL Alchemy Sessions @@ -221,7 +221,7 @@ def delete(key: str, session: Session = None) -> int: return session.query(Variable).filter(Variable.key == key).delete() def rotate_fernet_key(self): - """Rotate Fernet Key""" + """Rotate Fernet Key.""" fernet = get_fernet() if self._val and self.is_encrypted: self._val = fernet.rotate(self._val.encode("utf-8")).decode() diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py index f9e5abc77466b..826172f4f5a97 100644 --- a/airflow/models/xcom.py +++ b/airflow/models/xcom.py @@ -172,7 +172,10 @@ def set( execution_date: datetime.datetime, session: Session = NEW_SESSION, ) -> None: - """:sphinx-autoapi-skip:""" + """Store an XCom value. + + :sphinx-autoapi-skip: + """ @classmethod @provide_session @@ -188,7 +191,10 @@ def set( run_id: str | None = None, map_index: int = -1, ) -> None: - """:sphinx-autoapi-skip:""" + """Store an XCom value. + + :sphinx-autoapi-skip: + """ from airflow.models.dagrun import DagRun if not exactly_one(execution_date is not None, run_id is not None): @@ -350,7 +356,10 @@ def get_one( include_prior_dates: bool = False, session: Session = NEW_SESSION, ) -> Any | None: - """:sphinx-autoapi-skip:""" + """Retrieve an XCom value, optionally meeting certain criteria. + + :sphinx-autoapi-skip: + """ @staticmethod @provide_session @@ -366,7 +375,10 @@ def get_one( run_id: str | None = None, map_index: int | None = None, ) -> Any | None: - """:sphinx-autoapi-skip:""" + """Retrieve an XCom value, optionally meeting certain criteria. + + :sphinx-autoapi-skip: + """ if not exactly_one(execution_date is not None, run_id is not None): raise ValueError("Exactly one of run_id or execution_date must be passed") @@ -456,7 +468,10 @@ def get_many( limit: int | None = None, session: Session = NEW_SESSION, ) -> Query: - """:sphinx-autoapi-skip:""" + """Composes a query to get one or more XCom entries. + + :sphinx-autoapi-skip: + """ @staticmethod @provide_session @@ -473,7 +488,10 @@ def get_many( *, run_id: str | None = None, ) -> Query: - """:sphinx-autoapi-skip:""" + """Composes a query to get one or more XCom entries. + + :sphinx-autoapi-skip: + """ from airflow.models.dagrun import DagRun if not exactly_one(execution_date is not None, run_id is not None): @@ -571,7 +589,10 @@ def clear( task_id: str, session: Session = NEW_SESSION, ) -> None: - """:sphinx-autoapi-skip:""" + """Clear all XCom data from the database for the given task instance. + + :sphinx-autoapi-skip: + """ @staticmethod @provide_session @@ -585,7 +606,10 @@ def clear( run_id: str | None = None, map_index: int | None = None, ) -> None: - """:sphinx-autoapi-skip:""" + """Clear all XCom data from the database for the given task instance. + + :sphinx-autoapi-skip: + """ from airflow.models import DagRun # Given the historic order of this function (execution_date was first argument) to add a new optional @@ -662,7 +686,7 @@ def _deserialize_value(result: XCom, orm: bool) -> Any: @staticmethod def deserialize_value(result: XCom) -> Any: - """Deserialize XCom value from str or pickle object""" + """Deserialize XCom value from str or pickle object.""" return BaseXCom._deserialize_value(result, False) def orm_deserialize_value(self) -> Any: @@ -811,7 +835,7 @@ def _shim(**kwargs): def _get_function_params(function) -> list[str]: """ - Returns the list of variables names of a function + Returns the list of variables names of a function. :param function: The function to inspect """ @@ -823,7 +847,7 @@ def _get_function_params(function) -> list[str]: def resolve_xcom_backend() -> type[BaseXCom]: - """Resolves custom XCom class + """Resolves custom XCom class. Confirms that custom XCom class extends the BaseXCom. Compares the function signature of the custom XCom serialize_value to the base XCom serialize_value. diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py index d8b42ba81919d..85dbce98d4ea9 100644 --- a/airflow/models/xcom_arg.py +++ b/airflow/models/xcom_arg.py @@ -125,12 +125,12 @@ def apply_upstream_relationship(op: Operator, arg: Any): @property def roots(self) -> list[DAGNode]: - """Required by TaskMixin""" + """Required by TaskMixin.""" return [op for op, _ in self.iter_references()] @property def leaves(self) -> list[DAGNode]: - """Required by TaskMixin""" + """Required by TaskMixin.""" return [op for op, _ in self.iter_references()] def set_upstream( @@ -243,7 +243,7 @@ def __eq__(self, other: Any) -> bool: return self.operator == other.operator and self.key == other.key def __getitem__(self, item: str) -> XComArg: - """Implements xcomresult['some_result_key']""" + """Implements xcomresult['some_result_key'].""" if not isinstance(item, str): raise ValueError(f"XComArg only supports str lookup, received {type(item).__name__}") return PlainXComArg(operator=self.operator, key=item) @@ -269,7 +269,7 @@ def __repr__(self) -> str: def __str__(self) -> str: """ - Backward compatibility for old-style jinja used in Airflow Operators + Backward compatibility for old-style jinja used in Airflow Operators. **Example**: to use XComArg at BashOperator:: diff --git a/airflow/notifications/basenotifier.py b/airflow/notifications/basenotifier.py index 629927ce7d209..5f922812d43b7 100644 --- a/airflow/notifications/basenotifier.py +++ b/airflow/notifications/basenotifier.py @@ -30,7 +30,7 @@ class BaseNotifier(Templater): - """BaseNotifier class for sending notifications""" + """BaseNotifier class for sending notifications.""" template_fields: Sequence[str] = () template_ext: Sequence[str] = () @@ -41,7 +41,7 @@ def __init__(self): def _update_context(self, context: Context) -> Context: """ - Add additional context to the context + Add additional context to the context. :param context: The airflow context """ @@ -73,7 +73,7 @@ def render_template_fields( @abstractmethod def notify(self, context: Context) -> None: """ - Sends a notification + Sends a notification. :param context: The airflow context """ @@ -81,7 +81,7 @@ def notify(self, context: Context) -> None: def __call__(self, context: Context) -> None: """ - Send a notification + Send a notification. :param context: The airflow context """ diff --git a/airflow/operators/datetime.py b/airflow/operators/datetime.py index 620ecdf344c32..f56fc2d3d398a 100644 --- a/airflow/operators/datetime.py +++ b/airflow/operators/datetime.py @@ -30,7 +30,7 @@ class BranchDateTimeOperator(BaseBranchOperator): """ Branches into one of two lists of tasks depending on the current datetime. For more information on how to use this operator, take a look at the guide: - :ref:`howto/operator:BranchDateTimeOperator` + :ref:`howto/operator:BranchDateTimeOperator`. True branch will be returned when ``datetime.datetime.now()`` falls below ``target_upper`` and above ``target_lower``. diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 50895994266d5..7a632114c9da9 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -54,7 +54,7 @@ def task(python_callable: Callable | None = None, multiple_outputs: bool | None """ Deprecated function. Calls @task.python and allows users to turn a python function into - an Airflow task. Please use the following instead: + an Airflow task. Please use the following instead. from airflow.decorators import task diff --git a/airflow/operators/subdag.py b/airflow/operators/subdag.py index b46041123be6f..52f4b274032f3 100644 --- a/airflow/operators/subdag.py +++ b/airflow/operators/subdag.py @@ -132,6 +132,7 @@ def _reset_dag_run_and_task_instances(self, dag_run, execution_date): Set task instance states to allow for execution. Set the DagRun state to RUNNING and set the failed TaskInstances to None state for scheduler to pick up. + :param dag_run: DAG run :param execution_date: Execution date :return: None diff --git a/airflow/operators/weekday.py b/airflow/operators/weekday.py index 2b4e0f4698df6..d35204d31cbdf 100644 --- a/airflow/operators/weekday.py +++ b/airflow/operators/weekday.py @@ -30,7 +30,8 @@ class BranchDayOfWeekOperator(BaseBranchOperator): """ Branches into one of two lists of tasks depending on the current day. - For more information on how to use this operator, take a look at the guide: + For more information on how to use this operator, take a look at the guide. + :ref:`howto/operator:BranchDayOfWeekOperator` **Example** (with single day): :: diff --git a/airflow/policies.py b/airflow/policies.py index 8820c869ac6a3..c35ae827eeaa1 100644 --- a/airflow/policies.py +++ b/airflow/policies.py @@ -116,7 +116,10 @@ def get_dagbag_import_timeout(dag_file_path: str) -> int | float: # type: ignor class DefaultPolicy: - """:meta private:""" + """Default implementations of the policy functions. + + :meta private: + """ # Default implementations of the policy functions diff --git a/airflow/secrets/environment_variables.py b/airflow/secrets/environment_variables.py index 8cc7b3e11d629..8e62e9f891d20 100644 --- a/airflow/secrets/environment_variables.py +++ b/airflow/secrets/environment_variables.py @@ -34,7 +34,9 @@ class EnvironmentVariablesBackend(BaseSecretsBackend): def get_conn_uri(self, conn_id: str) -> str | None: """ Return URI representation of Connection conn_id. + :param conn_id: the connection id + :return: deserialized Connection """ warnings.warn( diff --git a/airflow/security/utils.py b/airflow/security/utils.py index 6ce61e36a74d5..46c31c2d74ac3 100644 --- a/airflow/security/utils.py +++ b/airflow/security/utils.py @@ -43,7 +43,7 @@ def get_components(principal) -> list[str] | None: """ Returns components retrieved from the kerberos principal. - -> (short name, instance (FQDN), realm) + -> (short name, instance (FQDN), realm). ``principal`` . """ diff --git a/airflow/serialization/helpers.py b/airflow/serialization/helpers.py index e9db8bc0c947e..07481c58dc355 100644 --- a/airflow/serialization/helpers.py +++ b/airflow/serialization/helpers.py @@ -26,7 +26,7 @@ def serialize_template_field(template_field: Any) -> str | dict | list | int | f """ Return a serializable representation of the templated_field. If a templated_field contains a Class or Instance for recursive templating, store them - as strings. If the templated_field is not recursive return the field + as strings. If the templated_field is not recursive return the field. :param template_field: Task's Templated Field """ diff --git a/airflow/serialization/pydantic/dag_run.py b/airflow/serialization/pydantic/dag_run.py index 16d57a84ea831..b7ebceb82670d 100644 --- a/airflow/serialization/pydantic/dag_run.py +++ b/airflow/serialization/pydantic/dag_run.py @@ -46,6 +46,6 @@ class DagRunPydantic(BaseModelPydantic): consumed_dataset_events: List[DatasetEventPydantic] class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py index 39c552eea9234..f3752f4ade1f6 100644 --- a/airflow/serialization/pydantic/dataset.py +++ b/airflow/serialization/pydantic/dataset.py @@ -32,7 +32,7 @@ class DagScheduleDatasetReferencePydantic(BaseModelPydantic): updated_at: datetime class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True @@ -50,7 +50,7 @@ class TaskOutletDatasetReferencePydantic(BaseModelPydantic): updated_at = datetime class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True @@ -69,7 +69,7 @@ class DatasetPydantic(BaseModelPydantic): producing_tasks: List[TaskOutletDatasetReferencePydantic] class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True @@ -87,6 +87,6 @@ class DatasetEventPydantic(BaseModelPydantic): dataset: DatasetPydantic class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True diff --git a/airflow/serialization/pydantic/job.py b/airflow/serialization/pydantic/job.py index f8eea470c36c1..99253e3b7af79 100644 --- a/airflow/serialization/pydantic/job.py +++ b/airflow/serialization/pydantic/job.py @@ -29,7 +29,7 @@ def check_runner_initialized(job_runner: Optional[BaseJobRunner], job_type: str) class JobPydantic(BaseModelPydantic): - """Serializable representation of the Job ORM SqlAlchemyModel used by internal API""" + """Serializable representation of the Job ORM SqlAlchemyModel used by internal API.""" id: Optional[int] dag_id: Optional[str] @@ -47,6 +47,6 @@ class JobPydantic(BaseModelPydantic): max_tis_per_query: Optional[int] class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index 9c87186873580..92a282b6bd397 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -25,7 +25,7 @@ class TaskInstancePydantic(BaseModelPydantic): - """Serializable representation of the TaskInstance ORM SqlAlchemyModel used by internal API""" + """Serializable representation of the TaskInstance ORM SqlAlchemyModel used by internal API.""" task_id: str dag_id: str @@ -58,7 +58,7 @@ class TaskInstancePydantic(BaseModelPydantic): run_as_user: Optional[str] class Config: - """Make sure it deals automatically with ORM classes of SQL Alchemy""" + """Make sure it deals automatically with ORM classes of SQL Alchemy.""" orm_mode = True diff --git a/airflow/serialization/serde.py b/airflow/serialization/serde.py index a62ebc2bef2a0..25b3b4f429bd3 100644 --- a/airflow/serialization/serde.py +++ b/airflow/serialization/serde.py @@ -64,7 +64,7 @@ def encode(cls: str, version: int, data: T) -> dict[str, str | int | T]: - """Encodes o so it can be understood by the deserializer""" + """Encodes o so it can be understood by the deserializer.""" return {CLASSNAME: cls, VERSION: version, DATA: data} @@ -268,7 +268,7 @@ def deserialize(o: T | None, full=True, type_hint: Any = None) -> object: def _convert(old: dict) -> dict: - """Converts an old style serialization to new style""" + """Converts an old style serialization to new style.""" if OLD_TYPE in old and OLD_DATA in old: return {CLASSNAME: old[OLD_TYPE], VERSION: DEFAULT_VERSION, DATA: old[OLD_DATA][OLD_DATA]} diff --git a/airflow/serialization/serializers/kubernetes.py b/airflow/serialization/serializers/kubernetes.py index 0ed9c96f71860..900d219da5bdb 100644 --- a/airflow/serialization/serializers/kubernetes.py +++ b/airflow/serialization/serializers/kubernetes.py @@ -49,7 +49,7 @@ def serialize(o: object) -> tuple[U, str, int, bool]: def safe_get_name(pod): """ We're running this in an except block, so we don't want it to - fail under any circumstances, e.g. by accessing an attribute that isn't there + fail under any circumstances, e.g. by accessing an attribute that isn't there. """ try: return pod.metadata.name diff --git a/airflow/timetables/simple.py b/airflow/timetables/simple.py index 8374fcdfc2b0f..1cfefe5a0f52b 100644 --- a/airflow/timetables/simple.py +++ b/airflow/timetables/simple.py @@ -110,7 +110,7 @@ def next_dagrun_info( class ContinuousTimetable(_TrivialTimetable): - """Timetable that schedules continually, while still respecting start_date and end_date + """Timetable that schedules continually, while still respecting start_date and end_date. This corresponds to ``schedule="@continuous"``. """ diff --git a/airflow/utils/airflow_flask_app.py b/airflow/utils/airflow_flask_app.py index 8a27456df5d61..751fcba8c3607 100644 --- a/airflow/utils/airflow_flask_app.py +++ b/airflow/utils/airflow_flask_app.py @@ -25,7 +25,7 @@ class AirflowApp(Flask): - """Airflow Flask Application""" + """Airflow Flask Application.""" appbuilder: AirflowAppBuilder dag_bag: DagBag diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index d5d55c20cbd48..d9e53ac07297d 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Utilities module for cli""" +"""Utilities module for cli.""" from __future__ import annotations import functools @@ -85,7 +85,7 @@ def action_logging(f: T) -> T: def wrapper(*args, **kwargs): """ An wrapper for cli functions. It assumes to have Namespace instance - at 1st positional argument + at 1st positional argument. :param args: Positional argument. It assumes to have Namespace instance at 1st positional argument @@ -181,7 +181,7 @@ def process_subdir(subdir: str | None): def get_dag_by_file_location(dag_id: str): - """Returns DAG of a given dag_id by looking up file location""" + """Returns DAG of a given dag_id by looking up file location.""" from airflow.models import DagBag, DagModel # Benefit is that logging from other dags in dagbag will not appear @@ -217,7 +217,7 @@ def _search_for_dag_file(val: str | None) -> str | None: def get_dag(subdir: str | None, dag_id: str) -> DAG: """ - Returns DAG of a given dag_id + Returns DAG of a given dag_id. First it we'll try to use the given subdir. If that doesn't work, we'll try to find the correct path (assuming it's a file) and failing that, use the configured @@ -239,7 +239,7 @@ def get_dag(subdir: str | None, dag_id: str) -> DAG: def get_dags(subdir: str | None, dag_id: str, use_regex: bool = False): - """Returns DAG(s) matching a given regex or dag_id""" + """Returns DAG(s) matching a given regex or dag_id.""" from airflow.models import DagBag if not use_regex: @@ -256,7 +256,7 @@ def get_dags(subdir: str | None, dag_id: str, use_regex: bool = False): @provide_session def get_dag_by_pickle(pickle_id: int, session: Session = NEW_SESSION) -> DAG: - """Fetch DAG from the database using pickling""" + """Fetch DAG from the database using pickling.""" from airflow.models import DagPickle dag_pickle = session.query(DagPickle).filter(DagPickle.id == pickle_id).first() @@ -267,7 +267,7 @@ def get_dag_by_pickle(pickle_id: int, session: Session = NEW_SESSION) -> DAG: def setup_locations(process, pid=None, stdout=None, stderr=None, log=None): - """Creates logging paths""" + """Creates logging paths.""" if not stderr: stderr = os.path.join(settings.AIRFLOW_HOME, f"airflow-{process}.err") if not stdout: @@ -284,7 +284,7 @@ def setup_locations(process, pid=None, stdout=None, stderr=None, log=None): def setup_logging(filename): - """Creates log file handler for daemon process""" + """Creates log file handler for daemon process.""" root = logging.getLogger() handler = NonCachingFileHandler(filename) formatter = logging.Formatter(settings.SIMPLE_LOG_FORMAT) @@ -297,7 +297,8 @@ def setup_logging(filename): def sigint_handler(sig, frame): """ - Returns without error on SIGINT or SIGTERM signals in interactive command mode + Returns without error on SIGINT or SIGTERM signals in interactive command mode. + e.g. CTRL+C or kill """ sys.exit(0) @@ -305,8 +306,9 @@ def sigint_handler(sig, frame): def sigquit_handler(sig, frame): """ - Helps debug deadlocks by printing stacktraces when this gets a SIGQUIT - e.g. kill -s QUIT or CTRL+\ + Helps debug deadlocks by printing stacktraces when this gets a SIGQUIT. + + e.g. kill -s QUIT or CTRL+ """ print(f"Dumping stack traces for all threads in PID {os.getpid()}") id_to_name = {th.ident: th.name for th in threading.enumerate()} @@ -329,7 +331,7 @@ class ColorMode: def should_use_colors(args) -> bool: - """Processes arguments and decides whether to enable color in output""" + """Processes arguments and decides whether to enable color in output.""" if args.color == ColorMode.ON: return True if args.color == ColorMode.OFF: diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index 0c93148308fbc..8962e352215ae 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -29,9 +29,10 @@ def register_pre_exec_callback(action_logger): """ Registers more action_logger function callback for pre-execution. + This function callback is expected to be called with keyword args. For more about the arguments that is being passed to the callback, - refer to airflow.utils.cli.action_logging() + refer to airflow.utils.cli.action_logging(). :param action_logger: An action logger function :return: None @@ -43,9 +44,10 @@ def register_pre_exec_callback(action_logger): def register_post_exec_callback(action_logger): """ Registers more action_logger function callback for post-execution. + This function callback is expected to be called with keyword args. For more about the arguments that is being passed to the callback, - refer to airflow.utils.cli.action_logging() + refer to airflow.utils.cli.action_logging(). :param action_logger: An action logger function :return: None diff --git a/airflow/utils/cli_app_builder.py b/airflow/utils/cli_app_builder.py index 8b0af5c6b0570..e8cf74e6ec6ed 100644 --- a/airflow/utils/cli_app_builder.py +++ b/airflow/utils/cli_app_builder.py @@ -31,7 +31,7 @@ @lru_cache(maxsize=None) def _return_appbuilder(app: Flask) -> AirflowAppBuilder: - """Returns an appbuilder instance for the given app""" + """Returns an appbuilder instance for the given app.""" init_appbuilder(app) init_plugins(app) return app.appbuilder # type: ignore[attr-defined] diff --git a/airflow/utils/code_utils.py b/airflow/utils/code_utils.py index 7783fec0a1edb..65172a0ebe17b 100644 --- a/airflow/utils/code_utils.py +++ b/airflow/utils/code_utils.py @@ -23,7 +23,7 @@ def get_python_source(x: Any) -> str | None: - """Helper function to get Python source (or not), preventing exceptions""" + """Helper function to get Python source (or not), preventing exceptions.""" if isinstance(x, str): return x diff --git a/airflow/utils/compression.py b/airflow/utils/compression.py index 3cb83f851bf27..8f4946346d636 100644 --- a/airflow/utils/compression.py +++ b/airflow/utils/compression.py @@ -24,7 +24,7 @@ def uncompress_file(input_file_name, file_extension, dest_dir): - """Uncompress gz and bz2 files""" + """Uncompress gz and bz2 files.""" if file_extension.lower() not in (".gz", ".bz2"): raise NotImplementedError( f"Received {file_extension} format. Only gz and bz2 files can currently be uncompressed." diff --git a/airflow/utils/dag_cycle_tester.py b/airflow/utils/dag_cycle_tester.py index 93e9e5a5a8fe0..3325a42f39f22 100644 --- a/airflow/utils/dag_cycle_tester.py +++ b/airflow/utils/dag_cycle_tester.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""DAG Cycle tester""" +"""DAG Cycle tester.""" from __future__ import annotations from collections import defaultdict, deque diff --git a/airflow/utils/dag_parsing_context.py b/airflow/utils/dag_parsing_context.py index 48e5acfff2555..27d2f8cab31c9 100644 --- a/airflow/utils/dag_parsing_context.py +++ b/airflow/utils/dag_parsing_context.py @@ -53,7 +53,7 @@ def _airflow_parsing_context_manager(dag_id: str | None = None, task_id: str | N def get_parsing_context() -> AirflowParsingContext: - """Return the current (DAG) parsing context info""" + """Return the current (DAG) parsing context info.""" return AirflowParsingContext( dag_id=os.environ.get(_AIRFLOW_PARSING_CONTEXT_DAG_ID), task_id=os.environ.get(_AIRFLOW_PARSING_CONTEXT_TASK_ID), diff --git a/airflow/utils/dates.py b/airflow/utils/dates.py index da84d792c65d0..820ace78b2a48 100644 --- a/airflow/utils/dates.py +++ b/airflow/utils/dates.py @@ -45,7 +45,7 @@ def date_range( """ Get a set of dates as a list based on a start, end and delta, delta can be something that can be added to `datetime.datetime` - or a cron expression as a `str` + or a cron expression as a `str`. .. code-block:: pycon >>> from airflow.utils.dates import date_range @@ -140,7 +140,7 @@ def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)): """ Returns the datetime of the form start_date + i * delta which is closest to dt for any non-negative integer i. - Note that delta may be a datetime.timedelta or a dateutil.relativedelta + Note that delta may be a datetime.timedelta or a dateutil.relativedelta. .. code-block:: pycon @@ -223,7 +223,7 @@ def infer_time_unit(time_seconds_arr): """ Determine the most appropriate time unit for an array of time durations specified in seconds. - e.g. 5400 seconds => 'minutes', 36000 seconds => 'hours' + e.g. 5400 seconds => 'minutes', 36000 seconds => 'hours'. """ if len(time_seconds_arr) == 0: return "hours" diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 27ad2ee15201a..d60655c7f0024 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -763,6 +763,7 @@ def _get_current_revision(session): def check_migrations(timeout): """ Function to wait for all airflow migrations to complete. + :param timeout: Timeout for the migration in seconds :return: None """ @@ -805,7 +806,7 @@ def _configured_alembic_environment() -> Generator[EnvironmentContext, None, Non def check_and_run_migrations(): - """Check and run migrations if necessary. Only use in a tty""" + """Check and run migrations if necessary. Only use in a tty.""" with _configured_alembic_environment() as env: context = env.get_context() source_heads = set(env.script.get_heads()) @@ -939,7 +940,7 @@ def synchronize_log_template(*, session: Session = NEW_SESSION) -> None: def check_conn_id_duplicates(session: Session) -> Iterable[str]: """ - Check unique conn_id in connection table + Check unique conn_id in connection table. :param session: session of the sqlalchemy """ @@ -962,7 +963,7 @@ def check_conn_id_duplicates(session: Session) -> Iterable[str]: def check_username_duplicates(session: Session) -> Iterable[str]: """ - Check unique username in User & RegisterUser table + Check unique username in User & RegisterUser table. :param session: session of the sqlalchemy :rtype: str @@ -1080,7 +1081,7 @@ def check_task_fail_for_duplicates(session): def check_conn_type_null(session: Session) -> Iterable[str]: """ - Check nullable conn_type column in Connection table + Check nullable conn_type column in Connection table. :param session: session of the sqlalchemy """ @@ -1452,7 +1453,7 @@ class BadReferenceConfig: @provide_session def _check_migration_errors(session: Session = NEW_SESSION) -> Iterable[str]: - """:session: session of the sqlalchemy""" + """:session: session of the sqlalchemy.""" check_functions: tuple[Callable[..., Iterable[str]], ...] = ( check_conn_id_duplicates, check_conn_type_null, @@ -1618,7 +1619,7 @@ def upgradedb( @provide_session def resetdb(session: Session = NEW_SESSION, skip_init: bool = False): - """Clear out the database""" + """Clear out the database.""" if not settings.engine: raise RuntimeError("The settings.engine must be set. This is a critical assertion") log.info("Dropping tables that exist") diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 03b233f740fe6..fa8696e8f72c1 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -50,7 +50,7 @@ @dataclass class _TableConfig: """ - Config class for performing cleanup on a table + Config class for performing cleanup on a table. :param table_name: the table :param extra_columns: any columns besides recency_column_name that we'll need in queries diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py index 645e65a637e0d..eaa624f6cfb63 100644 --- a/airflow/utils/decorators.py +++ b/airflow/utils/decorators.py @@ -58,7 +58,7 @@ def wrapper(*args, **kwargs): def remove_task_decorator(python_source: str, task_decorator_name: str) -> str: """ - Removes @task or similar decorators as well as @setup and @teardown + Removes @task or similar decorators as well as @setup and @teardown. :param python_source: The python source code :param task_decorator_name: the decorator name diff --git a/airflow/utils/dot_renderer.py b/airflow/utils/dot_renderer.py index 56f999f9e3811..3f35d1b21df61 100644 --- a/airflow/utils/dot_renderer.py +++ b/airflow/utils/dot_renderer.py @@ -56,7 +56,7 @@ def _draw_task( parent_graph: graphviz.Digraph, states_by_task_id: dict[Any, Any] | None, ) -> None: - """Draw a single task on the given parent_graph""" + """Draw a single task on the given parent_graph.""" if states_by_task_id: state = states_by_task_id.get(task.task_id, State.NONE) color = State.color_fg(state) @@ -80,7 +80,7 @@ def _draw_task( def _draw_task_group( task_group: TaskGroup, parent_graph: graphviz.Digraph, states_by_task_id: dict[str, str] | None ) -> None: - """Draw the given task_group and its children on the given parent_graph""" + """Draw the given task_group and its children on the given parent_graph.""" # Draw joins if task_group.upstream_group_ids or task_group.upstream_task_ids: parent_graph.node( diff --git a/airflow/utils/event_scheduler.py b/airflow/utils/event_scheduler.py index cc17ad976e372..33cc2b569086f 100644 --- a/airflow/utils/event_scheduler.py +++ b/airflow/utils/event_scheduler.py @@ -22,7 +22,7 @@ class EventScheduler(scheduler): - """General purpose event scheduler""" + """General purpose event scheduler.""" def call_regular_interval( self, @@ -31,7 +31,7 @@ def call_regular_interval( arguments=(), kwargs={}, ): - """Helper to call a function at (roughly) a given interval""" + """Helper to call a function at (roughly) a given interval.""" def repeat(*args, **kwargs): action(*args, **kwargs) diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 81089e06d4c26..f944aa0020e05 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -40,7 +40,7 @@ class _IgnoreRule(Protocol): - """Interface for ignore rules for structural subtyping""" + """Interface for ignore rules for structural subtyping.""" @staticmethod def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule | None: @@ -51,18 +51,18 @@ def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule @staticmethod def match(path: Path, rules: list[_IgnoreRule]) -> bool: - """Match a candidate absolute path against a list of rules""" + """Match a candidate absolute path against a list of rules.""" class _RegexpIgnoreRule(NamedTuple): - """Typed namedtuple with utility functions for regexp ignore rules""" + """Typed namedtuple with utility functions for regexp ignore rules.""" pattern: Pattern base_dir: Path @staticmethod def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule | None: - """Build an ignore rule from the supplied regexp pattern and log a useful warning if it is invalid""" + """Build an ignore rule from the supplied regexp pattern and log a useful warning if it is invalid.""" try: return _RegexpIgnoreRule(re.compile(pattern), base_dir) except re.error as e: @@ -71,7 +71,7 @@ def compile(pattern: str, base_dir: Path, definition_file: Path) -> _IgnoreRule @staticmethod def match(path: Path, rules: list[_IgnoreRule]) -> bool: - """Match a list of ignore rules against the supplied path""" + """Match a list of ignore rules against the supplied path.""" for rule in rules: if not isinstance(rule, _RegexpIgnoreRule): raise ValueError(f"_RegexpIgnoreRule cannot match rules of type: {type(rule)}") @@ -81,7 +81,7 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool: class _GlobIgnoreRule(NamedTuple): - """Typed namedtuple with utility functions for glob ignore rules""" + """Typed namedtuple with utility functions for glob ignore rules.""" pattern: Pattern raw_pattern: str @@ -90,7 +90,7 @@ class _GlobIgnoreRule(NamedTuple): @staticmethod def compile(pattern: str, _, definition_file: Path) -> _IgnoreRule | None: - """Build an ignore rule from the supplied glob pattern and log a useful warning if it is invalid""" + """Build an ignore rule from the supplied glob pattern and log a useful warning if it is invalid.""" relative_to: Path | None = None if pattern.strip() == "/": # "/" doesn't match anything in gitignore @@ -107,7 +107,7 @@ def compile(pattern: str, _, definition_file: Path) -> _IgnoreRule | None: @staticmethod def match(path: Path, rules: list[_IgnoreRule]) -> bool: - """Match a list of ignore rules against the supplied path""" + """Match a list of ignore rules against the supplied path.""" matched = False for r in rules: if not isinstance(r, _GlobIgnoreRule): @@ -123,7 +123,7 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool: def TemporaryDirectory(*args, **kwargs): - """This function is deprecated. Please use `tempfile.TemporaryDirectory`""" + """This function is deprecated. Please use `tempfile.TemporaryDirectory`.""" import warnings from tempfile import TemporaryDirectory as TmpDir @@ -265,6 +265,7 @@ def find_path_from_directory( ) -> Generator[str, None, None]: """ Recursively search the base path and return the list of file paths that should not be ignored. + :param base_dir_path: the base path to be searched :param ignore_file_name: the file name in which specifies the patterns of files/dirs to be ignored :param ignore_file_syntax: the syntax of patterns in the ignore file: regexp or glob diff --git a/airflow/utils/hashlib_wrapper.py b/airflow/utils/hashlib_wrapper.py index 2415f3d00e94e..0f756c0046261 100644 --- a/airflow/utils/hashlib_wrapper.py +++ b/airflow/utils/hashlib_wrapper.py @@ -25,6 +25,7 @@ def md5(data: bytes, *, usedforsecurity: bool | None = None): """ Safely allows calling the hashlib.md5 function with the "usedforsecurity" param. + :param data: The data to hash. :param usedforsecurity: The value to pass to the md5 function's "usedforsecurity" param. Defaults to None. diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index a53085213809c..a29cf07a59194 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -71,7 +71,7 @@ def validate_group_key(k: str, max_length: int = 200): def alchemy_to_dict(obj: Any) -> dict | None: - """Transforms a SQLAlchemy model instance into a dictionary""" + """Transforms a SQLAlchemy model instance into a dictionary.""" if not obj: return None output = {} @@ -101,7 +101,7 @@ def ask_yesno(question: str, default: bool | None = None) -> bool: def prompt_with_timeout(question: str, timeout: int, default: bool | None = None) -> bool: - """Ask the user a question and timeout if they don't respond""" + """Ask the user a question and timeout if they don't respond.""" def handler(signum, frame): raise AirflowException(f"Timeout {timeout}s reached") @@ -115,7 +115,7 @@ def handler(signum, frame): def is_container(obj: Any) -> bool: - """Test if an object is a container (iterable) but not a string""" + """Test if an object is a container (iterable) but not a string.""" return hasattr(obj, "__iter__") and not isinstance(obj, str) @@ -131,7 +131,7 @@ def as_tuple(obj: Any) -> tuple: def chunks(items: list[T], chunk_size: int) -> Generator[list[T], None, None]: - """Yield successive chunks of a given size from a list of items""" + """Yield successive chunks of a given size from a list of items.""" if chunk_size <= 0: raise ValueError("Chunk size must be a positive integer") for i in range(0, len(items), chunk_size): @@ -141,7 +141,7 @@ def chunks(items: list[T], chunk_size: int) -> Generator[list[T], None, None]: def reduce_in_chunks(fn: Callable[[S, list[T]], S], iterable: list[T], initializer: S, chunk_size: int = 0): """ Reduce the given list of items by splitting it into chunks - of the given size and passing each chunk through the reducer + of the given size and passing each chunk through the reducer. """ if len(iterable) == 0: return initializer @@ -152,7 +152,7 @@ def reduce_in_chunks(fn: Callable[[S, list[T]], S], iterable: list[T], initializ def as_flattened_list(iterable: Iterable[Iterable[T]]) -> list[T]: """ - Return an iterable with one level flattened + Return an iterable with one level flattened. >>> as_flattened_list((('blue', 'red'), ('green', 'yellow', 'pink'))) ['blue', 'red', 'green', 'yellow', 'pink'] @@ -173,7 +173,7 @@ def parse_template_string(template_string: str) -> tuple[str | None, jinja2.Temp def render_log_filename(ti: TaskInstance, try_number, filename_template) -> str: """ Given task instance, try_number, filename_template, return the rendered log - filename + filename. :param ti: task instance :param try_number: try_number of the task @@ -215,7 +215,7 @@ def merge_dicts(dict1: dict, dict2: dict) -> dict: def partition(pred: Callable[[T], bool], iterable: Iterable[T]) -> tuple[Iterable[T], Iterable[T]]: - """Use a predicate to partition entries into false entries and true entries""" + """Use a predicate to partition entries into false entries and true entries.""" iter_1, iter_2 = tee(iterable) return filterfalse(pred, iter_1), filter(pred, iter_2) @@ -242,9 +242,10 @@ def cross_downstream(*args, **kwargs): def build_airflow_url_with_query(query: dict[str, Any]) -> str: """ - Build airflow url using base_url and default_view and provided query + Build airflow url using base_url and default_view and provided query. + For example: - 'http://0.0.0.0:8000/base/graph?dag_id=my-task&root=&execution_date=2020-10-27T10%3A59%3A25.615587 + http://0.0.0.0:8000/base/graph?dag_id=my-task&root=&execution_date=2020-10-27T10%3A59%3A25.615587 """ import flask diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 74f342c6ee25b..3a0a70fb7585c 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -122,7 +122,7 @@ def object_hook(self, dct: dict) -> object: @staticmethod def orm_object_hook(dct: dict) -> object: - """Creates a readable representation of a serialized object""" + """Creates a readable representation of a serialized object.""" return deserialize(dct, False) diff --git a/airflow/utils/jwt_signer.py b/airflow/utils/jwt_signer.py index 39e0788d9b5e2..c52b391e35a5d 100644 --- a/airflow/utils/jwt_signer.py +++ b/airflow/utils/jwt_signer.py @@ -50,6 +50,7 @@ def __init__( def generate_signed_token(self, extra_payload: dict[str, Any]) -> str: """ Generate JWT with extra payload added. + :param extra_payload: extra payload that is added to the signed token :return: signed token """ diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 1640f59661302..28d8c628ccec9 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -523,5 +523,5 @@ def _read_from_logs_server(self, ti, worker_log_rel_path) -> tuple[list[str], li return messages, logs def _read_remote_logs(self, ti, try_number, metadata=None): - """Implement in subclasses to read from the remote service""" + """Implement in subclasses to read from the remote service.""" raise NotImplementedError diff --git a/airflow/utils/log/json_formatter.py b/airflow/utils/log/json_formatter.py index fc3e68403e92f..d191e69a20f20 100644 --- a/airflow/utils/log/json_formatter.py +++ b/airflow/utils/log/json_formatter.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""json_formatter module stores all related to ElasticSearch specific logger classes""" +"""json_formatter module stores all related to ElasticSearch specific logger classes.""" from __future__ import annotations import json diff --git a/airflow/utils/log/log_reader.py b/airflow/utils/log/log_reader.py index 63529e71e2924..97fe173e39368 100644 --- a/airflow/utils/log/log_reader.py +++ b/airflow/utils/log/log_reader.py @@ -32,7 +32,7 @@ class TaskLogReader: - """Task log reader""" + """Task log reader.""" STREAM_LOOP_SLEEP_SECONDS = 0.5 """Time to sleep between loops while waiting for more logs""" @@ -65,7 +65,7 @@ def read_log_chunks( def read_log_stream(self, ti: TaskInstance, try_number: int | None, metadata: dict) -> Iterator[str]: """ - Used to continuously read log to the end + Used to continuously read log to the end. :param ti: The Task Instance :param try_number: the task try number @@ -133,7 +133,7 @@ def render_log_filename( session: Session = NEW_SESSION, ) -> str: """ - Renders the log attachment filename + Renders the log attachment filename. :param ti: The task instance :param try_number: The task try number diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 5e650ea5be892..02f89f2812174 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -32,7 +32,10 @@ # Private: A sentinel objects class SetContextPropagate(enum.Enum): - """:meta private:""" + """Sentinel objects for log propagation contexts. + + :meta private: + """ # If a `set_context` function wants to _keep_ propagation set on it's logger it needs to return this # special value. @@ -61,7 +64,7 @@ def remove_escape_codes(text: str) -> str: class LoggingMixin: - """Convenience super-class to have a logger configured with the class name""" + """Convenience super-class to have a logger configured with the class name.""" _log: logging.Logger | None = None @@ -95,7 +98,7 @@ class ExternalLoggingMixin: @property @abc.abstractmethod def log_name(self) -> str: - """Return log name""" + """Return log name.""" @abc.abstractmethod def get_external_log_url(self, task_instance, try_number) -> str: @@ -114,7 +117,7 @@ def supports_external_link(self) -> bool: # IO generics (and apparently it has not even been intended) # See more: https://giters.com/python/typeshed/issues/6077 class StreamLogWriter(IOBase, IO[str]): # type: ignore[misc] - """Allows to redirect stdout and stderr to logger""" + """Allows to redirect stdout and stderr to logger.""" encoding: None = None @@ -150,7 +153,7 @@ def _propagate_log(self, message): def write(self, message): """ - Do whatever it takes to actually log the specified logging record + Do whatever it takes to actually log the specified logging record. :param message: message to log """ @@ -161,7 +164,7 @@ def write(self, message): self.flush() def flush(self): - """Ensure all logging output has been flushed""" + """Ensure all logging output has been flushed.""" buf = self._buffer if len(buf) > 0: self._buffer = "" @@ -213,7 +216,7 @@ def stream(self): def set_context(logger, value): """ - Walks the tree of loggers and tries to set the context for each handler + Walks the tree of loggers and tries to set the context for each handler. :param logger: logger :param value: value to set diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py index 6703139a3bc8e..381ba5aeaba77 100644 --- a/airflow/utils/log/secrets_masker.py +++ b/airflow/utils/log/secrets_masker.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Mask sensitive information from logs""" +"""Mask sensitive information from logs.""" from __future__ import annotations import collections.abc @@ -82,7 +82,7 @@ def get_sensitive_variables_fields(): def should_hide_value_for_key(name): - """Should the value for this given name (Variable name, or key in conn.extra_dejson) be hidden""" + """Should the value for this given name (Variable name, or key in conn.extra_dejson) be hidden.""" from airflow import settings if isinstance(name, str) and settings.HIDE_SENSITIVE_VAR_CONN_FIELDS: @@ -141,7 +141,7 @@ def _is_v1_env_var(v: Any) -> TypeGuard[V1EnvVar]: class SecretsMasker(logging.Filter): - """Redact secrets from logs""" + """Redact secrets from logs.""" replacer: re.Pattern | None = None patterns: set[str] diff --git a/airflow/utils/log/task_handler_with_custom_formatter.py b/airflow/utils/log/task_handler_with_custom_formatter.py index 4e7bbb6eae773..dc922c92a1fd8 100644 --- a/airflow/utils/log/task_handler_with_custom_formatter.py +++ b/airflow/utils/log/task_handler_with_custom_formatter.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Custom logging formatter for Airflow""" +"""Custom logging formatter for Airflow.""" from __future__ import annotations import logging @@ -30,7 +30,7 @@ class TaskHandlerWithCustomFormatter(logging.StreamHandler): - """Custom implementation of StreamHandler, a class which writes logging records for Airflow""" + """Custom implementation of StreamHandler, a class which writes logging records for Airflow.""" prefix_jinja_template: Template | None = None diff --git a/airflow/utils/log/timezone_aware.py b/airflow/utils/log/timezone_aware.py index e5b885cb90812..ac2d88f020105 100644 --- a/airflow/utils/log/timezone_aware.py +++ b/airflow/utils/log/timezone_aware.py @@ -25,8 +25,9 @@ class TimezoneAware(logging.Formatter): """ Override `default_time_format`, `default_msec_format` and `formatTime` to specify utc offset. utc offset is the matter, without it, time conversion could be wrong. - With this Formatter, `%(asctime)s` will be formatted containing utc offset. (ISO 8601) - (e.g. 2022-06-12T13:00:00.123+0000) + With this Formatter, `%(asctime)s` will be formatted containing utc offset. (ISO 8601). + + e.g. 2022-06-12T13:00:00.123+0000 """ default_time_format = "%Y-%m-%dT%H:%M:%S" diff --git a/airflow/utils/operator_resources.py b/airflow/utils/operator_resources.py index 4bd7801d6aa88..638034a81a920 100644 --- a/airflow/utils/operator_resources.py +++ b/airflow/utils/operator_resources.py @@ -153,7 +153,7 @@ def to_dict(self): @classmethod def from_dict(cls, resources_dict: dict): - """Create resources from resources dict""" + """Create resources from resources dict.""" cpus = resources_dict["cpus"]["qty"] ram = resources_dict["ram"]["qty"] disk = resources_dict["disk"]["qty"] diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py index df6620f6ac05a..50dc669efeb8e 100644 --- a/airflow/utils/process_utils.py +++ b/airflow/utils/process_utils.py @@ -164,6 +164,7 @@ def signal_procs(sig): def execute_in_subprocess(cmd: list[str], cwd: str | None = None) -> None: """ Execute a process and stream output to logger. + :param cmd: command and arguments to run :param cwd: Current working directory passed to the Popen constructor """ @@ -327,7 +328,7 @@ def set_new_process_group() -> None: Try to set current process to a new process group. That makes it easy to kill all sub-process of this at the OS-level, rather than having to iterate the child processes. - If current process spawn by system call ``exec()`` than keep current process group + If current process spawn by system call ``exec()`` than keep current process group. """ if os.getpid() == os.getsid(0): # If PID = SID than process a session leader, and it is not possible to change process group diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index 1e5bb4da367a0..1adabacafff57 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Utilities for creating a virtual environment""" +"""Utilities for creating a virtual environment.""" from __future__ import annotations import os diff --git a/airflow/utils/retries.py b/airflow/utils/retries.py index b0ba711276d8e..6adce3eb43779 100644 --- a/airflow/utils/retries.py +++ b/airflow/utils/retries.py @@ -31,7 +31,7 @@ def run_with_db_retries(max_retries: int = MAX_DB_RETRIES, logger: logging.Logger | None = None, **kwargs): - """Return Tenacity Retrying object with project specific default""" + """Return Tenacity Retrying object with project specific default.""" import tenacity # Default kwargs diff --git a/airflow/utils/scheduler_health.py b/airflow/utils/scheduler_health.py index 4facd0a75c40d..6108aee018a00 100644 --- a/airflow/utils/scheduler_health.py +++ b/airflow/utils/scheduler_health.py @@ -29,7 +29,7 @@ class HealthServer(BaseHTTPRequestHandler): - """Small webserver to serve scheduler health check""" + """Small webserver to serve scheduler health check.""" def do_GET(self): if self.path == "/health": diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py index d647fd74d38cc..8c1d62722a654 100644 --- a/airflow/utils/serve_logs.py +++ b/airflow/utils/serve_logs.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Serve logs process""" +"""Serve logs process.""" from __future__ import annotations import collections @@ -139,7 +139,7 @@ def load(self): def serve_logs(port=None): - """Serves logs generated by Worker""" + """Serves logs generated by Worker.""" setproctitle("airflow serve-logs") wsgi_app = create_app() diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 1ee482cc515d4..2e47290f6ecff 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -51,7 +51,6 @@ class UtcDateTime(TypeDecorator): """ Almost equivalent to :class:`~sqlalchemy.types.TIMESTAMP` with ``timezone=True`` option, but it differs from that by: - - Never silently take naive :class:`~datetime.datetime`, instead it always raise :exc:`ValueError` unless time zone aware value. - :class:`~datetime.datetime` value's :attr:`~datetime.datetime.tzinfo` @@ -59,8 +58,7 @@ class UtcDateTime(TypeDecorator): - Unlike SQLAlchemy's built-in :class:`~sqlalchemy.types.TIMESTAMP`, it never return naive :class:`~datetime.datetime`, but time zone aware value, even with SQLite or MySQL. - - Always returns TIMESTAMP in UTC - + - Always returns TIMESTAMP in UTC. """ impl = TIMESTAMP(timezone=True) @@ -120,7 +118,7 @@ class ExtendedJSON(TypeDecorator): cache_ok = True def db_supports_json(self): - """Checks if the database supports JSON (i.e. is NOT MSSQL)""" + """Checks if the database supports JSON (i.e. is NOT MSSQL).""" return not conf.get("database", "sql_alchemy_conn").startswith("mssql") def load_dialect_impl(self, dialect) -> TypeEngine: @@ -401,7 +399,7 @@ def nulls_first(col, session: Session) -> dict[str, Any]: """ Adds a nullsfirst construct to the column ordering. Currently only Postgres supports it. In MySQL & Sqlite NULL values are considered lower than any non-NULL value, therefore, NULL values - appear first when the order is ASC (ascending) + appear first when the order is ASC (ascending). """ if session.bind.dialect.name == "postgresql": return nullsfirst(col) @@ -431,7 +429,7 @@ def with_row_locks(query, session: Session, **kwargs): class CommitProhibitorGuard: - """Context manager class that powers prohibit_commit""" + """Context manager class that powers prohibit_commit.""" expected_commit = False @@ -484,7 +482,7 @@ def prohibit_commit(session): def is_lock_not_available_error(error: OperationalError): - """Check if the Error is about not being able to acquire lock""" + """Check if the Error is about not being able to acquire lock.""" # DB specific error codes: # Postgres: 55P03 # MySQL: 3572, 'Statement aborted because lock(s) could not be acquired immediately and NOWAIT diff --git a/airflow/utils/strings.py b/airflow/utils/strings.py index ba2bc95c7b0f4..421c47fce5ff3 100644 --- a/airflow/utils/strings.py +++ b/airflow/utils/strings.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Common utility functions with strings""" +"""Common utility functions with strings.""" from __future__ import annotations import string @@ -22,7 +22,7 @@ def get_random_string(length=8, choices=string.ascii_letters + string.digits): - """Generate random string""" + """Generate random string.""" return "".join(choice(choices) for _ in range(length)) diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index 7e59a1bce7eb5..85ec1eb0d3b09 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -187,7 +187,7 @@ def node_id(self): @property def is_root(self) -> bool: - """Returns True if this TaskGroup is the root TaskGroup. Otherwise False""" + """Returns True if this TaskGroup is the root TaskGroup. Otherwise False.""" return not self.group_id @property @@ -338,12 +338,12 @@ def has_task(self, task: BaseOperator) -> bool: @property def roots(self) -> list[BaseOperator]: - """Required by TaskMixin""" + """Required by TaskMixin.""" return list(self.get_roots()) @property def leaves(self) -> list[BaseOperator]: - """Required by TaskMixin""" + """Required by TaskMixin.""" return list(self.get_leaves()) def get_roots(self) -> Generator[BaseOperator, None, None]: @@ -358,7 +358,7 @@ def get_roots(self) -> Generator[BaseOperator, None, None]: def get_leaves(self) -> Generator[BaseOperator, None, None]: """ Returns a generator of tasks that are leaf tasks, i.e. those with no downstream - dependencies within the TaskGroup + dependencies within the TaskGroup. """ for task in self: if not any(self.has_task(child) for child in task.get_direct_relatives(upstream=False)): @@ -395,7 +395,7 @@ def downstream_join_id(self) -> str: return f"{self.group_id}.downstream_join_id" def get_task_group_dict(self) -> dict[str, TaskGroup]: - """Returns a flat dictionary of group_id: TaskGroup""" + """Returns a flat dictionary of group_id: TaskGroup.""" task_group_map = {} def build_map(task_group): @@ -411,7 +411,7 @@ def build_map(task_group): return task_group_map def get_child_by_label(self, label: str) -> DAGNode: - """Get a child task/TaskGroup by its label (i.e. task_id/group_id without the group_id prefix)""" + """Get a child task/TaskGroup by its label (i.e. task_id/group_id without the group_id prefix).""" return self.children[self.child_id(label)] def serialize_for_task_group(self) -> tuple[DagAttributeTypes, Any]: diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index bbe38aba2ee78..5d134697f9341 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -52,7 +52,7 @@ def is_naive(value): def utcnow() -> dt.datetime: """ - Get the current date and time in UTC + Get the current date and time in UTC. :return: """ @@ -67,7 +67,7 @@ def utcnow() -> dt.datetime: def utc_epoch() -> dt.datetime: """ - Gets the epoch in the users timezone + Gets the epoch in the users timezone. :return: """ @@ -93,7 +93,7 @@ def convert_to_utc(value: dt.datetime) -> DateTime: def convert_to_utc(value: dt.datetime | None) -> DateTime | None: """ Returns the datetime with the default timezone added if timezone - information was not associated + information was not associated. :param value: datetime :return: datetime with tzinfo @@ -190,7 +190,7 @@ def make_naive(value, timezone=None): def datetime(*args, **kwargs): """ - Wrapper around datetime.datetime that adds settings.TIMEZONE if tzinfo not specified + Wrapper around datetime.datetime that adds settings.TIMEZONE if tzinfo not specified. :return: datetime.datetime """ @@ -204,7 +204,7 @@ def datetime(*args, **kwargs): def parse(string: str, timezone=None) -> DateTime: """ - Parse a time string and return an aware datetime + Parse a time string and return an aware datetime. :param string: time string :param timezone: the timezone diff --git a/airflow/utils/types.py b/airflow/utils/types.py index b3c824274a831..788b072f39842 100644 --- a/airflow/utils/types.py +++ b/airflow/utils/types.py @@ -45,7 +45,7 @@ def is_arg_passed(arg: Union[ArgNotSet, None] = NOTSET) -> bool: class DagRunType(str, enum.Enum): - """Class with DagRun types""" + """Class with DagRun types.""" BACKFILL_JOB = "backfill" SCHEDULED = "scheduled" diff --git a/airflow/utils/weekday.py b/airflow/utils/weekday.py index 5fddd7ea916cc..cb404628c1330 100644 --- a/airflow/utils/weekday.py +++ b/airflow/utils/weekday.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Get the ISO standard day number of the week from a given day string""" +"""Get the ISO standard day number of the week from a given day string.""" from __future__ import annotations import enum @@ -23,7 +23,7 @@ @enum.unique class WeekDay(enum.IntEnum): - """Python Enum containing Days of the Week""" + """Python Enum containing Days of the Week.""" MONDAY = 1 TUESDAY = 2 @@ -36,7 +36,7 @@ class WeekDay(enum.IntEnum): @classmethod def get_weekday_number(cls, week_day_str: str): """ - Return the ISO Week Day Number for a Week Day + Return the ISO Week Day Number for a Week Day. :param week_day_str: Full Name of the Week Day. Example: "Sunday" :return: ISO Week Day Number corresponding to the provided Weekday @@ -50,7 +50,7 @@ def get_weekday_number(cls, week_day_str: str): @classmethod def convert(cls, day: str | WeekDay) -> int: - """Helper function that returns the day number in the week""" + """Helper function that returns the day number in the week.""" if isinstance(day, WeekDay): return day return cls.get_weekday_number(week_day_str=day) @@ -60,7 +60,7 @@ def validate_week_day( cls, week_day: str | WeekDay | Iterable[str] | Iterable[WeekDay], ) -> set[int]: - """Validate each item of iterable and create a set to ease compare of values""" + """Validate each item of iterable and create a set to ease compare of values.""" if not isinstance(week_day, Iterable): if isinstance(week_day, WeekDay): week_day = {week_day} diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py index 1b7cc48175138..002a81f4d589e 100644 --- a/airflow/utils/weight_rule.py +++ b/airflow/utils/weight_rule.py @@ -37,7 +37,7 @@ def is_valid(cls, weight_rule: str) -> bool: @classmethod @cache def all_weight_rules(cls) -> set[str]: - """Returns all weight rules""" + """Returns all weight rules.""" return set(cls.__members__.values()) def __str__(self) -> str: diff --git a/airflow/utils/yaml.py b/airflow/utils/yaml.py index c402c5425f76a..43201405f8958 100644 --- a/airflow/utils/yaml.py +++ b/airflow/utils/yaml.py @@ -34,7 +34,7 @@ def safe_load(stream: bytes | str | BinaryIO | TextIO) -> Any: - """Like yaml.safe_load, but use the C libyaml for speed where we can""" + """Like yaml.safe_load, but use the C libyaml for speed where we can.""" # delay import until use. from yaml import load as orig @@ -47,7 +47,7 @@ def safe_load(stream: bytes | str | BinaryIO | TextIO) -> Any: def dump(data: Any, **kwargs) -> str: - """Like yaml.safe_dump, but use the C libyaml for speed where we can""" + """Like yaml.safe_dump, but use the C libyaml for speed where we can.""" # delay import until use. from yaml import dump as orig diff --git a/airflow/www/api/experimental/endpoints.py b/airflow/www/api/experimental/endpoints.py index 012e1b2aa32a3..2af00eeee6168 100644 --- a/airflow/www/api/experimental/endpoints.py +++ b/airflow/www/api/experimental/endpoints.py @@ -43,7 +43,7 @@ def requires_authentication(function: T): - """Decorator for functions that require authentication""" + """Decorator for functions that require authentication.""" @wraps(function) def decorated(*args, **kwargs): @@ -157,9 +157,11 @@ def delete_dag(dag_id): def dag_runs(dag_id): """ Returns a list of Dag Runs for a specific DAG ID. + :query param state: a query string parameter '?state=queued|running|success...' :param dag_id: String identifier of a DAG + :return: List of DAG runs of a DAG with requested state, or all runs if the state is not specified """ @@ -178,14 +180,14 @@ def dag_runs(dag_id): @api_experimental.route("/test", methods=["GET"]) @requires_authentication def test(): - """Test endpoint to check authentication""" + """Test endpoint to check authentication.""" return jsonify(status="OK") @api_experimental.route("/info", methods=["GET"]) @requires_authentication def info(): - """Get Airflow Version""" + """Get Airflow Version.""" return jsonify(version=version) @@ -205,7 +207,7 @@ def get_dag_code(dag_id): @api_experimental.route("/dags//tasks/", methods=["GET"]) @requires_authentication def task_info(dag_id, task_id): - """Returns a JSON with a task's public instance variables""" + """Returns a JSON with a task's public instance variables.""" try: t_info = get_task(dag_id, task_id) except AirflowException as err: @@ -223,7 +225,7 @@ def task_info(dag_id, task_id): @api_experimental.route("/dags//paused/", methods=["GET"]) @requires_authentication def dag_paused(dag_id, paused): - """(Un)pauses a dag""" + """(Un)pauses a dag.""" is_paused = bool(paused == "true") models.DagModel.get_dagmodel(dag_id).set_is_paused( @@ -236,7 +238,7 @@ def dag_paused(dag_id, paused): @api_experimental.route("/dags//paused", methods=["GET"]) @requires_authentication def dag_is_paused(dag_id): - """Get paused state of a dag""" + """Get paused state of a dag.""" is_paused = models.DagModel.get_dagmodel(dag_id).is_paused return jsonify({"is_paused": is_paused}) @@ -317,7 +319,7 @@ def dag_run_status(dag_id, execution_date): @api_experimental.route("/latest_runs", methods=["GET"]) @requires_authentication def latest_dag_runs(): - """Returns the latest DagRun for each DAG formatted for the UI""" + """Returns the latest DagRun for each DAG formatted for the UI.""" from airflow.models import DagRun dagruns = DagRun.get_latest_runs() @@ -401,7 +403,7 @@ def delete_pool(name): @api_experimental.route("/lineage//", methods=["GET"]) @requires_authentication def get_lineage(dag_id: str, execution_date: str): - """Get Lineage details for a DagRun""" + """Get Lineage details for a DagRun.""" # Convert string datetime into actual datetime try: execution_dt = timezone.parse(execution_date) diff --git a/airflow/www/app.py b/airflow/www/app.py index 246381f003e4e..4e09148b4436a 100644 --- a/airflow/www/app.py +++ b/airflow/www/app.py @@ -66,7 +66,7 @@ def sync_appbuilder_roles(flask_app): - """Sync appbuilder roles to DB""" + """Sync appbuilder roles to DB.""" # Garbage collect old permissions/views after they have been modified. # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not @@ -76,7 +76,7 @@ def sync_appbuilder_roles(flask_app): def create_app(config=None, testing=False): - """Create a new instance of Airflow WWW app""" + """Create a new instance of Airflow WWW app.""" flask_app = Flask(__name__) flask_app.secret_key = conf.get("webserver", "SECRET_KEY") @@ -175,7 +175,7 @@ def create_app(config=None, testing=False): def cached_app(config=None, testing=False): - """Return cached instance of Airflow WWW app""" + """Return cached instance of Airflow WWW app.""" global app if not app: app = create_app(config=config, testing=testing) diff --git a/airflow/www/blueprints.py b/airflow/www/blueprints.py index 1ff07f5befd5c..0312a9ffa7163 100644 --- a/airflow/www/blueprints.py +++ b/airflow/www/blueprints.py @@ -24,5 +24,5 @@ @routes.route("/") def index(): - """Main Airflow page""" + """Main Airflow page.""" return redirect(url_for("Airflow.index")) diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index 29094969f7da2..59f48e3b519a4 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -60,7 +60,7 @@ def _mask_variable_fields(extra_fields): def _mask_connection_fields(extra_fields): - """Mask connection fields""" + """Mask connection fields.""" result = [] for k, v in extra_fields: if k == "extra": @@ -76,7 +76,7 @@ def _mask_connection_fields(extra_fields): def action_logging(func: Callable | None = None, event: str | None = None) -> Callable[[T], T]: - """Decorator to log user actions""" + """Decorator to log user actions.""" def log_action(f: T) -> T: @functools.wraps(f) @@ -132,7 +132,7 @@ def wrapper(*args, **kwargs): def gzipped(f: T) -> T: - """Decorator to make a view compressed""" + """Decorator to make a view compressed.""" @functools.wraps(f) def view_func(*args, **kwargs): diff --git a/airflow/www/extensions/init_appbuilder.py b/airflow/www/extensions/init_appbuilder.py index dbd2d93938282..b1c70daf47192 100644 --- a/airflow/www/extensions/init_appbuilder.py +++ b/airflow/www/extensions/init_appbuilder.py @@ -51,7 +51,8 @@ def dynamic_class_import(class_path): """ - Will dynamically import a class from a string path + Will dynamically import a class from a string path. + :param class_path: string with class path :return: class """ @@ -172,6 +173,7 @@ def __init__( def init_app(self, app, session): """ Will initialize the Flask app, supporting the app factory pattern. + :param app: :param session: The SQLAlchemy session """ @@ -239,7 +241,7 @@ def _init_extension(self, app): def _swap_url_filter(self): """ Use our url filtering util function so there is consistency between - FAB and Airflow routes + FAB and Airflow routes. """ from flask_appbuilder.security import views as fab_sec_views @@ -260,7 +262,8 @@ def post_init(self): @property def get_app(self): """ - Get current or configured flask app + Get current or configured flask app. + :return: Flask App """ if self.app: @@ -272,6 +275,7 @@ def get_app(self): def get_session(self): """ Get the current sqlalchemy session. + :return: SQLAlchemy Session """ return self.session @@ -279,7 +283,8 @@ def get_session(self): @property def app_name(self): """ - Get the App name + Get the App name. + :return: String with app name """ return self.get_app.config["APP_NAME"] @@ -287,7 +292,8 @@ def app_name(self): @property def app_theme(self): """ - Get the App theme name + Get the App theme name. + :return: String app theme name """ return self.get_app.config["APP_THEME"] @@ -295,7 +301,8 @@ def app_theme(self): @property def app_icon(self): """ - Get the App icon location + Get the App icon location. + :return: String with relative app icon location """ return self.get_app.config["APP_ICON"] @@ -307,7 +314,8 @@ def languages(self): @property def version(self): """ - Get the current F.A.B. version + Get the current F.A.B. version. + :return: String with the current F.A.B. version """ return __version__ @@ -558,18 +566,21 @@ def security_cleanup(self): You can use it always or just sometimes to perform a security cleanup. Warning this will delete any permission that is no longer part of any registered view or menu. - Remember invoke ONLY AFTER YOU HAVE REGISTERED ALL VIEWS + Remember invoke ONLY AFTER YOU HAVE REGISTERED ALL VIEWS. """ self.sm.security_cleanup(self.baseviews, self.menu) def security_converge(self, dry=False) -> dict: """ - This method is useful when you use: - - `class_permission_name` - - `previous_class_permission_name` - - `method_permission_name` - - `previous_method_permission_name` - migrates all permissions to the new names on all the Roles + Migrates all permissions to the new names on all the Roles. + + This method is useful when you use: + + - `class_permission_name` + - `previous_class_permission_name` + - `method_permission_name` + - `previous_method_permission_name` + :param dry: If True will not change DB :return: Dict with all computed necessary operations """ diff --git a/airflow/www/extensions/init_appbuilder_links.py b/airflow/www/extensions/init_appbuilder_links.py index 67bd1f6bb411c..f7d747013d58c 100644 --- a/airflow/www/extensions/init_appbuilder_links.py +++ b/airflow/www/extensions/init_appbuilder_links.py @@ -21,7 +21,7 @@ def init_appbuilder_links(app): - """Add links to the navbar""" + """Add links to the navbar.""" appbuilder = app.appbuilder appbuilder.add_link(name="DAGs", href="Airflow.index") diff --git a/airflow/www/extensions/init_jinja_globals.py b/airflow/www/extensions/init_jinja_globals.py index 1cbb0a12734ce..2d5778af4b4fd 100644 --- a/airflow/www/extensions/init_jinja_globals.py +++ b/airflow/www/extensions/init_jinja_globals.py @@ -28,7 +28,7 @@ def init_jinja_globals(app): - """Add extra globals variable to Jinja context""" + """Add extra globals variable to Jinja context.""" server_timezone = conf.get("core", "default_timezone") if server_timezone == "system": server_timezone = pendulum.local_timezone().name diff --git a/airflow/www/extensions/init_security.py b/airflow/www/extensions/init_security.py index a6640b6ad8c32..ab558bf4f4470 100644 --- a/airflow/www/extensions/init_security.py +++ b/airflow/www/extensions/init_security.py @@ -47,7 +47,7 @@ def apply_caching(response): def init_api_experimental_auth(app): - """Loads authentication backends""" + """Loads authentication backends.""" auth_backends = "airflow.api.auth.backend.default" try: auth_backends = conf.get("api", "auth_backends") diff --git a/airflow/www/extensions/init_session.py b/airflow/www/extensions/init_session.py index 8829962a3b941..fb26230c53b61 100644 --- a/airflow/www/extensions/init_session.py +++ b/airflow/www/extensions/init_session.py @@ -24,7 +24,7 @@ def init_airflow_session_interface(app): - """Set airflow session interface""" + """Set airflow session interface.""" config = app.config.copy() selected_backend = conf.get("webserver", "SESSION_BACKEND") # A bit of a misnomer - normally cookies expire whenever the browser is closed diff --git a/airflow/www/extensions/init_views.py b/airflow/www/extensions/init_views.py index d3dc3b7c62324..71efdbcf4b5f8 100644 --- a/airflow/www/extensions/init_views.py +++ b/airflow/www/extensions/init_views.py @@ -39,14 +39,14 @@ def init_flash_views(app): - """Init main app view - redirect to FAB""" + """Init main app view - redirect to FAB.""" from airflow.www.blueprints import routes app.register_blueprint(routes) def init_appbuilder_views(app): - """Initialize Web UI views""" + """Initialize Web UI views.""" from airflow.models import import_all_models import_all_models() @@ -125,7 +125,7 @@ def init_appbuilder_views(app): def init_plugins(app): - """Integrate Flask and FAB with plugins""" + """Integrate Flask and FAB with plugins.""" from airflow import plugins_manager plugins_manager.initialize_web_ui_plugins() @@ -154,7 +154,7 @@ def init_plugins(app): def init_error_handlers(app: Flask): - """Add custom errors handlers""" + """Add custom errors handlers.""" from airflow.www import views app.register_error_handler(500, views.show_traceback) @@ -162,7 +162,7 @@ def init_error_handlers(app: Flask): def set_cors_headers_on_response(response): - """Add response headers""" + """Add response headers.""" allow_headers = conf.get("api", "access_control_allow_headers") allow_methods = conf.get("api", "access_control_allow_methods") allow_origins = conf.get("api", "access_control_allow_origins") @@ -226,7 +226,7 @@ def validate_schema(self, data, url): def init_api_connexion(app: Flask) -> None: - """Initialize Stable API""" + """Initialize Stable API.""" base_path = "/api/v1" from airflow.www import views @@ -271,7 +271,7 @@ def _handle_method_not_allowed(ex): def init_api_internal(app: Flask, standalone_api: bool = False) -> None: - """Initialize Internal API""" + """Initialize Internal API.""" if not standalone_api and not conf.getboolean("webserver", "run_internal_api", fallback=False): return @@ -292,7 +292,7 @@ def init_api_internal(app: Flask, standalone_api: bool = False) -> None: def init_api_experimental(app): - """Initialize Experimental API""" + """Initialize Experimental API.""" if not conf.getboolean("api", "enable_experimental_api", fallback=False): return from airflow.www.api.experimental import endpoints diff --git a/airflow/www/extensions/init_wsgi_middlewares.py b/airflow/www/extensions/init_wsgi_middlewares.py index 809d8b2533669..3ea47e92c86f6 100644 --- a/airflow/www/extensions/init_wsgi_middlewares.py +++ b/airflow/www/extensions/init_wsgi_middlewares.py @@ -36,7 +36,7 @@ def _root_app(env: WSGIEnvironment, resp: StartResponse) -> Iterable[bytes]: def init_wsgi_middleware(flask_app: Flask) -> None: - """Handle X-Forwarded-* headers and base_url support""" + """Handle X-Forwarded-* headers and base_url support.""" # Apply DispatcherMiddleware base_url = urlsplit(conf.get("webserver", "base_url"))[2] if not base_url or base_url == "/": diff --git a/airflow/www/fab_security/manager.py b/airflow/www/fab_security/manager.py index 8f0363186bfa0..380e3ef9e3033 100644 --- a/airflow/www/fab_security/manager.py +++ b/airflow/www/fab_security/manager.py @@ -146,7 +146,7 @@ class BaseSecurityManager: @staticmethod def oauth_tokengetter(token=None): """Authentication (OAuth) token getter function. - Override to implement your own token getter method + Override to implement your own token getter method. """ return _oauth_tokengetter(token) @@ -296,7 +296,7 @@ def create_limiter(self, app: Flask) -> Limiter: def create_login_manager(self, app) -> LoginManager: """ - Override to implement your custom login manager instance + Override to implement your custom login manager instance. :param app: Flask app """ @@ -308,7 +308,7 @@ def create_login_manager(self, app) -> LoginManager: def create_jwt_manager(self, app) -> JWTManager: """ - Override to implement your custom JWT manager instance + Override to implement your custom JWT manager instance. :param app: Flask app """ @@ -351,22 +351,22 @@ def auth_type_provider_name(self): @property def get_url_for_registeruser(self): - """Gets the URL for Register User""" + """Gets the URL for Register User.""" return url_for(f"{self.registeruser_view.endpoint}.{self.registeruser_view.default_view}") @property def get_user_datamodel(self): - """Gets the User data model""" + """Gets the User data model.""" return self.user_view.datamodel @property def get_register_user_datamodel(self): - """Gets the Register User data model""" + """Gets the Register User data model.""" return self.registerusermodelview.datamodel @property def builtin_roles(self): - """Get the builtin roles""" + """Get the builtin roles.""" return self._builtin_roles @property @@ -375,42 +375,42 @@ def api_login_allow_multiple_providers(self): @property def auth_type(self): - """Get the auth type""" + """Get the auth type.""" return self.appbuilder.get_app.config["AUTH_TYPE"] @property def auth_username_ci(self): - """Gets the auth username for CI""" + """Gets the auth username for CI.""" return self.appbuilder.get_app.config.get("AUTH_USERNAME_CI", True) @property def auth_role_admin(self): - """Gets the admin role""" + """Gets the admin role.""" return self.appbuilder.get_app.config["AUTH_ROLE_ADMIN"] @property def auth_role_public(self): - """Gets the public role""" + """Gets the public role.""" return self.appbuilder.get_app.config["AUTH_ROLE_PUBLIC"] @property def auth_ldap_server(self): - """Gets the LDAP server object""" + """Gets the LDAP server object.""" return self.appbuilder.get_app.config["AUTH_LDAP_SERVER"] @property def auth_ldap_use_tls(self): - """Should LDAP use TLS""" + """Should LDAP use TLS.""" return self.appbuilder.get_app.config["AUTH_LDAP_USE_TLS"] @property def auth_user_registration(self): - """Will user self registration be allowed""" + """Will user self registration be allowed.""" return self.appbuilder.get_app.config["AUTH_USER_REGISTRATION"] @property def auth_user_registration_role(self): - """The default user self registration role""" + """The default user self registration role.""" return self.appbuilder.get_app.config["AUTH_USER_REGISTRATION_ROLE"] @property @@ -520,12 +520,12 @@ def auth_ldap_tls_keyfile(self): @property def openid_providers(self): - """Openid providers""" + """Openid providers.""" return self.appbuilder.get_app.config["OPENID_PROVIDERS"] @property def oauth_providers(self): - """Oauth providers""" + """Oauth providers.""" return self.appbuilder.get_app.config["OAUTH_PROVIDERS"] @property @@ -538,7 +538,7 @@ def auth_rate_limit(self) -> str: @property def current_user(self): - """Current user object""" + """Current user object.""" if current_user.is_authenticated: return g.user elif current_user_jwt: @@ -586,16 +586,15 @@ def get_oauth_token_key_name(self, provider): def get_oauth_token_secret_name(self, provider): """ - Returns the token_secret name for the oauth provider - if none is configured defaults to oauth_secret - this is configured using OAUTH_PROVIDERS and token_secret + Returns the token_secret name for the oauth provider if none is configured defaults to oauth_secret. + This is configured using OAUTH_PROVIDERS and token_secret. """ for _provider in self.oauth_providers: if _provider["name"] == provider: return _provider.get("token_secret", "oauth_token_secret") def set_oauth_session(self, provider, oauth_response): - """Set the current session with OAuth user secrets""" + """Set the current session with OAuth user secrets.""" # Get this provider key names for token_key and token_secret token_key = self.appbuilder.sm.get_oauth_token_key_name(provider) token_secret = self.appbuilder.sm.get_oauth_token_secret_name(provider) @@ -609,7 +608,7 @@ def set_oauth_session(self, provider, oauth_response): def get_oauth_user_info(self, provider, resp): """ Since there are different OAuth API's with different ways to - retrieve user info + retrieve user info. """ # for GITHUB if provider == "github" or provider == "githublocal": @@ -893,14 +892,14 @@ def update_user_auth_stat(self, user, success=True): def _rotate_session_id(self): """ Upon successful authentication when using the database session backend, - we need to rotate the session id + we need to rotate the session id. """ if conf.get("webserver", "SESSION_BACKEND") == "database": session.sid = str(uuid4()) def auth_user_db(self, username, password): """ - Method for authenticating user, auth db style + Method for authenticating user, auth db style. :param username: The username or registered email address @@ -1238,7 +1237,7 @@ def auth_user_ldap(self, username, password): def auth_user_oid(self, email): """ - Openid user Authentication + Openid user Authentication. :param email: user's email to authenticate """ @@ -1253,7 +1252,7 @@ def auth_user_oid(self, email): def auth_user_remote_user(self, username): """ - REMOTE_USER user Authentication + REMOTE_USER user Authentication. :param username: user's username for remote auth """ @@ -1370,7 +1369,7 @@ def auth_user_oauth(self, userinfo): return None def _has_access_builtin_roles(self, role, action_name: str, resource_name: str) -> bool: - """Checks permission on builtin role""" + """Checks permission on builtin role.""" perms = self.builtin_roles.get(role.name, []) for (_resource_name, _action_name) in perms: if re.match(_resource_name, resource_name) and re.match(_action_name, action_name): @@ -1381,9 +1380,9 @@ def _get_user_permission_resources( self, user: User | None, action_name: str, resource_names: list[str] | None = None ) -> set[str]: """ - Return a set of resource names with a certain action name - that a user has access to. Mainly used to fetch all menu permissions - on a single db call, will also check public permissions and builtin roles + Return a set of resource names with a certain action name that a user has access to. + Mainly used to fetch all menu permissions on a single db call, will also check public permissions + and builtin roles. """ if not resource_names: resource_names = [] @@ -1444,7 +1443,7 @@ def add_limit_view(self, baseview): def add_permissions_view(self, base_action_names, resource_name): # Keep name for compatibility with FAB. """ - Adds an action on a resource to the backend + Adds an action on a resource to the backend. :param base_action_names: list of permissions from view (all exposed methods): @@ -1490,7 +1489,7 @@ def add_permissions_view(self, base_action_names, resource_name): # Keep name f def add_permissions_menu(self, resource_name): """ - Adds menu_access to resource on permission_resource + Adds menu_access to resource on permission_resource. :param resource_name: The resource name @@ -1505,7 +1504,7 @@ def add_permissions_menu(self, resource_name): def security_cleanup(self, baseviews, menus): """ - Will cleanup all unused permissions from the database + Will cleanup all unused permissions from the database. :param baseviews: A list of BaseViews class :param menus: Menu class @@ -1529,47 +1528,47 @@ def security_cleanup(self, baseviews, menus): self.delete_resource(resource.name) def find_register_user(self, registration_hash): - """Generic function to return user registration""" + """Generic function to return user registration.""" raise NotImplementedError def add_register_user(self, username, first_name, last_name, email, password="", hashed_password=""): - """Generic function to add user registration""" + """Generic function to add user registration.""" raise NotImplementedError def del_register_user(self, register_user): - """Generic function to delete user registration""" + """Generic function to delete user registration.""" raise NotImplementedError def get_user_by_id(self, pk): - """Generic function to return user by it's id (pk)""" + """Generic function to return user by it's id (pk).""" raise NotImplementedError def find_user(self, username=None, email=None): - """Generic function find a user by it's username or email""" + """Generic function find a user by it's username or email.""" raise NotImplementedError def get_all_users(self): - """Generic function that returns all existing users""" + """Generic function that returns all existing users.""" raise NotImplementedError def get_role_permissions_from_db(self, role_id: int) -> list[Permission]: - """Get all DB permissions from a role id""" + """Get all DB permissions from a role id.""" raise NotImplementedError def add_user(self, username, first_name, last_name, email, role, password=""): - """Generic function to create user""" + """Generic function to create user.""" raise NotImplementedError def update_user(self, user): """ - Generic function to update user + Generic function to update user. :param user: User model to update to database """ raise NotImplementedError def count_users(self): - """Generic function to count the existing users""" + """Generic function to count the existing users.""" raise NotImplementedError def find_role(self, name): @@ -1585,7 +1584,7 @@ def get_all_roles(self): raise NotImplementedError def get_public_role(self): - """Returns all permissions from public role""" + """Returns all permissions from public role.""" raise NotImplementedError def get_action(self, name: str) -> Action: @@ -1603,12 +1602,12 @@ def filter_roles_by_perm_with_action(self, permission_name: str, role_ids: list[ def permission_exists_in_one_or_more_roles( self, resource_name: str, action_name: str, role_ids: list[int] ) -> bool: - """Finds and returns permission views for a group of roles""" + """Finds and returns permission views for a group of roles.""" raise NotImplementedError def create_action(self, name): """ - Adds a permission to the backend, model permission + Adds a permission to the backend, model permission. :param name: name of the permission: 'can_add','can_edit' etc... @@ -1656,7 +1655,7 @@ def create_resource(self, name): def delete_resource(self, name): """ - Deletes a Resource from the backend + Deletes a Resource from the backend. :param name: name of the Resource @@ -1732,7 +1731,7 @@ def remove_permission_from_role(self, role, permission) -> None: raise NotImplementedError def load_user(self, user_id): - """Load user by ID""" + """Load user by ID.""" return self.get_user_by_id(int(user_id)) def load_user_jwt(self, _jwt_header, jwt_data): @@ -1744,5 +1743,5 @@ def load_user_jwt(self, _jwt_header, jwt_data): @staticmethod def before_request(): - """Hook runs before request""" + """Hook runs before request.""" g.user = current_user diff --git a/airflow/www/fab_security/sqla/manager.py b/airflow/www/fab_security/sqla/manager.py index e55255d67c212..62decfb184b0c 100644 --- a/airflow/www/fab_security/sqla/manager.py +++ b/airflow/www/fab_security/sqla/manager.py @@ -43,7 +43,7 @@ class SecurityManager(BaseSecurityManager): """ Responsible for authentication, registering security views, - role and permission auto management + role and permission auto management. If you want to change anything just inherit and override, then pass your own security manager to AppBuilder. @@ -60,9 +60,9 @@ class SecurityManager(BaseSecurityManager): def __init__(self, appbuilder): """ - Class constructor - param appbuilder: - F.A.B AppBuilder main object + Class constructor. + + :param appbuilder: F.A.B AppBuilder main object """ super().__init__(appbuilder) user_datamodel = SQLAInterface(self.user_model) @@ -142,7 +142,7 @@ def add_register_user(self, username, first_name, last_name, email, password="", def del_register_user(self, register_user): """ - Deletes registration object from database + Deletes registration object from database. :param register_user: RegisterUser object to delete """ @@ -156,7 +156,7 @@ def del_register_user(self, register_user): return False def find_user(self, username=None, email=None): - """Finds user by username or email""" + """Finds user by username or email.""" if username: try: if self.auth_username_ci: @@ -194,7 +194,7 @@ def add_user( password="", hashed_password="", ): - """Generic function to create user""" + """Generic function to create user.""" try: user = self.user_model() user.first_name = first_name @@ -285,7 +285,7 @@ def permission_exists_in_one_or_more_roles( ) -> bool: """ Method to efficiently check if a certain permission exists - on a list of role id's. This is used by `has_access` + on a list of role id's. This is used by `has_access`. :param resource_name: The view's name to check if exists on one of the roles :param action_name: The permission name to check if exists @@ -314,7 +314,7 @@ def permission_exists_in_one_or_more_roles( return self.appbuilder.get_session.query(q).scalar() def filter_roles_by_perm_with_action(self, action_name: str, role_ids: list[int]): - """Find roles with permission""" + """Find roles with permission.""" return ( self.appbuilder.get_session.query(self.permission_model) .join( @@ -332,7 +332,7 @@ def filter_roles_by_perm_with_action(self, action_name: str, role_ids: list[int] def create_action(self, name): """ - Adds an action to the backend, model action + Adds an action to the backend, model action. :param name: name of the action: 'can_add','can_edit' etc... @@ -417,7 +417,7 @@ def create_resource(self, name) -> Resource: def delete_resource(self, name: str) -> bool: """ - Deletes a Resource from the backend + Deletes a Resource from the backend. :param name: name of the resource @@ -482,7 +482,7 @@ def get_resource_permissions(self, resource: Resource) -> Permission: def create_permission(self, action_name, resource_name) -> Permission | None: """ - Adds a permission on a resource to the backend + Adds a permission on a resource to the backend. :param action_name: name of the action to add: 'can_add','can_edit' etc... diff --git a/airflow/www/forms.py b/airflow/www/forms.py index d925498f574b3..8246846c9e488 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -94,7 +94,7 @@ def _get_default_timezone(self): class DateTimeForm(FlaskForm): - """Date filter form needed for task views""" + """Date filter form needed for task views.""" execution_date = DateTimeWithTimezoneField("Logical date", widget=AirflowDateTimePickerWidget()) @@ -102,7 +102,7 @@ class DateTimeForm(FlaskForm): class DateTimeWithNumRunsForm(FlaskForm): """ Date time and number of runs form for tree view, task duration - and landing times + and landing times. """ base_date = DateTimeWithTimezoneField( @@ -122,7 +122,7 @@ class DateTimeWithNumRunsForm(FlaskForm): class DateTimeWithNumRunsWithDagRunsForm(DateTimeWithNumRunsForm): - """Date time and number of runs and dag runs form for graph and gantt view""" + """Date time and number of runs and dag runs form for graph and gantt view.""" execution_date = SelectField("DAG run") @@ -154,7 +154,7 @@ def populate_obj(self, item): class TaskInstanceEditForm(DynamicForm): - """Form for editing TaskInstance""" + """Form for editing TaskInstance.""" dag_id = StringField(lazy_gettext("Dag Id"), validators=[InputRequired()], widget=BS3TextFieldROWidget()) task_id = StringField( diff --git a/airflow/www/security.py b/airflow/www/security.py index 50c484082ca65..b6bc48104c55a 100644 --- a/airflow/www/security.py +++ b/airflow/www/security.py @@ -58,7 +58,7 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin): - """Custom security manager, which introduces a permission model adapted to Airflow""" + """Custom security manager, which introduces a permission model adapted to Airflow.""" ########################################################################### # PERMISSIONS @@ -223,6 +223,7 @@ def _get_root_dag_id(self, dag_id: str) -> str: def init_role(self, role_name, perms) -> None: """ Initialize the role with actions and related resources. + :param role_name: :param perms: :return: @@ -254,7 +255,7 @@ def bulk_sync_roles(self, roles: Iterable[dict[str, Any]]) -> None: def delete_role(self, role_name: str) -> None: """ - Delete the given Role + Delete the given Role. :param role_name: the name of a role in the ab_role table """ @@ -441,7 +442,7 @@ def has_access(self, action_name: str, resource_name: str, user=None) -> bool: return False def _has_role(self, role_name_or_list: Container, user) -> bool: - """Whether the user has this role name""" + """Whether the user has this role name.""" if not isinstance(role_name_or_list, list): role_name_or_list = [role_name_or_list] return any(r.name in role_name_or_list for r in user.roles) @@ -462,15 +463,15 @@ def has_all_dags_access(self, user) -> bool: ) def can_edit_all_dags(self, user=None) -> bool: - """Has can_edit action on DAG resource""" + """Has can_edit action on DAG resource.""" return self.has_access(permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG, user) def can_read_all_dags(self, user=None) -> bool: - """Has can_read action on DAG resource""" + """Has can_read action on DAG resource.""" return self.has_access(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG, user) def clean_perms(self) -> None: - """FAB leaves faulty permissions that need to be cleaned up""" + """FAB leaves faulty permissions that need to be cleaned up.""" self.log.debug("Cleaning faulty perms") sesh = self.appbuilder.get_session perms = sesh.query(Permission).filter( @@ -526,7 +527,7 @@ def add_homepage_access_to_custom_roles(self) -> None: self.appbuilder.get_session.commit() def get_all_permissions(self) -> set[tuple[str, str]]: - """Returns all permissions as a set of tuples with the action and resource names""" + """Returns all permissions as a set of tuples with the action and resource names.""" return set( self.appbuilder.get_session.query(self.permission_model) .join(self.permission_model.action) @@ -553,7 +554,7 @@ def _get_all_non_dag_permissions(self) -> dict[tuple[str, str], Permission]: } def _get_all_roles_with_permissions(self) -> dict[str, Role]: - """Returns a dict with a key of role name and value of role with early loaded permissions""" + """Returns a dict with a key of role name and value of role with early loaded permissions.""" return { r.name: r for r in self.appbuilder.get_session.query(self.role_model).options( @@ -643,7 +644,7 @@ def sync_perm_for_dag( ) -> None: """ Sync permissions for given dag id. The dag id surely exists in our dag bag - as only / refresh button or DagBag will call this function + as only / refresh button or DagBag will call this function. :param dag_id: the ID of the DAG whose permissions should be updated :param access_control: a dict where each key is a rolename and @@ -784,7 +785,7 @@ def __init__(self, session: Session | None = None) -> None: class ApplessAirflowSecurityManager(AirflowSecurityManager): - """Security Manager that doesn't need the whole flask app""" + """Security Manager that doesn't need the whole flask app.""" def __init__(self, session: Session | None = None): self.appbuilder = FakeAppBuilder(session) diff --git a/airflow/www/session.py b/airflow/www/session.py index 9e7a0b5392bde..4e23d212fe2ea 100644 --- a/airflow/www/session.py +++ b/airflow/www/session.py @@ -22,7 +22,7 @@ class SesssionExemptMixin: - """Exempt certain blueprints/paths from autogenerated sessions""" + """Exempt certain blueprints/paths from autogenerated sessions.""" def save_session(self, *args, **kwargs): """Prevent creating session from REST API and health requests.""" @@ -34,8 +34,8 @@ def save_session(self, *args, **kwargs): class AirflowDatabaseSessionInterface(SesssionExemptMixin, SqlAlchemySessionInterface): - """Session interface that exempts some routes and stores session data in the database""" + """Session interface that exempts some routes and stores session data in the database.""" class AirflowSecureCookieSessionInterface(SesssionExemptMixin, SecureCookieSessionInterface): - """Session interface that exempts some routes and stores session data in a signed cookie""" + """Session interface that exempts some routes and stores session data in a signed cookie.""" diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 0f17984afac09..b018afb823b7f 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -219,7 +219,7 @@ def should_hide_value_for_key(key_name): def get_params(**kwargs): - """Return URL-encoded params""" + """Return URL-encoded params.""" return urlencode({d: v for d, v in kwargs.items() if v is not None}, True) @@ -386,12 +386,12 @@ def is_current(current, page): def epoch(dttm): - """Returns an epoch-type date (tuple with no timezone)""" + """Returns an epoch-type date (tuple with no timezone).""" return (int(time.mktime(dttm.timetuple())) * 1000,) def make_cache_key(*args, **kwargs): - """Used by cache to get a unique key per URL""" + """Used by cache to get a unique key per URL.""" path = request.path args = str(hash(frozenset(request.args.items()))) return (path + args).encode("ascii", "ignore") @@ -426,7 +426,7 @@ def task_instance_link(attr): def state_token(state): - """Returns a formatted string with HTML for a given State""" + """Returns a formatted string with HTML for a given State.""" color = State.color(state) fg_color = State.color_fg(state) return Markup( @@ -438,13 +438,13 @@ def state_token(state): def state_f(attr): - """Gets 'state' & returns a formatted string with HTML for a given State""" + """Gets 'state' & returns a formatted string with HTML for a given State.""" state = attr.get("state") return state_token(state) def nobr_f(attr_name): - """Returns a formatted string with HTML with a Non-breaking Text element""" + """Returns a formatted string with HTML with a Non-breaking Text element.""" def nobr(attr): f = attr.get(attr_name) @@ -454,7 +454,7 @@ def nobr(attr): def datetime_f(attr_name): - """Returns a formatted string with HTML for given DataTime""" + """Returns a formatted string with HTML for given DataTime.""" def dt(attr): f = attr.get(attr_name) @@ -464,7 +464,7 @@ def dt(attr): def datetime_html(dttm: DateTime | None) -> str: - """Return an HTML formatted string with time element to support timezone changes in UI""" + """Return an HTML formatted string with time element to support timezone changes in UI.""" as_iso = dttm.isoformat() if dttm else "" if not as_iso: return Markup("") @@ -476,7 +476,7 @@ def datetime_html(dttm: DateTime | None) -> str: def json_f(attr_name): - """Returns a formatted string with HTML for given JSON serializable""" + """Returns a formatted string with HTML for given JSON serializable.""" def json_(attr): f = attr.get(attr_name) @@ -542,12 +542,12 @@ def format_map_index(attr: dict) -> str: def pygment_html_render(s, lexer=lexers.TextLexer): - """Highlight text using a given Lexer""" + """Highlight text using a given Lexer.""" return highlight(s, lexer(), HtmlFormatter(linenos=True)) def render(obj, lexer): - """Render a given Python object with a given Pygments lexer""" + """Render a given Python object with a given Pygments lexer.""" out = "" if isinstance(obj, str): out = Markup(pygment_html_render(obj, lexer)) @@ -563,7 +563,7 @@ def render(obj, lexer): def json_render(obj, lexer): - """Render a given Python object with json lexer""" + """Render a given Python object with json lexer.""" out = "" if isinstance(obj, str): out = Markup(pygment_html_render(obj, lexer)) @@ -583,7 +583,7 @@ def wrapped_markdown(s, css_class="rich_doc"): def get_attr_renderer(): - """Return Dictionary containing different Pygments Lexers for Rendering & Highlighting""" + """Return Dictionary containing different Pygments Lexers for Rendering & Highlighting.""" return { "bash": lambda x: render(x, lexers.BashLexer), "bash_command": lambda x: render(x, lexers.BashLexer), @@ -615,7 +615,7 @@ def get_chart_height(dag): approximate the size of generated chart (otherwise the charts are tiny and unreadable when DAGs have a large number of tasks). Ideally nvd3 should allow for dynamic-height charts, that is charts that take up space based on the size of the components within. - TODO(aoen): See [AIRFLOW-1263] + TODO(aoen): See [AIRFLOW-1263]. """ return 600 + len(dag.tasks) * 10 @@ -791,7 +791,7 @@ def is_utcdatetime(self, col_name): return False def is_extendedjson(self, col_name): - """Checks if it is a special extended JSON type""" + """Checks if it is a special extended JSON type.""" from airflow.utils.sqlalchemy import ExtendedJSON if col_name in self.list_columns: @@ -841,7 +841,7 @@ def delete_all(self, items: list[Model]) -> bool: class UIAlert: """ - Helper for alerts messages shown on the UI + Helper for alerts messages shown on the UI. :param message: The message to display, either a string or Markup :param category: The category of the message, one of "info", "warning", "error", or any custom category. diff --git a/airflow/www/validators.py b/airflow/www/validators.py index d3e75b3888724..8dbe640e8cc86 100644 --- a/airflow/www/validators.py +++ b/airflow/www/validators.py @@ -82,7 +82,7 @@ def __call__(self, form, field): class ValidKey: """ - Validates values that will be used as keys + Validates values that will be used as keys. :param max_length: The maximum length of the given key diff --git a/airflow/www/views.py b/airflow/www/views.py index b47a7106b6351..85e1a223d36ec 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -147,14 +147,14 @@ def truncate_task_duration(task_duration): """ Cast the task_duration to an int was for optimization for large/huge dags if task_duration > 10s - otherwise we keep it as a float with 3dp + otherwise we keep it as a float with 3dp. """ return int(task_duration) if task_duration > 10.0 else round(task_duration, 3) def sanitize_args(args: dict[str, str]) -> dict[str, str]: """ - Remove all parameters starting with `_` + Remove all parameters starting with `_`. :param args: arguments of request :return: copy of the dictionary passed as input with args starting with `_` removed. @@ -163,7 +163,7 @@ def sanitize_args(args: dict[str, str]) -> dict[str, str]: def get_safe_url(url): - """Given a user-supplied URL, ensure it points to our web server""" + """Given a user-supplied URL, ensure it points to our web server.""" if not url: return url_for("Airflow.index") @@ -182,7 +182,7 @@ def get_safe_url(url): def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): - """Get Execution Data, Base Date & Number of runs from a Request""" + """Get Execution Data, Base Date & Number of runs from a Request.""" date_time = www_request.args.get("execution_date") run_id = www_request.args.get("run_id") # First check run id, then check execution date, if not fall back on the latest dagrun @@ -247,7 +247,7 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): def _safe_parse_datetime(v, allow_empty=False) -> datetime.datetime | None: """ - Parse datetime and return error message for invalid dates + Parse datetime and return error message for invalid dates. :param v: the string value to be parsed :param allow_empty: Set True to return none if empty str or None @@ -489,7 +489,7 @@ def get_mapped_group_summary(run_id: str, mapped_instances: Mapping[int, list[Ta def get_key_paths(input_dict): - """Return a list of dot-separated dictionary paths""" + """Return a list of dot-separated dictionary paths.""" for key, value in input_dict.items(): if isinstance(value, dict): for sub_key in get_key_paths(value): @@ -499,7 +499,7 @@ def get_key_paths(input_dict): def get_value_from_path(key_path, content): - """Return the value from a dictionary based on dot-separated path of keys""" + """Return the value from a dictionary based on dot-separated path of keys.""" elem = content for x in key_path.strip(".").split("."): try: @@ -539,7 +539,7 @@ def get_task_stats_from_query(qry): def redirect_or_json(origin, msg, status="", status_code=200): """ Some endpoints are called by javascript, - returning json will allow us to more elegantly handle side-effects in-page + returning json will allow us to more elegantly handle side-effects in-page. """ if request.headers.get("Accept") == "application/json": if status == "error" and status_code == 200: @@ -559,7 +559,7 @@ def redirect_or_json(origin, msg, status="", status_code=200): def not_found(error): - """Show Not Found on screen for any error in the Webserver""" + """Show Not Found on screen for any error in the Webserver.""" return ( render_template( "airflow/error.html", @@ -572,7 +572,7 @@ def not_found(error): def method_not_allowed(error): - """Show Method Not Allowed on screen for any error in the Webserver""" + """Show Method Not Allowed on screen for any error in the Webserver.""" return ( render_template( "airflow/error.html", @@ -585,7 +585,7 @@ def method_not_allowed(error): def show_traceback(error): - """Show Traceback for a given error""" + """Show Traceback for a given error.""" return ( render_template( "airflow/traceback.html", @@ -608,7 +608,7 @@ def show_traceback(error): class AirflowBaseView(BaseView): - """Base View to set Airflow related properties""" + """Base View to set Airflow related properties.""" from airflow import macros @@ -1034,7 +1034,7 @@ def dag_stats(self, session: Session = NEW_SESSION): ) @provide_session def task_stats(self, session: Session = NEW_SESSION): - """Task Statistics""" + """Task Statistics.""" allowed_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) if not allowed_dag_ids: @@ -1141,7 +1141,7 @@ def task_stats(self, session: Session = NEW_SESSION): ) @provide_session def last_dagruns(self, session: Session = NEW_SESSION): - """Last DAG runs""" + """Last DAG runs.""" allowed_dag_ids = get_airflow_app().appbuilder.sm.get_accessible_dag_ids(g.user) # Filter by post parameters @@ -2260,7 +2260,7 @@ def clear(self, *, session: Session = NEW_SESSION): @action_logging @provide_session def dagrun_clear(self, *, session: Session = NEW_SESSION): - """Clears the DagRun""" + """Clears the DagRun.""" dag_id = request.form.get("dag_id") dag_run_id = request.form.get("dag_run_id") confirmed = request.form.get("confirmed") == "true" @@ -2422,7 +2422,7 @@ def dagrun_failed(self): ) @action_logging def dagrun_success(self): - """Mark DagRun success""" + """Mark DagRun success.""" dag_id = request.form.get("dag_id") dag_run_id = request.form.get("dag_run_id") confirmed = request.form.get("confirmed") == "true" @@ -2849,7 +2849,7 @@ def legacy_calendar(self): @action_logging @provide_session def calendar(self, dag_id: str, session: Session = NEW_SESSION): - """Get DAG runs as calendar""" + """Get DAG runs as calendar.""" def _convert_to_date(session, column): """Convert column to date.""" @@ -3589,7 +3589,7 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): @provide_session def extra_links(self, *, session: Session = NEW_SESSION): """ - A restful endpoint that returns external links for a given Operator + A restful endpoint that returns external links for a given Operator. It queries the operator that sent the request for the links it wishes to provide for a given external link name. @@ -3650,7 +3650,7 @@ def extra_links(self, *, session: Session = NEW_SESSION): @action_logging @provide_session def graph_data(self, session: Session = NEW_SESSION): - """Get Graph Data""" + """Get Graph Data.""" dag_id = request.args.get("dag_id") dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) root = request.args.get("root") @@ -3709,7 +3709,7 @@ def task_instances(self): ] ) def grid_data(self): - """Returns grid data""" + """Returns grid data.""" dag_id = request.args.get("dag_id") dag = get_airflow_app().dag_bag.get_dag(dag_id) @@ -3764,7 +3764,7 @@ def grid_data(self): @expose("/object/next_run_datasets/") @auth.has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG)]) def next_run_datasets(self, dag_id): - """Returns datasets necessary, and their status, for the next dag run""" + """Returns datasets necessary, and their status, for the next dag run.""" dag = get_airflow_app().dag_bag.get_dag(dag_id) if not dag: return {"error": f"can't find dag {dag_id}"}, 404 @@ -3844,7 +3844,7 @@ def dataset_dependencies(self): @auth.has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_DATASET)]) def datasets_summary(self): """Get a summary of datasets, including the datetime they were last updated and how many updates - they've ever had + they've ever had. """ allowed_attrs = ["uri", "last_dataset_update"] @@ -4023,7 +4023,7 @@ def audit_log(self, dag_id: str, session: Session = NEW_SESSION): class ConfigurationView(AirflowBaseView): - """View to show Airflow Configurations""" + """View to show Airflow Configurations.""" default_view = "conf" @@ -4099,7 +4099,7 @@ def conf(self): class RedocView(AirflowBaseView): - """Redoc Open API documentation""" + """Redoc Open API documentation.""" default_view = "redoc" @@ -4116,7 +4116,7 @@ def redoc(self): class DagFilter(BaseFilter): - """Filter using DagIDs""" + """Filter using DagIDs.""" def apply(self, query, func): if get_airflow_app().appbuilder.sm.has_all_dags_access(g.user): @@ -4223,7 +4223,7 @@ def check_dag_edit_acl_for_actions( class SlaMissModelView(AirflowModelView): - """View to show SlaMiss table""" + """View to show SlaMiss table.""" route_base = "/slamiss" @@ -4326,7 +4326,7 @@ def _set_notification_property( class XComModelView(AirflowModelView): - """View to show records from XCom table""" + """View to show records from XCom table.""" route_base = "/xcom" @@ -4403,7 +4403,7 @@ def pre_update(self, item): # These field behaviours and testable connection types are rendered as scripts in the conn_create.html and # conn_edit.html templates. class ConnectionFormWidget(FormWidget): - """Form widget used to display connection""" + """Form widget used to display connection.""" @cached_property def field_behaviours(self) -> str: @@ -4432,7 +4432,7 @@ def refresh(obj=None): class ConnectionModelView(AirflowModelView): - """View to show records from Connections table""" + """View to show records from Connections table.""" route_base = "/connection" @@ -4556,7 +4556,7 @@ def action_muldelete(self, items): ] ) def action_mulduplicate(self, connections, session: Session = NEW_SESSION): - """Duplicate Multiple connections""" + """Duplicate Multiple connections.""" for selected_conn in connections: new_conn_id = selected_conn.conn_id match = re.search(r"_copy(\d+)$", selected_conn.conn_id) @@ -4683,7 +4683,7 @@ def prefill_form(self, form, pk): class PluginView(AirflowBaseView): - """View to show Airflow Plugins""" + """View to show Airflow Plugins.""" default_view = "list" @@ -4749,7 +4749,7 @@ def list(self): class ProviderView(AirflowBaseView): - """View to show Airflow Providers""" + """View to show Airflow Providers.""" default_view = "list" @@ -4807,7 +4807,7 @@ def _build_link(match_obj): class PoolModelView(AirflowModelView): - """View to show records from Pool table""" + """View to show records from Pool table.""" route_base = "/pool" @@ -4916,7 +4916,7 @@ def _can_create_variable() -> bool: class VariableModelView(AirflowModelView): - """View to show records from Variable table""" + """View to show records from Variable table.""" route_base = "/variable" @@ -4955,7 +4955,7 @@ class VariableModelView(AirflowModelView): base_order = ("key", "asc") def hidden_field_formatter(self): - """Formats hidden fields""" + """Formats hidden fields.""" key = self.get("key") val = self.get("val") if secrets_masker.should_hide_value_for_key(key): @@ -5025,7 +5025,7 @@ def action_varexport(self, items): @auth.has_access([(permissions.ACTION_CAN_CREATE, permissions.RESOURCE_VARIABLE)]) @action_logging(event=f"{permissions.RESOURCE_VARIABLE.lower()}.varimport") def varimport(self): - """Import variables""" + """Import variables.""" try: variable_dict = json.loads(request.files["file"].read()) except Exception: @@ -5050,7 +5050,7 @@ def varimport(self): class JobModelView(AirflowModelView): - """View to show records from Job table""" + """View to show records from Job table.""" route_base = "/job" @@ -5104,7 +5104,7 @@ class JobModelView(AirflowModelView): class DagRunModelView(AirflowPrivilegeVerifierModelView): - """View to show records from DagRun table""" + """View to show records from DagRun table.""" route_base = "/dagrun" @@ -5334,7 +5334,7 @@ def action_clear(self, drs: list[DagRun], session: Session = NEW_SESSION): class LogModelView(AirflowModelView): - """View to show records from Log table""" + """View to show records from Log table.""" route_base = "/log" @@ -5368,7 +5368,7 @@ class LogModelView(AirflowModelView): class TaskRescheduleModelView(AirflowModelView): - """View to show records from Task Reschedule table""" + """View to show records from Task Reschedule table.""" route_base = "/taskreschedule" @@ -5438,7 +5438,7 @@ def duration_f(self): class TriggerModelView(AirflowModelView): - """View to show records from Task Reschedule table""" + """View to show records from Task Reschedule table.""" route_base = "/triggerview" @@ -5478,7 +5478,7 @@ class TriggerModelView(AirflowModelView): class TaskInstanceModelView(AirflowPrivilegeVerifierModelView): - """View to show records from TaskInstance table""" + """View to show records from TaskInstance table.""" route_base = "/taskinstance" @@ -5661,7 +5661,7 @@ def set_task_instance_state(self, tis, target_state, session: Session = NEW_SESS @action_has_dag_edit_access @action_logging def action_set_running(self, tis): - """Set state to 'running'""" + """Set state to 'running'.""" self.set_task_instance_state(tis, State.RUNNING) self.update_redirect() return redirect(self.get_redirect()) @@ -5670,7 +5670,7 @@ def action_set_running(self, tis): @action_has_dag_edit_access @action_logging def action_set_failed(self, tis): - """Set state to 'failed'""" + """Set state to 'failed'.""" self.set_task_instance_state(tis, State.FAILED) self.update_redirect() return redirect(self.get_redirect()) @@ -5679,7 +5679,7 @@ def action_set_failed(self, tis): @action_has_dag_edit_access @action_logging def action_set_success(self, tis): - """Set state to 'success'""" + """Set state to 'success'.""" self.set_task_instance_state(tis, State.SUCCESS) self.update_redirect() return redirect(self.get_redirect()) @@ -5688,7 +5688,7 @@ def action_set_success(self, tis): @action_has_dag_edit_access @action_logging def action_set_retry(self, tis): - """Set state to 'up_for_retry'""" + """Set state to 'up_for_retry'.""" self.set_task_instance_state(tis, State.UP_FOR_RETRY) self.update_redirect() return redirect(self.get_redirect()) @@ -5704,7 +5704,7 @@ def action_set_skipped(self, tis): class AutocompleteView(AirflowBaseView): - """View to provide autocomplete results""" + """View to provide autocomplete results.""" @auth.has_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG)]) @provide_session @@ -5752,7 +5752,7 @@ def autocomplete(self, session: Session = NEW_SESSION): class DagDependenciesView(AirflowBaseView): - """View to show dependencies between DAGs""" + """View to show dependencies between DAGs.""" refresh_interval = datetime.timedelta( seconds=conf.getint( @@ -5774,7 +5774,7 @@ class DagDependenciesView(AirflowBaseView): @gzipped @action_logging def list(self): - """Display DAG dependencies""" + """Display DAG dependencies.""" title = "DAG Dependencies" if not self.nodes or not self.edges: diff --git a/airflow/www/widgets.py b/airflow/www/widgets.py index 7af5fa94ee8a7..d1a8539ec906a 100644 --- a/airflow/www/widgets.py +++ b/airflow/www/widgets.py @@ -24,13 +24,13 @@ class AirflowModelListWidget(RenderTemplateWidget): - """Airflow model list""" + """Airflow model list.""" template = "airflow/model_list.html" class AirflowDateTimePickerWidget: - """Airflow date time picker widget""" + """Airflow date time picker widget.""" data_template = ( '
' @@ -53,7 +53,7 @@ def __call__(self, field, **kwargs): class AirflowDateTimePickerROWidget(AirflowDateTimePickerWidget): - """Airflow Read-only date time picker widget""" + """Airflow Read-only date time picker widget.""" def __call__(self, field, **kwargs): kwargs["readonly"] = "true" @@ -61,7 +61,7 @@ def __call__(self, field, **kwargs): class BS3TextFieldROWidget(BS3TextFieldWidget): - """Read-only single-line text input Widget (BS3TextFieldWidget)""" + """Read-only single-line text input Widget (BS3TextFieldWidget).""" def __call__(self, field, **kwargs): kwargs["readonly"] = "true" @@ -69,7 +69,7 @@ def __call__(self, field, **kwargs): class BS3TextAreaROWidget(BS3TextAreaFieldWidget): - """Read-only multi-line text area Widget (BS3TextAreaROWidget)""" + """Read-only multi-line text area Widget (BS3TextAreaROWidget).""" def __call__(self, field, **kwargs): kwargs["readonly"] = "true" @@ -77,6 +77,6 @@ def __call__(self, field, **kwargs): class AirflowVariableShowWidget(RenderTemplateWidget): - """Airflow variable show widget""" + """Airflow variable show widget.""" template = "airflow/variable_show_widget.html" diff --git a/pyproject.toml b/pyproject.toml index 70fb24e6ae219..6e87f1cbb0730 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ extend-select = [ "D106", "D2", "D3", + # "D400", WIP: see #31135 # "D401", # Not enabled by ruff, but we don't want it "D402", "D403", diff --git a/setup.py b/setup.py index b8e4021f5b97c..9786e85acc40b 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def fill_provider_dependencies() -> dict[str, dict[str, list[str]]]: def airflow_test_suite() -> unittest.TestSuite: - """Test suite for Airflow tests""" + """Test suite for Airflow tests.""" test_loader = unittest.TestLoader() test_suite = test_loader.discover(str(AIRFLOW_SOURCES_ROOT / "tests"), pattern="test_*.py") return test_suite @@ -103,7 +103,7 @@ def finalize_options(self) -> None: @staticmethod def rm_all_files(files: list[str]) -> None: - """Remove all files from the list""" + """Remove all files from the list.""" for file in files: try: os.remove(file) @@ -719,7 +719,7 @@ def sort_extras_dependencies() -> dict[str, list[str]]: def get_provider_package_name_from_package_id(package_id: str) -> str: """ - Builds the name of provider package out of the package id provided/ + Builds the name of provider package out of the package id provided/. :param package_id: id of the package (like amazon or microsoft.azure) :return: full name of package in PyPI @@ -734,7 +734,7 @@ def get_excluded_providers() -> list[str]: def get_all_provider_packages() -> str: - """Returns all provider packages configured in setup.py""" + """Returns all provider packages configured in setup.py.""" excluded_providers = get_excluded_providers() return " ".join( get_provider_package_name_from_package_id(package) @@ -744,7 +744,7 @@ def get_all_provider_packages() -> str: class AirflowDistribution(Distribution): - """The setuptools.Distribution subclass with Airflow specific behaviour""" + """The setuptools.Distribution subclass with Airflow specific behaviour.""" def __init__(self, attrs=None): super().__init__(attrs)