Skip to content

Commit

Permalink
Refactor: Simplify code in utils (#33268)
Browse files Browse the repository at this point in the history
(cherry picked from commit cc8519d)
  • Loading branch information
eumiro authored and ephraimbuddy committed Aug 28, 2023
1 parent 3439f91 commit cff420b
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 8 deletions.
3 changes: 1 addition & 2 deletions airflow/utils/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import logging
import os
import zipfile
from collections import OrderedDict
from pathlib import Path
from typing import Generator, NamedTuple, Pattern, Protocol, overload

Expand Down Expand Up @@ -230,7 +229,7 @@ def _find_path_from_directory(
]
# evaluation order of patterns is important with negation
# so that later patterns can override earlier patterns
patterns = list(OrderedDict.fromkeys(patterns).keys())
patterns = list(dict.fromkeys(patterns))

dirs[:] = [subdir for subdir in dirs if not ignore_rule_type.match(Path(root) / subdir, patterns)]

Expand Down
11 changes: 5 additions & 6 deletions airflow/utils/log/file_task_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,12 +491,11 @@ def _init_file(self, ti):
@staticmethod
def _read_from_local(worker_log_path: Path) -> tuple[list[str], list[str]]:
messages = []
logs = []
files = list(worker_log_path.parent.glob(worker_log_path.name + "*"))
if files:
messages.extend(["Found local files:", *[f" * {x}" for x in sorted(files)]])
for file in sorted(files):
logs.append(Path(file).read_text())
paths = sorted(worker_log_path.parent.glob(worker_log_path.name + "*"))
if paths:
messages.append("Found local files:")
messages.extend(f" * {x}" for x in paths)
logs = [file.read_text() for file in paths]
return messages, logs

def _read_from_logs_server(self, ti, worker_log_rel_path) -> tuple[list[str], list[str]]:
Expand Down

0 comments on commit cff420b

Please sign in to comment.