Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion airflow-core/src/airflow/utils/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,11 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool:
for rule in rules:
if not isinstance(rule, _GlobIgnoreRule):
raise ValueError(f"_GlobIgnoreRule cannot match rules of type: {type(rule)}")
rel_path = str(path.relative_to(rule.relative_to) if rule.relative_to else path.name)
rel_obj = path.relative_to(rule.relative_to) if rule.relative_to else Path(path.name)
if path.is_dir():
rel_path = f"{rel_obj.as_posix()}/"
else:
rel_path = rel_obj.as_posix()
if (
rule.wild_match_pattern.include is not None
and rule.wild_match_pattern.match_file(rel_path) is not None
Expand Down
11 changes: 6 additions & 5 deletions airflow-core/tests/unit/plugins/test_plugin_ignore.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,20 +94,21 @@ def test_find_not_should_ignore_path_glob(self, tmp_path):
should_ignore_files = {
"test_notload.py",
"test_notload_sub.py",
"test_noneload_sub1.py",
"subdir1/test_noneload_sub1.py",
"subdir2/test_shouldignore.py",
"subdir3/test_notload_sub3.py",
}
should_not_ignore_files = {
"test_load.py",
"test_load_sub1.py",
"test_shouldignore.py", # moved to here because it should not ignore, as we do not ignore all
# things from subdir 2
"subdir1/test_load_sub1.py",
}
ignore_list_file = ".airflowignore_glob"
print("-" * 20)
for file_path in find_path_from_directory(plugin_folder_path, ignore_list_file, "glob"):
file_path = Path(file_path)
if file_path.is_file() and file_path.suffix == ".py":
detected_files.add(file_path.name)
rel_path = file_path.relative_to(plugin_folder_path).as_posix()
detected_files.add(rel_path)
print(file_path)

print("-" * 20)
Expand Down
65 changes: 65 additions & 0 deletions airflow-core/tests/unit/utils/test_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,71 @@ def test_might_contain_dag_with_default_callable(self):

assert file_utils.might_contain_dag(file_path=file_path_with_dag, safe_mode=True)

def test_airflowignore_negation_unignore_subfolder_file_glob(self, tmp_path):
"""Ensure negation rules can unignore a subfolder and a file inside it when using glob syntax.

Patterns:
* -> ignore everything
!subfolder/ -> unignore the subfolder (must match directory rule)
!subfolder/keep.py -> unignore a specific file inside the subfolder
"""
dags_root = tmp_path / "dags"
(dags_root / "subfolder").mkdir(parents=True)
# files
(dags_root / "drop.py").write_text("raise Exception('ignored')\n")
(dags_root / "subfolder" / "keep.py").write_text("# should be discovered\n")
(dags_root / "subfolder" / "drop.py").write_text("raise Exception('ignored')\n")

(dags_root / ".airflowignore").write_text(
"\n".join(
[
"*",
"!subfolder/",
"!subfolder/keep.py",
]
)
)

detected = set()
for raw in find_path_from_directory(dags_root, ".airflowignore", "glob"):
p = Path(raw)
if p.is_file() and p.suffix == ".py":
detected.add(p.relative_to(dags_root).as_posix())

assert detected == {"subfolder/keep.py"}

def test_airflowignore_negation_nested_with_globstar(self, tmp_path):
"""Negation with ** should work for nested subfolders."""
dags_root = tmp_path / "dags"
nested = dags_root / "a" / "b" / "subfolder"
nested.mkdir(parents=True)

# files
(dags_root / "ignore_top.py").write_text("raise Exception('ignored')\n")
(nested / "keep.py").write_text("# should be discovered\n")
(nested / "drop.py").write_text("raise Exception('ignored')\n")

(dags_root / ".airflowignore").write_text(
"\n".join(
[
"*",
"!a/",
"!a/b/",
"!**/subfolder/",
"!**/subfolder/keep.py",
"drop.py",
]
)
)

detected = set()
for raw in find_path_from_directory(dags_root, ".airflowignore", "glob"):
p = Path(raw)
if p.is_file() and p.suffix == ".py":
detected.add(p.relative_to(dags_root).as_posix())

assert detected == {"a/b/subfolder/keep.py"}

@conf_vars({("core", "might_contain_dag_callable"): "unit.utils.test_file.might_contain_dag"})
def test_might_contain_dag(self):
"""Test might_contain_dag_callable"""
Expand Down