Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix SFTPSensor.newer_than not working with jinja logical ds/ts expression #39056

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
bd1e9f9
Fixes https://github.com/apache/airflow/issues/36629
Apr 16, 2024
db0bf32
Fixes PR failed test
Apr 16, 2024
b2f784e
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland Apr 16, 2024
be1d171
Remove an parametrize duplicate tests
Apr 16, 2024
1519cad
Merge remote-tracking branch 'origin/fix-36629-sftp-sensor-newer-than…
Apr 16, 2024
130bc03
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland Apr 16, 2024
4757c9c
Fix formatting
Apr 17, 2024
d320ade
Merge remote-tracking branch 'origin/fix-36629-sftp-sensor-newer-than…
Apr 17, 2024
0828ebc
Fix formatting
Apr 17, 2024
c4737c3
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland Apr 17, 2024
3401b6e
Fixes https://github.com/apache/airflow/issues/36629
Apr 16, 2024
f350253
Fixes PR failed test
Apr 16, 2024
79e3f7b
Remove an parametrize duplicate tests
Apr 16, 2024
0f0a37f
update simple-salesforce type hints to support 1.12.6 (#39047)
hussein-awala Apr 16, 2024
4ced567
Fix formatting
Apr 17, 2024
d976f58
Add changelog for airflow python client 2.9.0 (#39060)
ephraimbuddy Apr 16, 2024
8b7dcc4
Upgrade to latest hatchling as build dependency (#39044)
potiuk Apr 16, 2024
6dc8c05
Prepare docs 1st wave (RC3) + ad hoc April 2024 (#38995) (#39054)
eladkal Apr 16, 2024
a06349e
[docs] update `DagBag` class docstring to include all params (#38814)
rawwar Apr 16, 2024
4616ced
Data aware scheduling docs edits (#38687)
lzdanski Apr 16, 2024
d8b4f23
Moves airflow import in deprecated pod_generator to local (#39062)
potiuk Apr 16, 2024
d1d2fca
KPO xcom sidecar PodDefault usage (#38951)
jedcunningham Apr 16, 2024
e8ba0d7
Fix formatting
Apr 17, 2024
1cd9e2f
Change date/time parsing method for newer_than parameter un SFTPSensor
Apr 19, 2024
7fdb2bf
Add examples in AWS auth manager documentation (#39040)
vincbeck Apr 16, 2024
ee81403
update document (#39068)
humit0 Apr 16, 2024
2d47202
Update hatchling to version 1.24.0 (#39072)
ephraimbuddy Apr 17, 2024
e55e09c
Check that the dataset<>task exists before trying to render graph (#3…
bbovenzi Apr 17, 2024
ca700e0
Merge remote-tracking branch 'origin/fix-36629-sftp-sensor-newer-than…
Apr 19, 2024
6ba4b7c
Change date/time parsing method for newer_than parameter un SFTPSensor
Apr 19, 2024
fe9b6a9
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland Apr 19, 2024
1daa5c2
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland Apr 19, 2024
ef0dbc1
Fix utc timezone in unit tests
May 6, 2024
1fea247
Merge remote-tracking branch 'origin/fix-36629-sftp-sensor-newer-than…
May 6, 2024
9dfec88
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland May 6, 2024
eb91af1
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland May 6, 2024
acb52be
Fix utc timezone in unit tests
May 6, 2024
c7b3721
Merge remote-tracking branch 'origin/fix-36629-sftp-sensor-newer-than…
May 6, 2024
0d22abb
Merge branch 'main' into fix-36629-sftp-sensor-newer-than-templating
grrolland May 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions airflow/providers/sftp/sensors/sftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from airflow.providers.sftp.hooks.sftp import SFTPHook
from airflow.providers.sftp.triggers.sftp import SFTPTrigger
from airflow.sensors.base import BaseSensorOperator, PokeReturnValue
from airflow.utils.timezone import convert_to_utc
from airflow.utils.timezone import convert_to_utc, parse

if TYPE_CHECKING:
from airflow.utils.context import Context
Expand All @@ -57,7 +57,7 @@ def __init__(
*,
path: str,
file_pattern: str = "",
newer_than: datetime | None = None,
newer_than: datetime | str | None = None,
sftp_conn_id: str = "sftp_default",
python_callable: Callable | None = None,
op_args: list | None = None,
Expand All @@ -70,7 +70,7 @@ def __init__(
self.file_pattern = file_pattern
self.hook: SFTPHook | None = None
self.sftp_conn_id = sftp_conn_id
self.newer_than: datetime | None = newer_than
self.newer_than: datetime | str | None = newer_than
self.python_callable: Callable | None = python_callable
self.op_args = op_args or []
self.op_kwargs = op_kwargs or {}
Expand Down Expand Up @@ -105,6 +105,8 @@ def poke(self, context: Context) -> PokeReturnValue | bool:
continue

if self.newer_than:
if isinstance(self.newer_than, str):
self.newer_than = parse(self.newer_than)
_mod_time = convert_to_utc(datetime.strptime(mod_time, "%Y%m%d%H%M%S"))
_newer_than = convert_to_utc(self.newer_than)
if _newer_than <= _mod_time:
Expand Down
20 changes: 17 additions & 3 deletions tests/providers/sftp/sensors/test_sftp.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# under the License.
from __future__ import annotations

from datetime import datetime
from datetime import datetime, timezone as stdlib_timezone
from unittest import mock
from unittest.mock import Mock, call, patch

Expand Down Expand Up @@ -97,11 +97,25 @@ def test_file_not_new_enough(self, sftp_hook_mock):
sftp_hook_mock.return_value.get_mod_time.assert_called_once_with("/path/to/file/1970-01-01.txt")
assert not output

@pytest.mark.parametrize(
"newer_than",
(
datetime(2020, 1, 2),
datetime(2020, 1, 2, tzinfo=stdlib_timezone.utc),
"2020-01-02",
"2020-01-02 00:00:00+00:00",
"2020-01-02 00:00:00.001+00:00",
"2020-01-02T00:00:00+00:00",
"2020-01-02T00:00:00Z",
"2020-01-02T00:00:00+04:00",
"2020-01-02T00:00:00.000001+04:00",
),
)
@patch("airflow.providers.sftp.sensors.sftp.SFTPHook")
def test_naive_datetime(self, sftp_hook_mock):
def test_multiple_datetime_format_in_newer_than(self, sftp_hook_mock, newer_than):
sftp_hook_mock.return_value.get_mod_time.return_value = "19700101000000"
sftp_sensor = SFTPSensor(
task_id="unit_test", path="/path/to/file/1970-01-01.txt", newer_than=datetime(2020, 1, 2)
task_id="unit_test", path="/path/to/file/1970-01-01.txt", newer_than=newer_than
)
context = {"ds": "1970-01-00"}
output = sftp_sensor.poke(context)
Expand Down