Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix retry_exponential_backoff divide by zero error when retry delay is zero #17003

Merged
merged 3 commits into from
Sep 30, 2021
Merged

Fix retry_exponential_backoff divide by zero error when retry delay is zero #17003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a76065
Add lower bound for min_backoff
edwardwang888 Sep 28, 2021
b821ac9
Fix test calculation
edwardwang888 Sep 28, 2021
b5acb5b
Expand test to cover zero seconds
edwardwang888 Sep 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions airflow/models/taskinstance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,14 @@ def next_retry_datetime(self):
# we must round up prior to converting to an int, otherwise a divide by zero error
# will occur in the modded_hash calculation.
min_backoff = int(math.ceil(delay.total_seconds() * (2 ** (self.try_number - 2))))

# In the case when delay.total_seconds() is 0, min_backoff will not be rounded up to 1.
# To address this, we impose a lower bound of 1 on min_backoff. This effectively makes
# the ceiling function unnecessary, but the ceiling function was retained to avoid
# introducing a breaking change.
if min_backoff < 1:
min_backoff = 1

# deterministic per task instance
ti_hash = int(
hashlib.sha1(
Expand Down
11 changes: 5 additions & 6 deletions tests/models/test_taskinstance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,13 +621,14 @@ def test_next_retry_datetime(self, dag_maker):
date = ti.next_retry_datetime()
assert date == ti.end_date + max_delay

def test_next_retry_datetime_short_intervals(self, dag_maker):
delay = datetime.timedelta(seconds=1)
@pytest.mark.parametrize("seconds", [0, 0.5, 1])
def test_next_retry_datetime_short_or_zero_intervals(self, dag_maker, seconds):
delay = datetime.timedelta(seconds=seconds)
max_delay = datetime.timedelta(minutes=60)

with dag_maker(dag_id='fail_dag'):
task = BashOperator(
task_id='task_with_exp_backoff_and_short_time_interval',
task_id='task_with_exp_backoff_and_short_or_zero_time_interval',
bash_command='exit 1',
retries=3,
retry_delay=delay,
Expand All @@ -639,9 +640,7 @@ def test_next_retry_datetime_short_intervals(self, dag_maker):
ti.end_date = pendulum.instance(timezone.utcnow())

date = ti.next_retry_datetime()
# between 1 * 2^0.5 and 1 * 2^1 (15 and 30)
period = ti.end_date.add(seconds=15) - ti.end_date.add(seconds=1)
assert date in period
assert date == ti.end_date + datetime.timedelta(seconds=1)

def test_reschedule_handling(self, dag_maker):
"""
Expand Down