From a4c04bf61498e9cff0c04460e8d50a2905fe98f5 Mon Sep 17 00:00:00 2001 From: ssuryad-amzn Date: Wed, 14 Jul 2021 16:32:59 -0700 Subject: [PATCH] Disable timeout on EBS volumes Timeouts on EBS volumes can occur due to various reasons, such as network outages, and can often be recovered after some time. Timeouts are handled by Linux by remounting the underlying filesystem as readonly which leads to a bad customer experience. While the I/O timeout can be specified for the nvme driver via the kernel commandline or modprobe.d, udev rules are the preferred way of setting per device timeouts. This is required as local NVMe volumes must have a finite timeout. --- 70-ec2-nvme-devices.rules | 4 ++++ amazon-ec2-utils.spec | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/70-ec2-nvme-devices.rules b/70-ec2-nvme-devices.rules index 4023d15..3df1ecd 100644 --- a/70-ec2-nvme-devices.rules +++ b/70-ec2-nvme-devices.rules @@ -14,5 +14,9 @@ KERNEL=="nvme[0-9]*n[0-9]*p[0-9]*", ENV{DEVTYPE}=="partition", ATTRS{serial}=="? # ebs nvme devices KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/sbin/ebsnvme-id -u /dev/%k", SYMLINK+="%c" KERNEL=="nvme[0-9]*n[0-9]*p[0-9]*", ENV{DEVTYPE}=="partition", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/sbin/ebsnvme-id -u /dev/%k", SYMLINK+="%c%n" + +# Do not timeout I/O operations on EBS volumes. +KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", ATTR{queue/io_timeout}="4294967295" + # instance store nvme devices KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon EC2 NVMe Instance Storage", ATTR{queue/io_timeout}="90000" diff --git a/amazon-ec2-utils.spec b/amazon-ec2-utils.spec index 521fbf2..8f9f664 100644 --- a/amazon-ec2-utils.spec +++ b/amazon-ec2-utils.spec @@ -1,7 +1,7 @@ Name: amazon-ec2-utils Summary: A set of tools for running in EC2 Version: 1.3 -Release: 2%{?dist} +Release: 3%{?dist} License: MIT Group: System Tools @@ -88,6 +88,9 @@ rm -rf $RPM_BUILD_ROOT %{_sysconfdir}/udev/rules.d/70-ec2-nvme-devices.rules %changelog +* Thu Jul 14 2021 Sai Harsha 1.3.3 +- Disable timeout on EBS volumes + * Thu Oct 29 2020 Frederick Lefebvre 1.3-2 - Add testing of python syntax to spec file