From eb07e56eaaf7ed9090132446100e5d42aedca0ed Mon Sep 17 00:00:00 2001
From: Alexander Ostapenko <ao@tempesta-tech.com>
Date: Mon, 12 Nov 2018 20:32:05 +0700
Subject: [PATCH 1/5] Fix #1073: Add support MSI/MSI-X interrupts during NIC
 queues configuration.

---
 scripts/tempesta.sh |  3 ++-
 scripts/tfw_lib.sh  | 61 ++++++++++++++++++++++++---------------------
 2 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/scripts/tempesta.sh b/scripts/tempesta.sh
index c2770abd39..9ddddd58d1 100755
--- a/scripts/tempesta.sh
+++ b/scripts/tempesta.sh
@@ -43,7 +43,8 @@ tdb_mod=tempesta_db
 tfw_mod=tempesta_fw
 declare -r LONG_OPTS="help,load,unload,start,stop,restart,reload"
 
-declare devs=$(ip addr show up | awk '/^[0-9]+/ { sub(/:/, "", $2); print $2}')
+declare devs=$(ip addr show up | grep -P '^[0-9]+' | grep -Pv '\bLOOPBACK\b' \
+	       | awk '{ sub(/:/, "", $2); print $2}')
 
 usage()
 {
diff --git a/scripts/tfw_lib.sh b/scripts/tfw_lib.sh
index e85b5313b8..271c0e1485 100644
--- a/scripts/tfw_lib.sh
+++ b/scripts/tfw_lib.sh
@@ -36,27 +36,32 @@ calc()
 distribute_queues()
 {
 	dev=$1
-	RXQ_MAX=$(ethtool -l $dev 2>/dev/null \
-		  | grep -m 1 RX | sed -e 's/RX\:\s*//')
+	RXQ_MAX=$2
 
-	echo "...distribute $dev queues"
+	echo "...set rx channels to $RXQ_MAX, please wait..."
+	# Set maximum number of available channels for better
+	# packets hashing.
+	ethtool -L $dev rx $RXQ_MAX >/dev/null 2>&1
+	# Wait for the interface reconfiguration.
+	opstate="$TFW_NETDEV_PATH/$dev/operstate"
+	while [ "$(cat $opstate)" = "down" ]; do
+		sleep 1
+	done
+
+	dev_irqs_path="/sys/class/net/$dev/device/msi_irqs"
+	irqs=($(grep $dev /proc/interrupts | sed -e 's/\s*\|:.*//g'))
+	if [ -z "$irqs" -a -d $dev_irqs_path ]; then
+		irqs=($(ls $dev_irqs_path))
+	fi
 
-	if [ -n "$RXQ_MAX" -a ${RXQ_MAX:-0} -gt 0 ]; then
-		echo "...set rx channels to $RXQ_MAX, please wait..."
-		# Set maximum number of available channels for better
-		# packets hashing.
-		ethtool -L $dev rx $RXQ_MAX >/dev/null 2>&1
-		# Wait for the interface reconfiguration.
-		opstate="$TFW_NETDEV_PATH/$dev/operstate"
-		while [ "$(cat $opstate)" = "down" ]; do
-			sleep 1
-		done
-	else
-		echo "...0 channels for $dev - skip"
+	if [ -z "$irqs" ]; then
+		echo "Error: cannot find interrupts for $dev"
 		return
 	fi
 
-	irqs=($(grep $dev /proc/interrupts | sed -e 's/\s*\|:.*//g'))
+	# Skip the first IRQ since this is general async interrupt
+	# for device (not assigned to any of the queues).
+	irqs=(${irqs[@]:1})
 	irq0=${irqs[0]}
 	for i in ${irqs[@]}; do
 		# Wrap around CPU mask if number of queues is
@@ -76,9 +81,8 @@ distribute_queues()
 	done
 }
 
-# Enable RPS for specified, or all by default, networking interfaces.
-# This is required for loopback interface for proper local delivery,
-# but physical interfaces can have RSS.
+# Enable RSS for networking interfaces. Enable RPS for those devices which
+# doesn't have enough hardware queues.
 tfw_set_net_queues()
 {
 	devs=$1
@@ -86,20 +90,21 @@ tfw_set_net_queues()
 	cpu_mask=$(perl -le 'printf("%x", (1 << '$CPUS_N') - 1)')
 
 	for dev in $devs; do
-		queues=$(ls -d /sys/class/net/$dev/queues/rx-* | wc -l)
-		if [ $queues -le $min_queues ]; then
-			echo "...enable RPS on $dev"
-			for rx in $TFW_NETDEV_PATH/$dev/queues/rx-*; do
-				echo $cpu_mask > $rx/rps_cpus
-			done
-		else
-
+	        queues=$(ethtool -l $dev 2>/dev/null \
+			 | grep -m 1 RX | sed -e 's/RX\:\s*//')
+		if [ -n "$queues" -a ${queues:-0} -gt $min_queues ]; then
 			# Switch off RPS for multi-queued interfaces.
 			for rx in $TFW_NETDEV_PATH/$dev/queues/rx-*; do
 				echo 0 > $rx/rps_cpus
 			done
 
-			distribute_queues $dev
+			echo "...distribute $dev queues"
+			distribute_queues $dev $queues
+		else
+			echo "...enable RPS on $dev"
+			for rx in $TFW_NETDEV_PATH/$dev/queues/rx-*; do
+				echo $cpu_mask > $rx/rps_cpus
+			done
 		fi
 	done
 }

From 1881ad19f6ae08ce0519939c19dabe100971f511 Mon Sep 17 00:00:00 2001
From: Alexander Ostapenko <ao@tempesta-tech.com>
Date: Wed, 14 Nov 2018 18:25:05 +0700
Subject: [PATCH 2/5] Changes according review comments (#1073).

---
 scripts/tempesta.sh |  3 +++
 scripts/tfw_lib.sh  | 16 ++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/scripts/tempesta.sh b/scripts/tempesta.sh
index 9ddddd58d1..1e0ed92058 100755
--- a/scripts/tempesta.sh
+++ b/scripts/tempesta.sh
@@ -43,6 +43,9 @@ tdb_mod=tempesta_db
 tfw_mod=tempesta_fw
 declare -r LONG_OPTS="help,load,unload,start,stop,restart,reload"
 
+# Exclude loopback interface since it needn't any tuning here: it hasn't RSS
+# while RPS just add unnecessary overhead for it (traffic redistribution, IPIs
+# introduction etc.).
 declare devs=$(ip addr show up | grep -P '^[0-9]+' | grep -Pv '\bLOOPBACK\b' \
 	       | awk '{ sub(/:/, "", $2); print $2}')
 
diff --git a/scripts/tfw_lib.sh b/scripts/tfw_lib.sh
index 271c0e1485..47ad9c69b6 100644
--- a/scripts/tfw_lib.sh
+++ b/scripts/tfw_lib.sh
@@ -41,13 +41,22 @@ distribute_queues()
 	echo "...set rx channels to $RXQ_MAX, please wait..."
 	# Set maximum number of available channels for better
 	# packets hashing.
-	ethtool -L $dev rx $RXQ_MAX >/dev/null 2>&1
+	res=$(ethtool -L $dev rx $RXQ_MAX 2>&1)
+	if [ $? -ne 0 -a -z "$(echo $res | grep -P '^rx unmodified, ignoring')" ]
+	then
+	    printf "Error: cannot set new queues configuration for %s\n: %s"
+	    return
+	fi
+
 	# Wait for the interface reconfiguration.
 	opstate="$TFW_NETDEV_PATH/$dev/operstate"
 	while [ "$(cat $opstate)" = "down" ]; do
 		sleep 1
 	done
 
+	# Interrupts may not have interface-like description in
+	# '/proc/interrupts' - so, to find the vectors we also need
+	# to check the MSI directory for device.
 	dev_irqs_path="/sys/class/net/$dev/device/msi_irqs"
 	irqs=($(grep $dev /proc/interrupts | sed -e 's/\s*\|:.*//g'))
 	if [ -z "$irqs" -a -d $dev_irqs_path ]; then
@@ -90,8 +99,8 @@ tfw_set_net_queues()
 	cpu_mask=$(perl -le 'printf("%x", (1 << '$CPUS_N') - 1)')
 
 	for dev in $devs; do
-	        queues=$(ethtool -l $dev 2>/dev/null \
-			 | grep -m 1 RX | sed -e 's/RX\:\s*//')
+		queues=$(ethtool -l $dev 2>/dev/null \
+				| grep -m 1 RX | sed -e 's/RX\:\s*//')
 		if [ -n "$queues" -a ${queues:-0} -gt $min_queues ]; then
 			# Switch off RPS for multi-queued interfaces.
 			for rx in $TFW_NETDEV_PATH/$dev/queues/rx-*; do
@@ -108,4 +117,3 @@ tfw_set_net_queues()
 		fi
 	done
 }
-

From 939280e1a081bfcb2aadb8058cb067a64f69d976 Mon Sep 17 00:00:00 2001
From: Alexander Ostapenko <ao@tempesta-tech.com>
Date: Wed, 14 Nov 2018 19:20:33 +0700
Subject: [PATCH 3/5] Minor output correction (#1073).

---
 scripts/tfw_lib.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/tfw_lib.sh b/scripts/tfw_lib.sh
index 47ad9c69b6..512f09082a 100644
--- a/scripts/tfw_lib.sh
+++ b/scripts/tfw_lib.sh
@@ -44,8 +44,9 @@ distribute_queues()
 	res=$(ethtool -L $dev rx $RXQ_MAX 2>&1)
 	if [ $? -ne 0 -a -z "$(echo $res | grep -P '^rx unmodified, ignoring')" ]
 	then
-	    printf "Error: cannot set new queues configuration for %s\n: %s"
-	    return
+		printf "Error: cannot set new queues count for %s:\n %s\n" \
+			$dev "$res"
+		return
 	fi
 
 	# Wait for the interface reconfiguration.

From ca9466e0e2bab7575260e3691820885aa9438caa Mon Sep 17 00:00:00 2001
From: Alexander Ostapenko <ao@tempesta-tech.com>
Date: Thu, 15 Nov 2018 19:58:08 +0700
Subject: [PATCH 4/5] Ban assigned IRQs for irqbalance (#1073).

---
 scripts/tempesta.sh |  9 ++++++++
 scripts/tfw_lib.sh  | 51 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/scripts/tempesta.sh b/scripts/tempesta.sh
index 1e0ed92058..025da15e64 100755
--- a/scripts/tempesta.sh
+++ b/scripts/tempesta.sh
@@ -208,6 +208,15 @@ stop()
 	echo "...unload Tempesta modules"
 	unload_modules
 
+	systemctl status irqbalance.service >/dev/null
+	if [ $? -eq 0 -a -f $IRQB_CONF_PATH ]; then
+		echo "...revert irqbalance config"
+		perl -i.orig -ple '
+			s/^('"$BAN_CONF_VAR"'=).*$/$1/;
+		' $IRQB_CONF_PATH
+		systemctl restart irqbalance.service >/dev/null
+	fi
+
 	echo "done"
 }
 
diff --git a/scripts/tfw_lib.sh b/scripts/tfw_lib.sh
index 512f09082a..d8c16bac99 100644
--- a/scripts/tfw_lib.sh
+++ b/scripts/tfw_lib.sh
@@ -27,12 +27,55 @@ declare -r TFW_NAME=`basename $0` # program name (comm name in ps)
 declare -r TFW_NETDEV_PATH="/sys/class/net/"
 declare -r TFW_SCRIPTS="$TFW_ROOT/scripts"
 declare -r CPUS_N=$(grep -c processor /proc/cpuinfo)
+declare -r IRQB_CONF_PATH="/etc/default/irqbalance"
+declare -r SYSD_IRQB_PATH="/lib/systemd/system/irqbalance.service"
+declare -r BAN_CONF_VAR="BAN_IRQS"
+declare -a IRQS_GLOB_LIST
 
 calc()
 {
 	echo "$1" | bc -iq | tail -1
 }
 
+# Assigned IRQs can be reassigned by irqbalance. To avoid this, assigned vectors
+# should be added to irqbalance config with '--banirq' option.
+irqbalance_ban_irqs()
+{
+	args_str=""
+	sysd_conf_var="ExecStart"
+
+	echo "...ban IRQs for irqbalance..."
+	for irq in ${IRQS_GLOB_LIST[@]}; do
+		args_str=$args_str"--banirq=$irq "
+	done
+
+	conf_var_str="$BAN_CONF_VAR=$args_str"
+	perl -i.orig -ple '
+		if ($found = /^'"$BAN_CONF_VAR"'=.*$/) {
+			$_ = "'"$conf_var_str"'";
+		}
+		END {
+			unless ($found) {
+				open(CONFIG, ">>", $ARGV);
+				select(CONFIG);
+				printf("\n%s\n", "'"$conf_var_str"'");
+				select(STDOUT);
+			}
+		}
+	' $IRQB_CONF_PATH
+
+	perl -i.orig -ple '
+		if (/^('"$sysd_conf_var"'=.*)$/) {
+			unless (/.*\$'"$BAN_CONF_VAR"'\b.*/) {
+				$_ = $1 . " \$'"$BAN_CONF_VAR"'";
+			}
+		}
+	' $SYSD_IRQB_PATH
+
+	systemctl daemon-reload >/dev/null
+	systemctl restart irqbalance.service >/dev/null
+}
+
 distribute_queues()
 {
 	dev=$1
@@ -89,6 +132,8 @@ distribute_queues()
 			}
 		' > /proc/irq/$i/smp_affinity
 	done
+
+	IRQS_GLOB_LIST+=(${irqs[@]})
 }
 
 # Enable RSS for networking interfaces. Enable RPS for those devices which
@@ -117,4 +162,10 @@ tfw_set_net_queues()
 			done
 		fi
 	done
+
+	if [ ${#IRQS_GLOB_LIST[@]} -ne 0 -a -f $SYSD_IRQB_PATH \
+	     -a -f $IRQB_CONF_PATH ]; then
+		systemctl status irqbalance.service >/dev/null
+		[ $? -ne 0 ] || irqbalance_ban_irqs
+	fi
 }

From 7ea10c56e5a982e5ef9b302cd01e6b7c097a383b Mon Sep 17 00:00:00 2001
From: Alexander Ostapenko <ao@tempesta-tech.com>
Date: Wed, 21 Nov 2018 20:09:19 +0700
Subject: [PATCH 5/5] Changes according review comments and minor corrections
 (#1073).

---
 scripts/tempesta.sh |  9 +--------
 scripts/tfw_lib.sh  | 28 +++++++++++++++++++++++-----
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/scripts/tempesta.sh b/scripts/tempesta.sh
index 025da15e64..eeffcf962f 100755
--- a/scripts/tempesta.sh
+++ b/scripts/tempesta.sh
@@ -208,14 +208,7 @@ stop()
 	echo "...unload Tempesta modules"
 	unload_modules
 
-	systemctl status irqbalance.service >/dev/null
-	if [ $? -eq 0 -a -f $IRQB_CONF_PATH ]; then
-		echo "...revert irqbalance config"
-		perl -i.orig -ple '
-			s/^('"$BAN_CONF_VAR"'=).*$/$1/;
-		' $IRQB_CONF_PATH
-		systemctl restart irqbalance.service >/dev/null
-	fi
+	tfw_irqbalance_revert
 
 	echo "done"
 }
diff --git a/scripts/tfw_lib.sh b/scripts/tfw_lib.sh
index d8c16bac99..9da003f363 100644
--- a/scripts/tfw_lib.sh
+++ b/scripts/tfw_lib.sh
@@ -29,7 +29,7 @@ declare -r TFW_SCRIPTS="$TFW_ROOT/scripts"
 declare -r CPUS_N=$(grep -c processor /proc/cpuinfo)
 declare -r IRQB_CONF_PATH="/etc/default/irqbalance"
 declare -r SYSD_IRQB_PATH="/lib/systemd/system/irqbalance.service"
-declare -r BAN_CONF_VAR="BAN_IRQS"
+declare -r BAN_CONF_VAR="TFW_BAN_IRQS"
 declare -a IRQS_GLOB_LIST
 
 calc()
@@ -37,8 +37,8 @@ calc()
 	echo "$1" | bc -iq | tail -1
 }
 
-# Assigned IRQs can be reassigned by irqbalance. To avoid this, assigned vectors
-# should be added to irqbalance config with '--banirq' option.
+# Assigned IRQs can be reassigned by irqbalance. To avoid this, assigned
+# vectors should be added to irqbalance config with '--banirq' option.
 irqbalance_ban_irqs()
 {
 	args_str=""
@@ -51,14 +51,20 @@ irqbalance_ban_irqs()
 
 	conf_var_str="$BAN_CONF_VAR=$args_str"
 	perl -i.orig -ple '
-		if ($found = /^'"$BAN_CONF_VAR"'=.*$/) {
+		if (/^'"$BAN_CONF_VAR"'=.*$/) {
 			$_ = "'"$conf_var_str"'";
+			$found = 1;
 		}
 		END {
 			unless ($found) {
 				open(CONFIG, ">>", $ARGV);
 				select(CONFIG);
-				printf("\n%s\n", "'"$conf_var_str"'");
+				print("\n# Do not edit this variable. It is"
+				    . " automatically generated by start\n#"
+				    . " script of Tempesta FW and is intended"
+				    . " to avoid reassigning of specified"
+				    . " IRQs.");
+				printf("%s\n", "'"$conf_var_str"'");
 				select(STDOUT);
 			}
 		}
@@ -169,3 +175,15 @@ tfw_set_net_queues()
 		[ $? -ne 0 ] || irqbalance_ban_irqs
 	fi
 }
+
+tfw_irqbalance_revert()
+{
+	systemctl status irqbalance.service >/dev/null
+	if [ $? -eq 0 -a -f $IRQB_CONF_PATH ]; then
+		echo "...revert irqbalance config"
+		perl -i.orig -ple '
+			s/^('"$BAN_CONF_VAR"'=).*$/$1/;
+		' $IRQB_CONF_PATH
+		systemctl restart irqbalance.service >/dev/null
+	fi
+}