From 3b7e18342bad03dffca5c85aca59c5bfd9852481 Mon Sep 17 00:00:00 2001 From: Maksym Belei Date: Fri, 26 Mar 2021 15:40:39 +0200 Subject: [PATCH 1/2] [warm-reboot] Fix failures of warm reboot on disconnect of ssh session * Starting the script in background mode and detaching the process from terminal session to prevent failures, caused by closing or sudden disconnecting of terminal session. * Redirecting output of the script to the file to prevent failures, when the script tries to write an output to file descriptor of the nonexistant terminal session. * Adding new parameter to script to be able to explicitly run the script in foreground mode with output to the terminal. * Updating the command reference doc according to the changes. Signed-off-by: Maksym Belei --- doc/Command-Reference.md | 17 +++++++++++++++-- scripts/fast-reboot | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 15efe35ca6..f850674da8 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8925,9 +8925,15 @@ command will setup everything needed to perform warm reboot. This command requires root privilege. +This command works in background mode with detaching from its terminal session +to prevent errors, cased by closing/disconnecting of the terminal session. +All the output of the command is redirected to the file: /var/log/warm-reboot.txt. +To run the command in foreground mode with output to the terminal, run the command +with parameter '-d'. + - Usage: ``` - warm-reboot [-h|-?|-v|-f|-r|-k|-x|-c |-s] + warm-reboot [-h|-?|-v|-f|-i|-r|-k|-x|-c |-s|-t|-d] ``` - Parameters: @@ -8935,17 +8941,24 @@ This command requires root privilege. -h,-? : get this help -v : turn on verbose mode -f : force execution + -i : ignore MD5-checksum-verification of ASIC configuration files -r : reboot with /sbin/reboot -k : reboot with /sbin/kexec -e [default] -x : execute script with -x flag -c : specify control plane assistant IP list -s : strict mode: do not proceed without: - control plane assistant IP list. + -t : Don't tag the current kube images as latest + -d : Don't detach the process from the terminal session ``` - Example: ``` - admin@sonic:~$ sudo warm-reboot -v + admin@sonic:~$ sudo warm-reboot + Detaching the process from the terminal session. Redirecting output to /var/log/warm-reboot.txt. + ``` + ``` + admin@sonic:~$ sudo warm-reboot -v -d Tue Oct 22 23:20:53 UTC 2019 Pausing orchagent ... Tue Oct 22 23:20:53 UTC 2019 Stopping radv ... Tue Oct 22 23:20:54 UTC 2019 Stopping bgp ... diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 557a6d0f68..ea2d295c9e 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -24,6 +24,8 @@ PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin" LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health" SSD_FW_UPDATE="ssd-fw-upgrade" TAG_LATEST=yes +DETACH=yes +LOG_PATH="/var/log/${REBOOT_TYPE}.txt" # Require 100M available on the hard drive for warm reboot temp files, # Size is in 1K blocks: @@ -71,13 +73,14 @@ function showHelpAndExit() echo " -s : strict mode: do not proceed without:" echo " - control plane assistant IP list." echo " -t : Don't tag the current kube images as latest" + echo " -d : Don't detach the process from the terminal session" exit "${EXIT_SUCCESS}" } function parseOptions() { - while getopts "vfidh?rkxc:s" opt; do + while getopts "vfih?rkxc:sd" opt; do case ${opt} in h|\? ) showHelpAndExit @@ -112,6 +115,9 @@ function parseOptions() t ) TAG_LATEST=no ;; + d ) + DETACH=no + ;; esac done } @@ -457,6 +463,13 @@ then exit "${EXIT_FAILURE}" fi +# re-run the script in background mode with detaching from the terminal session +if [[ x"${DETACH}" == x"yes" ]]; then + echo "Detaching the process from the terminal session. Redirecting output to ${LOG_PATH}." + $0 "$@" -d &>$LOG_PATH & disown %% + exit $? +fi + sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type) # Check reboot type supported From d898699832b61c8b733760b50206ca9bc8813057 Mon Sep 17 00:00:00 2001 From: Marat Gubaiev Date: Tue, 26 Oct 2021 17:39:02 +0300 Subject: [PATCH 2/2] -D : detached mode - closing terminal will not cause stopping reboot --- doc/Command-Reference.md | 18 +++--------------- scripts/fast-reboot | 33 ++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index f850674da8..cf47643625 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8925,15 +8925,9 @@ command will setup everything needed to perform warm reboot. This command requires root privilege. -This command works in background mode with detaching from its terminal session -to prevent errors, cased by closing/disconnecting of the terminal session. -All the output of the command is redirected to the file: /var/log/warm-reboot.txt. -To run the command in foreground mode with output to the terminal, run the command -with parameter '-d'. - - Usage: ``` - warm-reboot [-h|-?|-v|-f|-i|-r|-k|-x|-c |-s|-t|-d] + warm-reboot [-h|-?|-v|-f|-r|-k|-x|-c |-s|-D] ``` - Parameters: @@ -8941,24 +8935,18 @@ with parameter '-d'. -h,-? : get this help -v : turn on verbose mode -f : force execution - -i : ignore MD5-checksum-verification of ASIC configuration files -r : reboot with /sbin/reboot -k : reboot with /sbin/kexec -e [default] -x : execute script with -x flag -c : specify control plane assistant IP list -s : strict mode: do not proceed without: - control plane assistant IP list. - -t : Don't tag the current kube images as latest - -d : Don't detach the process from the terminal session + -D : detached mode - closing terminal will not cause stopping reboot ``` - Example: ``` - admin@sonic:~$ sudo warm-reboot - Detaching the process from the terminal session. Redirecting output to /var/log/warm-reboot.txt. - ``` - ``` - admin@sonic:~$ sudo warm-reboot -v -d + admin@sonic:~$ sudo warm-reboot -v Tue Oct 22 23:20:53 UTC 2019 Pausing orchagent ... Tue Oct 22 23:20:53 UTC 2019 Stopping radv ... Tue Oct 22 23:20:54 UTC 2019 Stopping bgp ... diff --git a/scripts/fast-reboot b/scripts/fast-reboot index ea2d295c9e..9cd1a2a38d 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -24,7 +24,7 @@ PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin" LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health" SSD_FW_UPDATE="ssd-fw-upgrade" TAG_LATEST=yes -DETACH=yes +DETACH=no LOG_PATH="/var/log/${REBOOT_TYPE}.txt" # Require 100M available on the hard drive for warm reboot temp files, @@ -73,14 +73,14 @@ function showHelpAndExit() echo " -s : strict mode: do not proceed without:" echo " - control plane assistant IP list." echo " -t : Don't tag the current kube images as latest" - echo " -d : Don't detach the process from the terminal session" + echo " -D : detached mode - closing terminal will not cause stopping reboot" exit "${EXIT_SUCCESS}" } function parseOptions() { - while getopts "vfih?rkxc:sd" opt; do + while getopts "vfidh?rkxc:sD" opt; do #TODO "t" is missing case ${opt} in h|\? ) showHelpAndExit @@ -115,8 +115,8 @@ function parseOptions() t ) TAG_LATEST=no ;; - d ) - DETACH=no + D ) + DETACH=yes ;; esac done @@ -283,8 +283,13 @@ function backup_database() function setup_control_plane_assistant() { if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then - debug "Setting up control plane assistant: ${ASSISTANT_IP_LIST} ..." - ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m set + # TH3 HW is not capable of VxLAN programming thus skipping TH3 platforms + if [[ "${HWSKU}" != "DellEMC-Z9332f-M-O16C64" && "${HWSKU}" != "DellEMC-Z9332f-M-O16C64-lab" ]]; then + debug "Setting up control plane assistant: ${ASSISTANT_IP_LIST} ..." + ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m set + else + debug "${HWSKU} Not capable to support CPA. Skipping gracefully ..." + fi elif [[ X"${STRICT}" == X"yes" ]]; then debug "Strict mode: fail due to lack of control plane assistant ..." exit ${EXIT_NO_CONTROL_PLANE_ASSISTANT} @@ -294,8 +299,11 @@ function setup_control_plane_assistant() function teardown_control_plane_assistant() { if [[ -n "${ASSISTANT_IP_LIST}" && -x ${ASSISTANT_SCRIPT} ]]; then - debug "Tearing down control plane assistant: ${ASSISTANT_IP_LIST} ..." - ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m reset + # TH3 HW is not capable of VxLAN programming thus skipping TH3 platforms + if [[ "${HWSKU}" != "DellEMC-Z9332f-M-O16C64" && "${HWSKU}" != "DellEMC-Z9332f-M-O16C64-lab" ]]; then + debug "Tearing down control plane assistant: ${ASSISTANT_IP_LIST} ..." + ${ASSISTANT_SCRIPT} -s ${ASSISTANT_IP_LIST} -m reset + fi fi } @@ -306,6 +314,7 @@ function is_secureboot() { function setup_reboot_variables() { # Kernel and initrd image + HWSKU=$(show platform summary --json | python -c 'import sys, json; print(json.load(sys.stdin)["hwsku"])') NEXT_SONIC_IMAGE=$(sonic-installer list | grep "Next: " | cut -d ' ' -f 2) IMAGE_PATH="/host/image-${NEXT_SONIC_IMAGE#SONiC-OS-}" if grep -q aboot_platform= /host/machine.conf; then @@ -463,13 +472,15 @@ then exit "${EXIT_FAILURE}" fi + # re-run the script in background mode with detaching from the terminal session -if [[ x"${DETACH}" == x"yes" ]]; then +if [[ x"${DETACH}" == x"yes" && x"${ALREADY_DETACHED}" == x"" ]]; then echo "Detaching the process from the terminal session. Redirecting output to ${LOG_PATH}." - $0 "$@" -d &>$LOG_PATH & disown %% + ALREADY_DETACHED=yes $0 "$@" &>$LOG_PATH & disown %% exit $? fi + sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type) # Check reboot type supported