Skip to content

Commit

Permalink
[warm-reboot]: added automated recover for ISSU file (sonic-net#1466)
Browse files Browse the repository at this point in the history
#### What I did
Modified pre-check for file `/host/warmboot/issu_bank.txt` in `warm-reboot` script.
Added automated recovery of  `/host/warmboot/issu_bank.txt`

#### How I did it
Modified the `warm-reboot` script.

#### How to verify it
Need to somehow corrupt the `/host/warmboot/issu_bank.txt` (list below) and then run the `warm-reboot` command.
For testing I have modified the `warm-reboot` script - added corruption command before `check_issu_bank_file` function call.

1. Remove `issu_bank.txt`
2. Clear `issu_bank.txt`
3. Change characters count in `issu_bank.txt`
4. Write invalid content into `issu_bank.txt`

#### Previous command output (if the output of a command-line utility has changed)
```
root@arc-switch1041:~# warm-reboot
(/host/warmboot/issu_bank.txt) does NOT exist or empty ...
To recover (/host/warmboot/issu_bank.txt) file, do the following:
$ docker exec -it syncd sx_api_dbg_generate_dump.py
$ docker exec -it syncd cat /tmp/sdkdump | grep 'ISSU Bank'
Command above will print the VALUE of ISSU BANK - 0 or 1, use this VALUE in the next command
$ printf VALUE > /host/warmboot/issu_bank.txt
```

#### New command output (if the output of a command-line utility has changed)
```
root@arc-switch1041:~# warm-reboot
(/host/warmboot/issu_bank.txt) does NOT exist or empty ...
Recovering the (/host/warmboot/issu_bank.txt) file
```
  • Loading branch information
vadymhlushko-mlnx committed Mar 11, 2021
1 parent a4ae643 commit e1c52a0
Showing 1 changed file with 8 additions and 18 deletions.
26 changes: 8 additions & 18 deletions scripts/fast-reboot
Original file line number Diff line number Diff line change
Expand Up @@ -170,26 +170,21 @@ function request_pre_shutdown()
}
}
function recover_issu_bank_file_instruction()
function recover_issu_bank_file()
{
debug "To recover (${ISSU_BANK_FILE}) file, do the following:"
debug "$ docker exec -it syncd sx_api_dbg_generate_dump.py"
debug "$ docker exec -it syncd cat /tmp/sdkdump | grep 'ISSU Bank'"
debug "Command above will print the VALUE of ISSU BANK - 0 or 1, use this VALUE in the next command"
debug "$ printf VALUE > /host/warmboot/issu_bank.txt"
debug "Recovering the (${ISSU_BANK_FILE}) file"
docker exec -i syncd sx_api_dbg_generate_dump.py
issu_bank_value=`docker exec -i syncd cat /tmp/sdkdump | grep 'ISSU Bank' | grep -o -E '[0-9]+'`
printf $issu_bank_value > /host/warmboot/issu_bank.txt
}
function check_issu_bank_file()
{
ISSU_BANK_FILE=/host/warmboot/issu_bank.txt
MLNX_ISSU_BANK_BROKEN=102
if [[ ! -s "$ISSU_BANK_FILE" ]]; then
error "(${ISSU_BANK_FILE}) does NOT exist or empty ..."
recover_issu_bank_file_instruction
if [[ "$1" = true ]]; then
exit "${MLNX_ISSU_BANK_BROKEN}"
fi
recover_issu_bank_file
return
fi
Expand All @@ -199,10 +194,7 @@ function check_issu_bank_file()
if [[ $issu_file_chars_count != 1 ]] ||
[[ "$issu_file_content" != "0" && "$issu_file_content" != "1" ]]; then
error "(${ISSU_BANK_FILE}) is broken ..."
recover_issu_bank_file_instruction
if [[ "$1" = true ]]; then
exit "${MLNX_ISSU_BANK_BROKEN}"
fi
recover_issu_bank_file
fi
}
Expand Down Expand Up @@ -599,10 +591,8 @@ if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; t
# Pre-shutdown syncd
initialize_pre_shutdown
BEFORE_PRE_SHUTDOWN=true
if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
check_issu_bank_file "$BEFORE_PRE_SHUTDOWN"
check_issu_bank_file
fi
request_pre_shutdown
Expand Down

0 comments on commit e1c52a0

Please sign in to comment.