Skip to content

Commit fb14b98

Browse files
lixiaoyunermssonicbld
authored andcommitted
Add health check probe for k8s upgrade containers. (#15223)
#### Why I did it After k8s upgrade a container, k8s can only know the container is running, don't know the service's status inside container. So we need a probe inside container, k8s will call the probe to check whether the container is really ready. ##### Work item tracking - Microsoft ADO **(number only)**: 22453004 #### How I did it Add a health check probe inside config engine container, the probe will check whether the start service exit normally or not if the start service exists and call the python script to do container self-related specific checks if the script is there. The python script should be implemented by feature owner if it's needed. more details: [design doc](https://github.com/sonic-net/SONiC/blob/master/doc/kubernetes/health-check.md) #### How to verify it Check path /usr/bin/readiness_probe.sh inside container. #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 - [x] 202205 - [x] 202211 #### Tested branch (Please provide the tested image version) - [x] 20220531.28
1 parent 9cd3319 commit fb14b98

File tree

6 files changed

+43
-0
lines changed

6 files changed

+43
-0
lines changed

dockers/docker-config-engine-bullseye/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ RUN pip3 uninstall -y enum34
4141

4242
# Copy files
4343
COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
44+
COPY ["files/readiness_probe.sh", "/usr/bin/"]
4445
COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
4546

4647
## Clean up

dockers/docker-config-engine-buster/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ RUN pip3 uninstall -y enum34
4141

4242
# Copy files
4343
COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
44+
COPY ["files/readiness_probe.sh", "/usr/bin/"]
4445
COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
4546

4647
## Clean up

rules/docker-config-engine-bullseye.mk

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
1717
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_LOAD_DOCKERS += $(DOCKER_BASE_BULLSEYE)
1818
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(SWSS_VARS_TEMPLATE)
1919
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
20+
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
2021
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
2122

2223
$(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS = $($(DOCKER_BASE_BULLSEYE)_DBG_DEPENDS) \

rules/docker-config-engine-buster.mk

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
1717
$(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
1818
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)
1919
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
20+
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
2021
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
2122

2223
$(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) \

rules/sonic-ctrmgrd.mk

+4
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,16 @@ $($(SONIC_CTRMGRD)_CFG_JSON)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
2020
$(SONIC_CTRMGRD)_SERVICE = ctrmgrd.service
2121
$($(SONIC_CTRMGRD)_SERVICE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
2222

23+
$(SONIC_CTRMGRD)_HEALTH_PROBE = readiness_probe.sh
24+
$($(SONIC_CTRMGRD)_HEALTH_PROBE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
25+
2326
SONIC_PYTHON_WHEELS += $(SONIC_CTRMGRD)
2427

2528
$(SONIC_CTRMGRD)_FILES = $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
2629
$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
2730
$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_CFG_JSON)
2831
$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_SERVICE)
32+
$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
2933

3034
SONIC_COPY_FILES += $($(SONIC_CTRMGRD)_FILES)
3135

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
# This script is used by k8s to check the readiness of containers
3+
# Check if the container is readiness or not, exit code 0 means readiness, others mean not readiness
4+
5+
#### exit code contract, k8s only cares zero or not none-zero, but we want to use none-zero code to indicate different error
6+
# 0: readiness
7+
# 1: if the hook script is python code, the default crash exit code is 1
8+
# 2: supervisor start service doesn't exit normally
9+
# other exit code: returned by post_check_script, define in the post_check_script, should not include 1,2
10+
11+
# check if the start service exists
12+
# if the start service doesn't exist, do nothing
13+
# if the start service exists, check if it exits normally
14+
# if the start service doesn't exit normally, exit with code 2
15+
pre_check_service_name="start"
16+
no_process_string="ERROR (no such process)"
17+
service_status=$(supervisorctl status $pre_check_service_name)
18+
if [[ $service_status != *"$no_process_string"* ]] && [[ $(echo $service_status |awk '{print $2}') != 'EXITED' ]]; then
19+
exit 2
20+
fi
21+
22+
# feature owner can add their own readiness check script
23+
# check if the post_check_script exists
24+
# if the post_check_script exists, run it
25+
# if the post_check_script exits with non-zero code, exit with the code
26+
post_check_script="/usr/bin/readiness_probe_hook"
27+
if [ -x $post_check_script ]; then
28+
$post_check_script
29+
post_check_result=$?
30+
if [ $post_check_result != 0 ]; then
31+
exit $post_check_result
32+
fi
33+
fi
34+
35+
exit 0

0 commit comments

Comments
 (0)