From 4302c61645055ded7f851ad4b99bc1a54ab40818 Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Fri, 4 Dec 2020 09:47:37 +0000 Subject: [PATCH 1/7] [MultiDB] add mutidb warmboot support - restoring database --- dockers/docker-database/Dockerfile.j2 | 1 + .../docker-database/docker-database-init.sh | 14 ++++++++++ dockers/docker-database/flush_unused_database | 26 +++++++++++++++++++ dockers/docker-database/supervisord.conf.j2 | 8 ++++++ 4 files changed, 49 insertions(+) create mode 100755 dockers/docker-database/flush_unused_database diff --git a/dockers/docker-database/Dockerfile.j2 b/dockers/docker-database/Dockerfile.j2 index 84f485e4fe83..aef22e24379c 100644 --- a/dockers/docker-database/Dockerfile.j2 +++ b/dockers/docker-database/Dockerfile.j2 @@ -58,5 +58,6 @@ COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] COPY ["files/sysctl-net.conf", "/etc/sysctl.d/"] COPY ["critical_processes", "/etc/supervisor"] COPY ["files/update_chassisdb_config", "/usr/local/bin/"] +COPY ["flush_unused_database", "/usr/local/bin/"] ENTRYPOINT ["/usr/local/bin/docker-database-init.sh"] diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index 30a65af89b7f..8482043f6543 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -79,4 +79,18 @@ else fi rm $db_cfg_file_tmp +# flush unused database after warmboot restoration +DUMPFILE=/var/lib/redis/dump.rdb +redis_inst_list=`/usr/bin/python -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"` +for inst in $redis_inst_list +do + mkdir -p /var/lib/$inst + if [[ -f $DUMPFILE ]]; then + # copy warmboot rdb file into each new instance location + cp $DUMPFILE /var/lib/$inst/dump.rdb + else + echo -n > /var/lib/$inst/dump.rdb + fi +done + exec /usr/local/bin/supervisord diff --git a/dockers/docker-database/flush_unused_database b/dockers/docker-database/flush_unused_database new file mode 100755 index 000000000000..16605e82aba5 --- /dev/null +++ b/dockers/docker-database/flush_unused_database @@ -0,0 +1,26 @@ +#!/usr/bin/python +import swsssdk +import redis +import subprocess +import time + +output = subprocess.check_output('sonic-db-cli PING', stderr=subprocess.STDOUT, shell=True) +while('PONG' not in output): + time.sleep(1) + +instlists = swsssdk.SonicDBConfig.get_instancelist() +for instname, v in instlists.items(): + insthost = v['hostname'] + instsocket = v['unix_socket_path'] + + dblists = swsssdk.SonicDBConfig.get_dblist() + for dbname in dblists: + dbid = swsssdk.SonicDBConfig.get_dbid(dbname) + dbinst = swsssdk.SonicDBConfig.get_instancename(dbname) + + # this DB is on current instance, skip flush + if dbinst == instname: + continue + + r = redis.Redis(host=insthost, unix_socket_path=instsocket, db=dbid) + r.flushdb() diff --git a/dockers/docker-database/supervisord.conf.j2 b/dockers/docker-database/supervisord.conf.j2 index 97b8a4822530..a1d10f78bd1e 100644 --- a/dockers/docker-database/supervisord.conf.j2 +++ b/dockers/docker-database/supervisord.conf.j2 @@ -33,3 +33,11 @@ stdout_logfile=syslog stderr_logfile=syslog {% endfor %} {% endif %} + +[program:flushdb] +command=/bin/bash -c "sleep 300 && /usr/local/bin/flush_unused_database" +priority=3 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog From f896470f0cd39ca3b8009d5bf179d3ce919f6cff Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Fri, 4 Dec 2020 21:04:07 +0000 Subject: [PATCH 2/7] [MultiDB] address review --- dockers/docker-database/docker-database-init.sh | 2 +- dockers/docker-database/flush_unused_database | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index 8482043f6543..960c3666e697 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -79,7 +79,7 @@ else fi rm $db_cfg_file_tmp -# flush unused database after warmboot restoration +# copy dump.rdb file to each instance for restoration DUMPFILE=/var/lib/redis/dump.rdb redis_inst_list=`/usr/bin/python -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"` for inst in $redis_inst_list diff --git a/dockers/docker-database/flush_unused_database b/dockers/docker-database/flush_unused_database index 16605e82aba5..bc37cfb26d92 100755 --- a/dockers/docker-database/flush_unused_database +++ b/dockers/docker-database/flush_unused_database @@ -4,8 +4,9 @@ import redis import subprocess import time -output = subprocess.check_output('sonic-db-cli PING', stderr=subprocess.STDOUT, shell=True) +output = "" while('PONG' not in output): + output = subprocess.check_output('sonic-db-cli PING', stderr=subprocess.STDOUT, shell=True) time.sleep(1) instlists = swsssdk.SonicDBConfig.get_instancelist() From 549e6bc68b2f161a7b2213b9d16d2418027398b2 Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Fri, 4 Dec 2020 21:33:53 +0000 Subject: [PATCH 3/7] [MultiDB] address review --- dockers/docker-database/flush_unused_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-database/flush_unused_database b/dockers/docker-database/flush_unused_database index bc37cfb26d92..72ec2d0874ec 100755 --- a/dockers/docker-database/flush_unused_database +++ b/dockers/docker-database/flush_unused_database @@ -6,7 +6,7 @@ import time output = "" while('PONG' not in output): - output = subprocess.check_output('sonic-db-cli PING', stderr=subprocess.STDOUT, shell=True) + output = subprocess.Popen(['sonic-db-cli', 'PING'], stdout=subprocess.PIPE).communicate()[0] time.sleep(1) instlists = swsssdk.SonicDBConfig.get_instancelist() From 1744ecc55e2686da255f0e03513d0ed7fc2332c8 Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Fri, 4 Dec 2020 21:58:02 +0000 Subject: [PATCH 4/7] [MultiDB] address review --- dockers/docker-database/flush_unused_database | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dockers/docker-database/flush_unused_database b/dockers/docker-database/flush_unused_database index 72ec2d0874ec..10cacb5e0b33 100755 --- a/dockers/docker-database/flush_unused_database +++ b/dockers/docker-database/flush_unused_database @@ -4,9 +4,10 @@ import redis import subprocess import time -output = "" -while('PONG' not in output): +while(True): output = subprocess.Popen(['sonic-db-cli', 'PING'], stdout=subprocess.PIPE).communicate()[0] + if 'PONG' in output: + break time.sleep(1) instlists = swsssdk.SonicDBConfig.get_instancelist() From 956577c8a87438a0199d1f96cda6b16066b93f3d Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Sat, 5 Dec 2020 19:54:29 +0000 Subject: [PATCH 5/7] [MultiDB] address review --- dockers/docker-database/docker-database-init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index 960c3666e697..d3ce01e1fc6f 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -87,7 +87,7 @@ do mkdir -p /var/lib/$inst if [[ -f $DUMPFILE ]]; then # copy warmboot rdb file into each new instance location - cp $DUMPFILE /var/lib/$inst/dump.rdb + cp -n $DUMPFILE /var/lib/$inst/dump.rdb else echo -n > /var/lib/$inst/dump.rdb fi From 99c6a3c9ae5fe140b98dc4d9867247d6261d33b2 Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Sun, 6 Dec 2020 05:51:43 +0000 Subject: [PATCH 6/7] [MultiDB] using cp -u option to overwrite older file only --- dockers/docker-database/docker-database-init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index d3ce01e1fc6f..3ff6c068a9e8 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -87,7 +87,7 @@ do mkdir -p /var/lib/$inst if [[ -f $DUMPFILE ]]; then # copy warmboot rdb file into each new instance location - cp -n $DUMPFILE /var/lib/$inst/dump.rdb + cp -u $DUMPFILE /var/lib/$inst/dump.rdb else echo -n > /var/lib/$inst/dump.rdb fi From ec865a1ab0c18a89f52b81630d5f5a0102b96126 Mon Sep 17 00:00:00 2001 From: dzhangalibaba Date: Mon, 7 Dec 2020 06:33:31 +0000 Subject: [PATCH 7/7] [MultiDB] check if dst and src same file before cp --- dockers/docker-database/docker-database-init.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh index 3ff6c068a9e8..63fa43d4751c 100755 --- a/dockers/docker-database/docker-database-init.sh +++ b/dockers/docker-database/docker-database-init.sh @@ -87,7 +87,9 @@ do mkdir -p /var/lib/$inst if [[ -f $DUMPFILE ]]; then # copy warmboot rdb file into each new instance location - cp -u $DUMPFILE /var/lib/$inst/dump.rdb + if [[ "$DUMPFILE" != "/var/lib/$inst/dump.rdb" ]]; then + cp $DUMPFILE /var/lib/$inst/dump.rdb + fi else echo -n > /var/lib/$inst/dump.rdb fi