Multi DB with namespace support, Introducing the database_global.json… (

#4477)

* Multi DB with namespace support, Introducing the database_global.json file
for supporting accessing DB's in other namespaces for service running in
linux host

* Updates based on comments

* Adding the j2 templates for database_config and database_global files.

* Updating to retrieve the redis DIR's to be mounted from database_global.json file.

* Additional check to see if asic.conf file exists before sourcing it.

* Updates based on PR comments discussion.

* Review comments update

* Updates to the argument "-n" for namespace used in both context of parsing minigraph and multi DB access.

* Update with the attribute "persistence_for_warm_boot" that was added to database_config.json file earlier.

* Removing the database_config.json file to avioid confusion in future.
We use the database_config.json.j2 file to generate database_config.json files dynamically.

* Update the comments for sudo usage in docker_image_ctrl.j2

* Update with the new logic in PING PONG tests using sonic-db-cli. With this we wait till the
PONG response is received when redis server is up.

* Similar changes in swss and syncd scripts for the PING tests with sonic-db-cli

* Updated with a missing , in the database_config.json.j2 file, Do pip install of j2cli in docker-base-buster.

dockers/docker-base-stretch/Dockerfile.j2

@@ -44,6 +44,7 @@ RUN apt-get update &&        \
         perl                 \
         procps               \
         python               \
+        python-pip           \
         rsyslog              \
         vim-tiny             \
 # Install dependencies of supervisor
@@ -63,6 +64,9 @@ RUN apt-get -y install       \
         net-tools
 {% endif %}
 
+# For templating
+RUN pip install j2cli
+
 RUN mkdir -p /etc/supervisor /var/log/supervisor
 
 RUN apt-get -y purge   \

dockers/docker-database/Dockerfile.j2

@@ -34,7 +34,8 @@ RUN apt-get clean -y                                  && \
 
 COPY ["supervisord.conf.j2", "/usr/share/sonic/templates/"]
 COPY ["docker-database-init.sh", "/usr/local/bin/"]
-COPY ["database_config.json", "/etc/default/sonic-db/"]
+COPY ["database_config.json.j2", "/usr/share/sonic/templates/"]
+COPY ["database_global.json.j2", "/usr/share/sonic/templates/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
 

dockers/docker-database/database_config.json → dockers/docker-database/database_config.json.j2

@@ -3,7 +3,8 @@
         "redis":{
             "hostname" : "127.0.0.1",
             "port" : 6379,
-            "unix_socket_path" : "/var/run/redis/redis.sock"
+            "unix_socket_path" : "/var/run/redis{{NAMESPACE_ID}}/redis.sock",
+            "persistence_for_warm_boot" : "yes"
         }
     },
     "DATABASES" : {

dockers/docker-database/database_global.json.j2

@@ -0,0 +1,21 @@
+{% set namespace_cnt = NAMESPACE_COUNT|int %}
+{
+    "INCLUDES" : [
+        {
+            "include" : "../../redis/sonic-db/database_config.json"
+        },
+{% if namespace_cnt > 1 %}
+{% for ns in range(namespace_cnt) %}
+        {
+            "namespace" : "{{NAMESPACE_PREFIX}}{{ns}}",
+            "include" : "../../redis{{ns}}/sonic-db/database_config.json"
+{% if ns == namespace_cnt-1  %}
+        }
+{% else %}
+        },
+{% endif %}
+{% endfor %}
+    ],
+    "VERSION" : "1.0"
+}
+{% endif %}

dockers/docker-database/docker-database-init.sh

@@ -1,13 +1,26 @@
 #!/usr/bin/env bash
 
-mkdir -p /var/run/redis/sonic-db
-if [ -f /etc/sonic/database_config.json ]; then
-    cp /etc/sonic/database_config.json /var/run/redis/sonic-db
+REDIS_DIR=/var/run/redis$NAMESPACE_ID
+mkdir -p $REDIS_DIR/sonic-db
+
+if [ -f /etc/sonic/database_config$NAMESPACE_ID.json ]; then
+    cp /etc/sonic/database_config$NAMESPACE_ID.json $REDIS_DIR/sonic-db/database_config.json
 else
-    cp /etc/default/sonic-db/database_config.json /var/run/redis/sonic-db
+    j2 /usr/share/sonic/templates/database_config.json.j2 > $REDIS_DIR/sonic-db/database_config.json
 fi
 
 mkdir -p /etc/supervisor/conf.d/
+
+# copy/generate the database_global.json file if this is global database service in multi asic platform.
+if [[ $NAMESPACE_ID == "" ]] && [[ $NAMESPACE_COUNT -gt 1 ]]
+then
+    if [ -f /etc/sonic/database_global.json ]; then
+        cp /etc/sonic/database_global.json $REDIS_DIR/sonic-db/database_global.json
+    else
+        j2 /usr/share/sonic/templates/database_global.json.j2 > $REDIS_DIR/sonic-db/database_global.json
+    fi
+fi
+
 # generate all redis server supervisord configuration file
 sonic-cfggen -j /var/run/redis/sonic-db/database_config.json -t /usr/share/sonic/templates/supervisord.conf.j2 > /etc/supervisor/conf.d/supervisord.conf
 

files/build_templates/docker_image_ctl.j2

@@ -64,7 +64,7 @@ function preStartAction()
         docker cp /tmp/dump.rdb database$DEV:/var/lib/redis/
     fi
 {%- elif docker_container_name == "snmp" %}
-    sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB HSET 'DEVICE_METADATA|localhost' chassis_serial_number $(decode-syseeprom -s)
+    $SONIC_DB_CLI STATE_DB HSET 'DEVICE_METADATA|localhost' chassis_serial_number $(decode-syseeprom -s)
 {%- else %}
     : # nothing
 {%- endif %}
@@ -77,9 +77,9 @@ function postStartAction()
         docker exec -i database$DEV sysctl -w net.ipv6.conf.all.disable_ipv6=0
         link_namespace $DEV
     fi
+
     # Wait until redis starts
-    # TODO: should use $SONIC_DB_CLI if Judy's PR 4477 is in first, otherwise PR 4477 should change this part
-    until [[ $(/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli PING | grep -c PONG) -gt 0 ]]; do
+    until [[ $($SONIC_DB_CLI PING | grep -c PONG) -gt 0 ]]; do
       sleep 1;
     done
 
@@ -89,23 +89,25 @@ function postStartAction()
         # If there is a config_db.json dump file, load it.
         if [ -r /etc/sonic/config_db$DEV.json ]; then
             if [ -r /etc/sonic/init_cfg.json ]; then
-                sonic-netns-exec "$NET_NS" sonic-cfggen -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db$DEV.json --write-to-db
+                $SONIC_CFGGEN -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db$DEV.json --write-to-db
             else
-                sonic-netns-exec "$NET_NS" sonic-cfggen -j /etc/sonic/config_db$DEV.json --write-to-db
+                $SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db
             fi
         fi
 
         if [[ "$BOOT_TYPE" == "fast" ]]; then
             # set the key to expire in 3 minutes
-            /usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
+            $SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
         fi
 
-        /usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
+        $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
     fi
 
     if [[ -x /usr/bin/db_migrator.py ]]; then
         # Migrate the DB to the latest schema version if needed
-        /usr/bin/db_migrator.py -o migrate
+        if [ -z "$DEV" ]; then
+            /usr/bin/db_migrator.py -o migrate
+        fi
     fi
 {%- elif docker_container_name == "swss" %}
     docker exec swss$DEV rm -f /ready   # remove cruft
@@ -137,14 +139,20 @@ start() {
     BOOT_TYPE=`getBootType`
 
     # Obtain our platform as we will mount directories with these names in each docker
-    PLATFORM=`sonic-netns-exec "$NET_NS" sonic-cfggen -H -v DEVICE_METADATA.localhost.platform`
+    PLATFORM=`$SONIC_CFGGEN -H -v DEVICE_METADATA.localhost.platform`
+
+    # Parse the device specific asic conf file, if it exists
+    ASIC_CONF=/usr/share/sonic/device/$PLATFORM/asic.conf
+    if [ -f "$ASIC_CONF" ]; then
+        source $ASIC_CONF
+    fi
 
     {%- if docker_container_name == "database" %}
     # Don't mount HWSKU in {{docker_container_name}} container.
     HWSKU=""
     {%- else %}
     # Obtain our HWSKU as we will mount directories with these names in each docker
-    HWSKU=`sonic-netns-exec "$NET_NS" sonic-cfggen -d -v 'DEVICE_METADATA["localhost"]["hwsku"]'`
+    HWSKU=`$SONIC_CFGGEN -d -v 'DEVICE_METADATA["localhost"]["hwsku"]'`
     {%- endif %}
 
     DOCKERCHECK=`docker inspect --type container {{docker_container_name}}$DEV 2>/dev/null`
@@ -173,18 +181,64 @@ start() {
 
     {%- if docker_container_name == "database" %}
     echo "Creating new {{docker_container_name}}$DEV container"
+    if [ -z "$DEV" ]; then
+        # if database_global exists in old_config, use it; otherwise use the default one in new image
+        if [ -f /etc/sonic/old_config/database_global.json ]; then
+            echo "Use database_global.json from old system..."
+            mv /etc/sonic/old_config/database_global.json /etc/sonic/
+        fi
+    fi
     # if database_config exists in old_config, use it; otherwise use the default one in new image
-    if [ -f /etc/sonic/old_config/database_config.json ]; then
+    if [ -f /etc/sonic/old_config/database_config$DEV.json ]; then
         echo "Use database_config.json from old system..."
-        mv /etc/sonic/old_config/database_config.json /etc/sonic/
+        mv /etc/sonic/old_config/database_config$DEV.json /etc/sonic/
     fi
     {%- else %}
     echo "Creating new {{docker_container_name}}$DEV container with HWSKU $HWSKU"
     {%- endif %}
 
+    # In Multi ASIC platforms the global database config file database_global.json will exist.
+    # Parse the file and get the include path for the database_config.json files used in
+    # various namesapces. The database_config paths are relative to the DIR of SONIC_DB_GLOBAL_JSON.
+    SONIC_DB_GLOBAL_JSON="/var/run/redis/sonic-db/database_global.json"
+    if [ -f "$SONIC_DB_GLOBAL_JSON" ]; then
+        # TODO Create a separate python script with the below logic and invoke it here.
+        redis_dir_list=`/usr/bin/python -c "import sys; import os; import json; f=open(sys.argv[1]); \
+                        global_db_dir = os.path.dirname(sys.argv[1]); data=json.load(f); \
+                        print(\" \".join([os.path.normpath(global_db_dir+'/'+elem['include']).partition('sonic-db')[0]\
+                        for elem in data['INCLUDES'] if 'namespace' in elem])); f.close()" $SONIC_DB_GLOBAL_JSON`
+    fi
+
     if [ -z "$DEV" ]; then
         NET="host"
+
+        # For Multi-ASIC platform we have to mount the redis paths for database instances running in different
+        # namespaces, into the single instance dockers like snmp, pmon on linux host. These global dockers
+        # will need to get/set tables from databases in different namespaces.
+        # /var/run/redis0 ---> mounted as --> /var/run/redis0
+        # /var/run/redis1 ---> mounted as --> /var/run/redis1 .. etc
+        # The below logic extracts the base DIR's where database_config.json's for various namespaces exist.
+        # redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"
+
+        {%- if docker_container_name != "database" %}
+        if [ -n "$redis_dir_list" ]; then
+            for redis_dir in $redis_dir_list
+            do
+                REDIS_MNT=$REDIS_MNT" -v $redis_dir:$redis_dir:rw "
+            done
+        fi
+        {%- endif %}
     else
+        # This part of code is applicable for Multi-ASIC platforms. Here we mount the namespace specific
+        # redis directory into the docker running in that namespace. Below eg: is for namespace "asic1"
+        # /var/run/redis1 ---> mounted as --> /var/run/redis1
+        # redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"
+        if [ -n "$redis_dir_list" ]; then
+            id=`expr $DEV + 1`
+            redis_dir=`echo $redis_dir_list | cut -d " " -f $id`
+            REDIS_MNT=" -v $redis_dir:$redis_dir:rw "
+        fi
+
         {%- if docker_container_name == "database" %}
         NET="bridge"
         {%- else %}
@@ -232,6 +286,7 @@ start() {
         -v /etc/sonic/frr/$DEV:/etc/frr:rw \
 {%- endif %}
         -v /var/run/redis$DEV:/var/run/redis:rw \
+        $REDIS_MNT \
         -v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \
 {%- if docker_container_name != "database" %}
         -v /usr/share/sonic/device/$PLATFORM/$HWSKU/$DEV:/usr/share/sonic/hwsku:ro \
@@ -240,6 +295,11 @@ start() {
         --tmpfs /tmp \
 {%- endif %}
         --tmpfs /var/tmp \
+{%- if docker_container_name == "database" %}
+        --env "NAMESPACE_ID"="$DEV" \
+        --env "NAMESPACE_PREFIX"="$NAMESPACE_PREFIX" \
+        --env "NAMESPACE_COUNT"=$NUM_ASIC \
+{%- endif %}
         --name={{docker_container_name}}$DEV {{docker_image_name}}:latest || {
             echo "Failed to docker run" >&1
             exit 4
@@ -265,10 +325,21 @@ stop() {
 
 OP=$1
 DEV=$2 # namespace/device number to operate on
+NAMESPACE_PREFIX="asic"
 if [ "$DEV" ]; then
-    NET_NS="asic$DEV" #name of the network namespace
-else
-    NET_NS=""    
+    NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
+
+    # While using -n (namespace) argument, sonic-cfggen/sonic-db-cli uses redis UNIX socket 
+    # for accessing redis DB in a namespace. This unix socket has permission restrictions since 
+    # it is created by systemd database.servce started with [User] as [root].
+    # sudo is needed here for services which are started by systemd with [User] as [admin]
+    # and needs to override this unix socket permission restrictions.
+    SONIC_CFGGEN="sudo sonic-cfggen -n $NET_NS"
+    SONIC_DB_CLI="sudo sonic-db-cli -n $NET_NS"
+ else
+    NET_NS=""
+    SONIC_CFGGEN="sonic-cfggen"
+    SONIC_DB_CLI="sonic-db-cli"
 fi
 
 case "$1" in

files/scripts/swss.sh

@@ -28,8 +28,8 @@ function unlock_service_state_change()
 
 function check_warm_boot()
 {
-    SYSTEM_WARM_START=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
-    SERVICE_WARM_START=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
+    SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
+    SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
     if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then
         WARM_BOOT="true"
     else
@@ -40,7 +40,7 @@ function check_warm_boot()
 function validate_restore_count()
 {
     if [[ x"$WARM_BOOT" == x"true" ]]; then
-        RESTORE_COUNT=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_TABLE|orchagent" restore_count`
+        RESTORE_COUNT=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_TABLE|orchagent" restore_count`
         # We have to make sure db data has not been flushed.
         if [[ -z "$RESTORE_COUNT" ]]; then
             WARM_BOOT="false"
@@ -51,13 +51,12 @@ function validate_restore_count()
 function wait_for_database_service()
 {
     # Wait for redis server start before database clean
-    # TODO: should use $SONIC_DB_CLI if Judy's PR 4477 is in first, otherwise PR 4477 should change this part
-    until [[ $(/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli PING | grep -c PONG) -gt 0 ]]; do
+    until [[ $($SONIC_DB_CLI PING | grep -c PONG) -gt 0 ]]; do
       sleep 1;
     done
 
     # Wait for configDB initialization
-    until [[ $(sonic-netns-exec "$NET_NS" sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
+    until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
         do sleep 1;
     done
 }
@@ -67,7 +66,7 @@ function wait_for_database_service()
 # $2 the string of a list of table prefixes
 function clean_up_tables()
 {
-    sonic-netns-exec "$NET_NS" sonic-db-cli $1 EVAL "
+    $SONIC_DB_CLI $1 EVAL "
     local tables = {$2}
     for i = 1, table.getn(tables) do
         local matches = redis.call('KEYS', tables[i])
@@ -135,10 +134,10 @@ start() {
     # Don't flush DB during warm boot
     if [[ x"$WARM_BOOT" != x"true" ]]; then
         debug "Flushing APP, ASIC, COUNTER, CONFIG, and partial STATE databases ..."
-        sonic-netns-exec "$NET_NS" sonic-db-cli APPL_DB FLUSHDB
-        sonic-netns-exec "$NET_NS" sonic-db-cli ASIC_DB FLUSHDB
-        sonic-netns-exec "$NET_NS" sonic-db-cli COUNTERS_DB FLUSHDB
-        sonic-netns-exec "$NET_NS" sonic-db-cli FLEX_COUNTER_DB FLUSHDB
+        $SONIC_DB_CLI APPL_DB FLUSHDB
+        $SONIC_DB_CLI ASIC_DB FLUSHDB
+        $SONIC_DB_CLI COUNTERS_DB FLUSHDB
+        $SONIC_DB_CLI FLEX_COUNTER_DB FLUSHDB
         clean_up_tables STATE_DB "'PORT_TABLE*', 'MGMT_PORT_TABLE*', 'VLAN_TABLE*', 'VLAN_MEMBER_TABLE*', 'LAG_TABLE*', 'LAG_MEMBER_TABLE*', 'INTERFACE_TABLE*', 'MIRROR_SESSION*', 'VRF_TABLE*', 'FDB_TABLE*'"
     fi
 
@@ -209,10 +208,13 @@ SERVICE="swss"
 PEER="syncd"
 DEBUGLOG="/tmp/swss-syncd-debug$DEV.log"
 LOCKFILE="/tmp/swss-syncd-lock$DEV"
+NAMESPACE_PREFIX="asic"
 if [ "$DEV" ]; then
-    NET_NS="asic$DEV" #name of the network namespace
+    NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
+    SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
 else
-    NET_NS=""   
+    NET_NS=""
+    SONIC_DB_CLI="sonic-db-cli"
 fi
 
 case "$1" in