[202012] Fix CONFIG_DB_INITIALIZED flag check logic and set/reset fla…

…g for warmboot (#16225)

Cherry pick of #15685

MSFT ADO: 24274591

#### Why I did it

Two changes:
### 1  Fix a day1 issue, where check to wait until `CONFIG_DB_INITIALIZED` is incorrect.
There are multiple places where same incorrect logic is used.

Current logic (`until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];`) will always result in pass, irrespective of the result of GET operation.
```
root@str2-7060cx-32s-29:~# sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED"
1
root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; do echo "entered here"; done
root@str2-7060cx-32s-29:~# 

root@str2-7060cx-32s-29:~# 
root@str2-7060cx-32s-29:~# sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED"                                             
0
root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; do echo "entered here"; done
root@str2-7060cx-32s-29:~# 
```

Fix this logic by checking for value of flag to be "1".
```
root@str2-7060cx-32s-29:~# until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do echo "entered here"; done
entered here
entered here
entered here
```

This gap in logic was highlighted when another fix was merged: #14933
The issue being fixed here caused warmboot-finalizer to not wait until config-db is initialized.

### 2 Set and unset CONFIG_DB_INITIALIZED for warm-reboot case

Currently, during warm shutdown `CONFIG_DB_INITIALIZED`'s value is stored in redis db backup. This is restored back when the dump is loaded during warm-recovery.
So the value of `CONFIG_DB_INITIALIZED` does not depend on config db's state, however it remain what it was before reboot.

Fix this by setting `CONFIG_DB_INITIALIZED` to 0 as when the DB is loaded, and set it to 1 after db_migrator is done.

files/build_templates/docker_image_ctl.j2

@@ -126,28 +126,18 @@ function postStartAction()
                     $SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db
                 fi
             fi
-
-            if [[ "$BOOT_TYPE" == "fast" ]]; then
-                # set the key to expire in 3 minutes
-                $SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
-            fi
-
-            $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
         fi
 
-        if [ -e /tmp/pending_config_migration ]; then
+        if [ -e /tmp/pending_config_migration ] || [ -e /tmp/pending_config_initialization ]; then
             # this is first boot to a new image, config-setup execution is pending.
-            # For fast/cold reboot case, DB contains nothing at this point
-            # Call db_migrator after config-setup loads the config (from old config or minigraph)
-            echo "Delaying db_migrator until config migration is over"
+            # for warmboot case, DB is loaded but migration is still pending
+            # For firstbboot/fast/cold reboot case, DB contains nothing at this point
+            # unset CONFIG_DB_INITIALIZED to indicate pending config load and migration
+            # This flag will be set to "1" after DB migration/initialization is completed as part of config-setup
+            $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0"
         else
-            # this is not a first time boot to a new image. Datbase container starts w/ old pre-existing config
-            if [[ -x /usr/local/bin/db_migrator.py ]]; then
-                # Migrate the DB to the latest schema version if needed
-                if [ -z "$DEV" ]; then
-                    /usr/local/bin/db_migrator.py -o migrate
-                fi
-            fi
+            # set CONFIG_DB_INITIALIZED to indicate end of config load and migration
+            $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
         fi
         # Add redis UDS to the redis group and give read/write access to the group
         REDIS_SOCK="/var/run/redis${DEV}/redis.sock"

files/image_config/config-setup/config-setup

@@ -251,6 +251,7 @@ do_config_initialization()
     fi
 
     rm -f /tmp/pending_config_initialization
+    sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
 }
 
 # Restore config-setup post migration hooks from a backup copy
@@ -299,13 +300,14 @@ check_all_config_db_present()
 }
 
 # DB schema is subject to change between two images
-# Perform DB schema migration after loading backup config from previous image
+# Perform DB schema migration after loading backup config/minigraph from previous image
 do_db_migration()
 {
     if [[ -x /usr/local/bin/db_migrator.py ]]; then
         # Migrate the DB to the latest schema version if needed
         /usr/local/bin/db_migrator.py -o migrate
     fi
+    sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
 }
 
 # Perform configuration migration from backup copy.

files/image_config/warmboot-finalizer/finalize-warmboot.sh

@@ -56,7 +56,7 @@ function wait_for_database_service()
     done
 
     # Wait for configDB initialization
-    until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
+    until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]];
         do sleep 1;
     done
 

files/scripts/swss.sh

@@ -65,7 +65,7 @@ function wait_for_database_service()
     done
 
     # Wait for configDB initialization
-    until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
+    until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]];
         do sleep 1;
     done
 }

files/scripts/syncd_common.sh

@@ -63,7 +63,7 @@ function wait_for_database_service()
     done
 
     # Wait for configDB initialization
-    until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
+    until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]];
         do sleep 1;
     done
 }