Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Port 202012 DB migration changes to newer branches #2515

Merged
merged 2 commits into from
Nov 23, 2022
Merged

Port 202012 DB migration changes to newer branches #2515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d43913a
Port 202012 DB migration changes to newer branches
vaibhavhd Nov 22, 2022
6daa3c6
Fix UT failures
vaibhavhd Nov 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 74 additions & 2 deletions scripts/db_migrator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
import re

from sonic_py_common import device_info, logger
from swsssdk import ConfigDBConnector, SonicDBConfig
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
from db_migrator_constants import RESTAPI, TELEMETRY, CONSOLE_SWITCH

INIT_CFG_FILE = '/etc/sonic/init_cfg.json'

Expand Down Expand Up @@ -489,6 +491,61 @@ def migrate_qos_fieldval_reference_format(self):
self.migrate_qos_db_fieldval_reference_remove(qos_table_list, self.configDB, self.configDB.CONFIG_DB, '|')
return True

def migrate_vxlan_config(self):
log.log_notice('Migrate VXLAN table config')
# Collect VXLAN data from config DB
vxlan_data = self.configDB.keys(self.configDB.CONFIG_DB, "VXLAN_TUNNEL*")
if not vxlan_data:
# do nothing if vxlan entries are not present in configdb
return
for vxlan_table in vxlan_data:
vxlan_map_mapping = self.configDB.get_all(self.configDB.CONFIG_DB, vxlan_table)
tunnel_keys = vxlan_table.split(self.configDB.KEY_SEPARATOR)
tunnel_keys[0] = tunnel_keys[0] + "_TABLE"
vxlan_table = self.appDB.get_db_separator(self.appDB.APPL_DB).join(tunnel_keys)
for field, value in vxlan_map_mapping.items():
# add entries from configdb to appdb only when they are missing
if not self.appDB.hexists(self.appDB.APPL_DB, vxlan_table, field):
log.log_notice('Copying vxlan entries from configdb to appdb: updated {} with {}:{}'.format(
vxlan_table, field, value))
self.appDB.set(self.appDB.APPL_DB, vxlan_table, field, value)

def migrate_restapi(self):
# RESTAPI - add missing key
log.log_notice('Migrate RESTAPI configuration')
config = self.configDB.get_entry('RESTAPI', 'config')
if not config:
self.configDB.set_entry("RESTAPI", "config", RESTAPI.get("config"))
certs = self.configDB.get_entry('RESTAPI', 'certs')
if not certs:
self.configDB.set_entry("RESTAPI", "certs", RESTAPI.get("certs"))

def migrate_telemetry(self):
# TELEMETRY - add missing key
log.log_notice('Migrate TELEMETRY configuration')
gnmi = self.configDB.get_entry('TELEMETRY', 'gnmi')
if not gnmi:
self.configDB.set_entry("TELEMETRY", "gnmi", TELEMETRY.get("gnmi"))
certs = self.configDB.get_entry('TELEMETRY', 'certs')
if not certs:
self.configDB.set_entry("TELEMETRY", "certs", TELEMETRY.get("certs"))

def migrate_console_switch(self):
# CONSOLE_SWITCH - add missing key
log.log_notice('Migrate CONSOLE_SWITCH configuration')
console_mgmt = self.configDB.get_entry('CONSOLE_SWITCH', 'console_mgmt')
if not console_mgmt:
self.configDB.set_entry("CONSOLE_SWITCH", "console_mgmt",
CONSOLE_SWITCH.get("console_mgmt"))

def migrate_device_metadata(self):
# DEVICE_METADATA - synchronous_mode entry
log.log_notice('Migrate DEVICE_METADATA missing configuration (synchronous_mode=enable)')
metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
if 'synchronous_mode' not in metadata:
metadata['synchronous_mode'] = 'enable'
self.configDB.set_entry('DEVICE_METADATA', 'localhost', metadata)

def migrate_port_qos_map_global(self):
"""
Generate dscp_to_tc_map for switch.
Expand Down Expand Up @@ -651,10 +708,25 @@ def version_2_0_0(self):

def version_2_0_1(self):
"""
Version 2_0_1.
This is the latest version for 202012 branch
Handle and migrate missing config that results from cross branch upgrade to
202012 as target.
"""
log.log_info('Handling version_2_0_1')
self.migrate_vxlan_config()
self.migrate_restapi()
self.migrate_telemetry()
self.migrate_console_switch()
self.migrate_device_metadata()

self.set_version('version_2_0_2')
return 'version_2_0_2'

def version_2_0_2(self):
"""
Version 2_0_2
This is the latest version for 202012 branch
"""
log.log_info('Handling version_2_0_2')
self.set_version('version_3_0_0')
return 'version_3_0_0'

Expand Down
32 changes: 32 additions & 0 deletions scripts/db_migrator_constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
RESTAPI = {
"config": {
"client_auth": "true",
"log_level": "info",
"allow_insecure": "false"
},
"certs": {
"server_key": "/etc/sonic/credentials/restapiserver.key",
"ca_crt": "/etc/sonic/credentials/AME_ROOT_CERTIFICATE.pem",
"server_crt": "/etc/sonic/credentials/restapiserver.crt",
"client_crt_cname": "client.restapi.sonic.gbl"
}
}

TELEMETRY = {
"gnmi": {
"client_auth": "true",
"log_level": "2",
"port": "50051"
},
"certs": {
"server_key": "/etc/sonic/telemetry/streamingtelemetryserver.key",
"ca_crt": "/etc/sonic/telemetry/dsmsroot.cer",
"server_crt": "/etc/sonic/telemetry/streamingtelemetryserver.cer"
}
}

CONSOLE_SWITCH = {
"console_mgmt": {
"enabled": "no"
}
}
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,8 @@
"default_pfcwd_status": "disable",
"bgp_asn": "65100",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"PORT|Ethernet0": {
"lanes": "0,1,2,3",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,8 @@
"default_pfcwd_status": "disable",
"bgp_asn": "65100",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"PORT|Ethernet0": {
"lanes": "0,1,2,3",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,8 @@
"default_pfcwd_status": "disable",
"bgp_asn": "65100",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"PORT|Ethernet0": {
"lanes": "0,1,2,3",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_5.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,8 @@
"default_pfcwd_status": "disable",
"bgp_asn": "65100",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"PORT|Ethernet0": {
"lanes": "0,1,2,3",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_1_0_6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,8 @@
"default_pfcwd_status": "disable",
"bgp_asn": "65100",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"PORT|Ethernet0": {
"lanes": "0,1,2,3",
Expand Down
3 changes: 2 additions & 1 deletion tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,7 +689,8 @@
"bgp_asn": "65100",
"buffer_model": "dynamic",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"LOSSLESS_TRAFFIC_PATTERN|AZURE": {
"small_packet_percentage": "100",
Expand Down
5 changes: 3 additions & 2 deletions tests/db_migrator_input/config_db/acs-msn2700-t0-version_3_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,7 +689,8 @@
"bgp_asn": "65100",
"buffer_model": "dynamic",
"deployment_id": "1",
"type": "ToRRouter"
"type": "ToRRouter",
"synchronous_mode": "enable"
},
"LOSSLESS_TRAFFIC_PATTERN|AZURE": {
"small_packet_percentage": "100",
Expand Down Expand Up @@ -1016,4 +1017,4 @@
"VERSIONS|DATABASE": {
"VERSION": "version_3_0_3"
}
}
}
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_5.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_1_0_6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,7 @@
"hwsku": "ACS-MSN2700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn2700-r0",
"mac": "00:01:02:03:04:00",
Expand Down
5 changes: 3 additions & 2 deletions tests/db_migrator_input/config_db/acs-msn2700-t1-version_3_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -761,7 +761,8 @@
"bgp_asn": "65100",
"buffer_model": "dynamic",
"deployment_id": "1",
"docker_routing_config_mode": "unified"
"docker_routing_config_mode": "unified",
"synchronous_mode": "enable"
},
"LOSSLESS_TRAFFIC_PATTERN|AZURE": {
"small_packet_percentage": "100",
Expand Down Expand Up @@ -1088,4 +1089,4 @@
"VERSIONS|DATABASE": {
"VERSION": "version_3_0_3"
}
}
}
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_5.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_1_0_6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "ToRRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
5 changes: 3 additions & 2 deletions tests/db_migrator_input/config_db/acs-msn3700-t0-version_3_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,7 +817,8 @@
"buffer_model": "dynamic",
"deployment_id": "1",
"docker_routing_config_mode": "separated",
"cloudtype": "None"
"cloudtype": "None",
"synchronous_mode": "enable"
},
"LOSSLESS_TRAFFIC_PATTERN|AZURE": {
"small_packet_percentage": "100",
Expand Down Expand Up @@ -1902,4 +1903,4 @@
"VERSIONS|DATABASE": {
"VERSION": "version_3_0_3"
}
}
}
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_5.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
1 change: 1 addition & 0 deletions tests/db_migrator_input/config_db/acs-msn3700-t1-version_1_0_6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@
"hwsku": "ACS-MSN3700",
"default_bgp_status": "up",
"type": "LeafRouter",
"synchronous_mode": "enable",
"region": "None",
"hostname": "sonic",
"platform": "x86_64-mlnx_msn3700-r0",
Expand Down
Loading