Skip to content

Commit

Permalink
Merge pull request sonic-net#14 from abdosi/feature
Browse files Browse the repository at this point in the history 
What I did:
Added Support to render Feature Table using Device running metadata
Also added support to render 'has_asic_scope' field of Feature Table.

Why I did:
Details: sonic-net#11796
  • Loading branch information
@abdosi
abdosi authored Sep 22, 2022
2 parents bceb13e + 4d6cad7 commit 6e45acc
Show file tree
Hide file tree
Showing 3 changed files with 670 additions and 72 deletions.
89 changes: 69 additions & 20 deletions scripts/hostcfgd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,30 +147,31 @@ class Feature(object):
"""

self.name = feature_name
self.state = self._get_target_state(feature_cfg.get('state'), device_config or {})
self.state = self._get_feature_table_key_render_value(feature_cfg.get('state'), device_config or {}, ['enabled', 'disabled', 'always_enabled', 'always_disabled'])
self.auto_restart = feature_cfg.get('auto_restart', 'disabled')
self.has_timer = safe_eval(feature_cfg.get('has_timer', 'False'))
self.has_global_scope = safe_eval(feature_cfg.get('has_global_scope', 'True'))
self.has_per_asic_scope = safe_eval(feature_cfg.get('has_per_asic_scope', 'False'))
self.has_per_asic_scope = safe_eval(self._get_feature_table_key_render_value(feature_cfg.get('has_per_asic_scope', 'False'), device_config or {}, ['True', 'False']))

def _get_target_state(self, state_configuration, device_config):
""" Returns the target state for the feature by rendering the state field as J2 template.
def _get_feature_table_key_render_value(self, configuration, device_config, expected_values):
""" Returns the target value for the feature by rendering the configuration as J2 template.
Args:
state_configuration (str): State configuration from CONFIG_DB
deviec_config (dict): DEVICE_METADATA section of CONFIG_DB
configuration (str): Feature Table value from CONFIG_DB for given key
device_config (dict): DEVICE_METADATA section of CONFIG_DB and populated Device Running Metadata
expected_values (list): Expected set of Feature Table value for given key
Returns:
(str): Target feature state
(str): Target feature table value for given key
"""

if state_configuration is None:
if configuration is None:
return None

template = jinja2.Template(state_configuration)
target_state = template.render(device_config)
if target_state not in ('enabled', 'disabled', 'always_enabled', 'always_disabled'):
raise ValueError('Invalid state rendered for feature {}: {}'.format(self.name, target_state))
return target_state
template = jinja2.Template(configuration)
target_value = template.render(device_config)
if target_value not in expected_values:
raise ValueError('Invalid value rendered for feature {}: {}'.format(self.name, target_value))
return target_value

def compare_state(self, feature_name, feature_cfg):
if self.name != feature_name or not isinstance(feature_cfg, dict):
Expand Down Expand Up @@ -198,6 +199,7 @@ class FeatureHandler(object):
self._device_config = device_config
self._cached_config = {}
self.is_multi_npu = device_info.is_multi_npu()
self._device_running_config = device_info.get_device_runtime_metadata()

def handler(self, feature_name, op, feature_cfg):
if not feature_cfg:
Expand All @@ -206,7 +208,11 @@ class FeatureHandler(object):
self._feature_state_table._del(feature_name)
return

feature = Feature(feature_name, feature_cfg, self._device_config)
device_config = {}
device_config.update(self._device_config)
device_config.update(self._device_running_config)

feature = Feature(feature_name, feature_cfg, device_config)
self._cached_config.setdefault(feature_name, Feature(feature_name, {}))

# Change auto-restart configuration first.
Expand All @@ -231,14 +237,18 @@ class FeatureHandler(object):
"""
Summary:
Updates the state field in the FEATURE|* tables as the state field
might have to be rendered based on DEVICE_METADATA table
might have to be rendered based on DEVICE_METADATA table and generated Device Running Metadata
"""
for feature_name in feature_table.keys():
if not feature_name:
syslog.syslog(syslog.LOG_WARNING, "Feature is None")
continue

device_config = {}
device_config.update(self._device_config)
device_config.update(self._device_running_config)

feature = Feature(feature_name, feature_table[feature_name], self._device_config)
feature = Feature(feature_name, feature_table[feature_name], device_config)

self._cached_config.setdefault(feature_name, feature)
self.update_systemd_config(feature)
Expand Down Expand Up @@ -284,8 +294,47 @@ class FeatureHandler(object):
self.disable_feature(feature)
syslog.syslog(syslog.LOG_INFO, "Feature {} is stopped and disabled".format(feature.name))

self.sync_feature_asic_scope(feature)

return True

def sync_feature_asic_scope(self, feature_config):
"""Updates the has_per_asic_scope field in the FEATURE|* tables as the field
might have to be rendered based on DEVICE_METADATA table or Device Running configuration.
Disable the ASIC instance service unit file it the render value is False and update config
Args:
feature: An object represents a feature's configuration in `FEATURE`
table of `CONFIG_DB`.
Returns:
None.
"""

cmds = []
feature_names, feature_suffixes = self.get_multiasic_feature_instances(feature_config, True)
for feature_name in feature_names:
if '@' not in feature_name:
continue
unit_file_state = self.get_systemd_unit_state("{}.{}".format(feature_name, feature_suffixes[-1]))
if not unit_file_state:
continue
if unit_file_state != "disabled" and not feature_config.has_per_asic_scope:
for suffix in reversed(feature_suffixes):
cmds.append("sudo systemctl stop {}.{}".format(feature_name, suffix))
cmds.append("sudo systemctl disable {}.{}".format(feature_name, feature_suffixes[-1]))
cmds.append("sudo systemctl mask {}.{}".format(feature_name, feature_suffixes[-1]))
for cmd in cmds:
syslog.syslog(syslog.LOG_INFO, "Running cmd: '{}'".format(cmd))
try:
run_cmd(cmd, raise_exception=True)
except Exception as err:
syslog.syslog(syslog.LOG_ERR, "Feature '{}.{}' failed to be stopped and disabled"
.format(feature.name, feature_suffixes[-1]))
self.set_feature_state(feature, self.FEATURE_STATE_FAILED)
return
self._config_db.mod_entry('FEATURE', feature_config.name, {'has_per_asic_scope': str(feature_config.has_per_asic_scope)})

def update_systemd_config(self, feature_config):
"""Updates `Restart=` field in feature's systemd configuration file
according to the value of `auto_restart` field in `FEATURE` table of `CONFIG_DB`.
Expand Down Expand Up @@ -324,12 +373,12 @@ class FeatureHandler(object):
except Exception as err:
syslog.syslog(syslog.LOG_ERR, "Failed to reload systemd configuration files!")

def get_multiasic_feature_instances(self, feature):
def get_multiasic_feature_instances(self, feature, all_instance=False):
# Create feature name suffix depending feature is running in host or namespace or in both
feature_names = (
([feature.name] if feature.has_global_scope or not self.is_multi_npu else []) +
([(feature.name + '@' + str(asic_inst)) for asic_inst in range(device_info.get_num_npus())
if feature.has_per_asic_scope and self.is_multi_npu])
if self.is_multi_npu and (all_instance or feature.has_per_asic_scope)])
)

if not feature_names:
Expand Down Expand Up @@ -359,7 +408,7 @@ class FeatureHandler(object):
for feature_name in feature_names:
# Check if it is already enabled, if yes skip the system call
unit_file_state = self.get_systemd_unit_state("{}.{}".format(feature_name, feature_suffixes[-1]))
if unit_file_state == "enabled":
if unit_file_state == "enabled" or not unit_file_state:
continue

for suffix in feature_suffixes:
Expand Down Expand Up @@ -389,7 +438,7 @@ class FeatureHandler(object):
for feature_name in feature_names:
# Check if it is already disabled, if yes skip the system call
unit_file_state = self.get_systemd_unit_state("{}.{}".format(feature_name, feature_suffixes[-1]))
if unit_file_state in ("disabled", "masked"):
if unit_file_state in ("disabled", "masked") or not unit_file_state:
continue

for suffix in reversed(feature_suffixes):
Expand Down
125 changes: 74 additions & 51 deletions tests/hostcfgd/hostcfgd_test.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import sys
import swsscommon as swsscommon_package
from sonic_py_common import device_info
from swsscommon import swsscommon

from parameterized import parameterized
Expand Down Expand Up @@ -46,7 +47,7 @@ def checks_config_table(self, feature_table, expected_table):

return True if not ddiff else False

def checks_systemd_config_file(self, feature_table):
def checks_systemd_config_file(self, feature_table, feature_systemd_name_map=None):
"""Checks whether the systemd configuration file of each feature was created or not
and whether the `Restart=` field in the file is set correctly or not.
Expand All @@ -69,13 +70,16 @@ def checks_systemd_config_file(self, feature_table):
elif "disabled" in auto_restart_status:
auto_restart_status = "disabled"

feature_systemd_config_file_path = systemd_config_file_path.format(feature_name)
is_config_file_existing = os.path.exists(feature_systemd_config_file_path)
assert is_config_file_existing, "Systemd configuration file of feature '{}' does not exist!".format(feature_name)
feature_systemd_list = feature_systemd_name_map[feature_name] if feature_systemd_name_map else [feature_name]

with open(feature_systemd_config_file_path) as systemd_config_file:
status = systemd_config_file.read().strip()
assert status == '[Service]\nRestart={}'.format(truth_table[auto_restart_status])
for feature_systemd in feature_systemd_list:
feature_systemd_config_file_path = systemd_config_file_path.format(feature_systemd)
is_config_file_existing = os.path.exists(feature_systemd_config_file_path)
assert is_config_file_existing, "Systemd configuration file of feature '{}' does not exist!".format(feature_systemd)

with open(feature_systemd_config_file_path) as systemd_config_file:
status = systemd_config_file.read().strip()
assert status == '[Service]\nRestart={}'.format(truth_table[auto_restart_status])

def get_state_db_set_calls(self, feature_table):
"""Returns a Mock call objects which recorded the `set` calls to `FEATURE` table in `STATE_DB`.
Expand Down Expand Up @@ -120,31 +124,42 @@ def test_sync_state_field(self, test_scenario_name, config_data, fs):
MockConfigDb.set_config_db(config_data['config_db'])
feature_state_table_mock = mock.Mock()
with mock.patch('hostcfgd.subprocess') as mocked_subprocess:
popen_mock = mock.Mock()
attrs = config_data['popen_attributes']
popen_mock.configure_mock(**attrs)
mocked_subprocess.Popen.return_value = popen_mock

device_config = {}
device_config['DEVICE_METADATA'] = MockConfigDb.CONFIG_DB['DEVICE_METADATA']
feature_handler = hostcfgd.FeatureHandler(MockConfigDb(), feature_state_table_mock, device_config)

feature_table = MockConfigDb.CONFIG_DB['FEATURE']
feature_handler.sync_state_field(feature_table)

is_any_difference = self.checks_config_table(MockConfigDb.get_config_db()['FEATURE'],
config_data['expected_config_db']['FEATURE'])
assert is_any_difference, "'FEATURE' table in 'CONFIG_DB' is modified unexpectedly!"

feature_table_state_db_calls = self.get_state_db_set_calls(feature_table)

self.checks_systemd_config_file(config_data['config_db']['FEATURE'])
mocked_subprocess.check_call.assert_has_calls(config_data['enable_feature_subprocess_calls'],
any_order=True)
mocked_subprocess.check_call.assert_has_calls(config_data['daemon_reload_subprocess_call'],
any_order=True)
feature_state_table_mock.set.assert_has_calls(feature_table_state_db_calls)
self.checks_systemd_config_file(config_data['config_db']['FEATURE'])
with mock.patch("sonic_py_common.device_info.get_device_runtime_metadata", return_value=config_data['device_runtime_metadata']):
with mock.patch("sonic_py_common.device_info.is_multi_npu", return_value=True if 'num_npu' in config_data else False):
with mock.patch("sonic_py_common.device_info.get_num_npus", return_value=config_data['num_npu'] if 'num_npu' in config_data else 1):
popen_mock = mock.Mock()
attrs = config_data['popen_attributes']
popen_mock.configure_mock(**attrs)
mocked_subprocess.Popen.return_value = popen_mock

device_config = {}
device_config['DEVICE_METADATA'] = MockConfigDb.CONFIG_DB['DEVICE_METADATA']
device_config.update(config_data['device_runtime_metadata'])

feature_handler = hostcfgd.FeatureHandler(MockConfigDb(), feature_state_table_mock, device_config)

feature_table = MockConfigDb.CONFIG_DB['FEATURE']
feature_handler.sync_state_field(feature_table)

feature_systemd_name_map = {}
for feature_name in feature_table.keys():
feature = hostcfgd.Feature(feature_name, feature_table[feature_name], device_config)
feature_names, _ = feature_handler.get_multiasic_feature_instances(feature)
feature_systemd_name_map[feature_name] = feature_names

is_any_difference = self.checks_config_table(MockConfigDb.get_config_db()['FEATURE'],
config_data['expected_config_db']['FEATURE'])
assert is_any_difference, "'FEATURE' table in 'CONFIG_DB' is modified unexpectedly!"

feature_table_state_db_calls = self.get_state_db_set_calls(feature_table)

self.checks_systemd_config_file(config_data['config_db']['FEATURE'], feature_systemd_name_map)
mocked_subprocess.check_call.assert_has_calls(config_data['enable_feature_subprocess_calls'],
any_order=True)
mocked_subprocess.check_call.assert_has_calls(config_data['daemon_reload_subprocess_call'],
any_order=True)
feature_state_table_mock.set.assert_has_calls(feature_table_state_db_calls)
self.checks_systemd_config_file(config_data['config_db']['FEATURE'], feature_systemd_name_map)

@parameterized.expand(HOSTCFGD_TEST_VECTOR)
@patchfs
Expand All @@ -165,25 +180,33 @@ def test_handler(self, test_scenario_name, config_data, fs):
MockConfigDb.set_config_db(config_data['config_db'])
feature_state_table_mock = mock.Mock()
with mock.patch('hostcfgd.subprocess') as mocked_subprocess:
popen_mock = mock.Mock()
attrs = config_data['popen_attributes']
popen_mock.configure_mock(**attrs)
mocked_subprocess.Popen.return_value = popen_mock

device_config = {}
device_config['DEVICE_METADATA'] = MockConfigDb.CONFIG_DB['DEVICE_METADATA']
feature_handler = hostcfgd.FeatureHandler(MockConfigDb(), feature_state_table_mock, device_config)

feature_table = MockConfigDb.CONFIG_DB['FEATURE']

for feature_name, feature_config in feature_table.items():
feature_handler.handler(feature_name, 'SET', feature_config)

self.checks_systemd_config_file(config_data['config_db']['FEATURE'])
mocked_subprocess.check_call.assert_has_calls(config_data['enable_feature_subprocess_calls'],
any_order=True)
mocked_subprocess.check_call.assert_has_calls(config_data['daemon_reload_subprocess_call'],
any_order=True)
with mock.patch("sonic_py_common.device_info.get_device_runtime_metadata", return_value=config_data['device_runtime_metadata']):
with mock.patch("sonic_py_common.device_info.is_multi_npu", return_value=True if 'num_npu' in config_data else False):
with mock.patch("sonic_py_common.device_info.get_num_npus", return_value=config_data['num_npu'] if 'num_npu' in config_data else 1):
popen_mock = mock.Mock()
attrs = config_data['popen_attributes']
popen_mock.configure_mock(**attrs)
mocked_subprocess.Popen.return_value = popen_mock

device_config = {}
device_config['DEVICE_METADATA'] = MockConfigDb.CONFIG_DB['DEVICE_METADATA']
device_config.update(config_data['device_runtime_metadata'])
feature_handler = hostcfgd.FeatureHandler(MockConfigDb(), feature_state_table_mock, device_config)

feature_table = MockConfigDb.CONFIG_DB['FEATURE']

feature_systemd_name_map = {}
for feature_name, feature_config in feature_table.items():
feature_handler.handler(feature_name, 'SET', feature_config)
feature = hostcfgd.Feature(feature_name, feature_table[feature_name], device_config)
feature_names, _ = feature_handler.get_multiasic_feature_instances(feature)
feature_systemd_name_map[feature_name] = feature_names

self.checks_systemd_config_file(config_data['config_db']['FEATURE'], feature_systemd_name_map)
mocked_subprocess.check_call.assert_has_calls(config_data['enable_feature_subprocess_calls'],
any_order=True)
mocked_subprocess.check_call.assert_has_calls(config_data['daemon_reload_subprocess_call'],
any_order=True)

def test_feature_config_parsing(self):
swss_feature = hostcfgd.Feature('swss', {
Expand Down
Loading

0 comments on commit 6e45acc

Please sign in to comment.