From ea7ab45f2d051a2e0160c78a1561865a5e42677b Mon Sep 17 00:00:00 2001 From: Deepak Nayak Date: Wed, 27 Jul 2022 12:23:11 +0530 Subject: [PATCH 1/5] CORTX-32348: Part 1 - cfgen changes Signed-off-by: Deepak Nayak --- cfgen/cfgen | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfgen/cfgen b/cfgen/cfgen index 6f6a119c0..bb765a168 100755 --- a/cfgen/cfgen +++ b/cfgen/cfgen @@ -189,6 +189,8 @@ nodes: meta_data: # device path for meta-data; # optional, Motr will use "/var/motr/m0d-/" # by default + log: [ ] # e.g. [ "/dev/loop0", "/dev/loop1", "/dev/loop2" ] + # Empty list means no log device. data: [ ] # e.g. [ "/dev/loop0", "/dev/loop1", "/dev/loop2" ] # Empty list means no IO service. m0_clients: @@ -437,6 +439,9 @@ def enrich_cluster_desc(desc: Dict[str, Any], mock_p: bool) -> None: if 'meta_data' in m0d['io_disks']: if m0d['io_disks']['meta_data'] is None: del m0d['io_disks']['meta_data'] + if 'log' in m0d['io_disks']: + if m0d['io_disks']['log'] is None: + del m0d['io_disks']['log'] m0d['_io_disks'] = get_disks_from_cdf(node['hostname'], mock_p, m0d['io_disks']['data']) @@ -514,8 +519,15 @@ Make sure the value of data_iface in the CDF is correct.""") if 'meta_data' in d: _assert(d['meta_data'], f"Node {name}: 'io_disks.meta_data'" 'must not be an empty string') - _assert(d['meta_data'] not in d['data'], f"Node {name}: " - 'Meta-data disk must not belong io_disks.data') + _assert(d['meta_data'] not in d['data'] or d['log'], + f"Node {name}: Meta-data disk must not belong " + 'in io_disks.data or io_disks.log') + if 'log' in d: + _assert('' not in d['log'], f"Node {name}: Empty strings " + "in 'io_disks.log' are not allowed") + _assert(d['log'] not in d['data'] or d['meta_data'], + f"Node {name}: Log disk must not belong " + 'in io_disks.data or io_disks.meta_data') disks: List[Disk] = [] for m0d in node['m0_servers']: From c48a4582c387ed787319e46d97b36e8e6feb6b6e Mon Sep 17 00:00:00 2001 From: Deepak Nayak Date: Wed, 27 Jul 2022 22:22:43 +0530 Subject: [PATCH 2/5] CORTX-32348: Parts 1 and 2: Changes in cfgen and update-consul-conf Signed-off-by: Deepak Nayak --- cfgen/cfgen | 18 ++++++++++++++---- utils/update-consul-conf | 12 ++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cfgen/cfgen b/cfgen/cfgen index bb765a168..2c1abe9be 100755 --- a/cfgen/cfgen +++ b/cfgen/cfgen @@ -1217,13 +1217,14 @@ class ConfProcess(ToDhall): def __init__(self, nr_cpu: int, memsize_MB: int, endpoint, services: List[Oid], meta_data: str = None, - proc_name: str = None): + log: List[str] = [], proc_name: str = None): _assert(nr_cpu > 0, 'nr_cpu must be positive') self.nr_cpu = nr_cpu _assert(memsize_MB > 0, 'memsize_MB must be positive') self.memsize_MB = memsize_MB self.endpoint = endpoint self.meta_data = meta_data + self.log = log self.proc_name = proc_name _assert(all(x.type is ObjT.service for x in services), 'All services must be of ObjT.service type') @@ -1294,13 +1295,16 @@ class ConfProcess(ToDhall): portalgroup=portal) proc_id = new_oid(ObjT.process) meta_data = None + log = [] if proc_desc is not None and proc_t is ProcT.m0_server: meta_data = proc_desc['io_disks'].get('meta_data') + log = proc_desc['io_disks'].get('log') m0conf[proc_id] = cls(nr_cpu=facts['processorcount'], memsize_MB=facts['_memsize_MB'], endpoint=ep, meta_data=meta_data, + log=log, services=[], proc_name=proc_name) m0conf[parent].processes.append(proc_id) @@ -2453,7 +2457,8 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: ConsulService = NamedTuple('ConsulService', [ ('node_name', str), ('proc_id', Oid), ('ep', Any), - ('svc_id', Oid), ('stype', str), ('meta_data', Optional[str])]) + ('svc_id', Oid), ('stype', str), ('meta_data', Optional[str]), + ('log', Optional[List[str]])]) def processes() -> Iterator[ConsulService]: for node_id, node in m0conf.items(): @@ -2469,7 +2474,8 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: ep=proc.endpoint, svc_id=svc_id, stype=stype, - meta_data=proc.meta_data) + meta_data=proc.meta_data, + log=proc.log) # Add Motr clients to consul kv if proc_id in cluster.m0_clients: yield ConsulService(node_name=node.name, @@ -2477,7 +2483,8 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: ep=proc.endpoint, svc_id=cluster.m0_clients[proc_id], stype=m0conf[proc_id].proc_name, - meta_data=None) + meta_data=None, + log=[]) def sns_pools() -> List[Oid]: for root_id, root in m0conf.items(): @@ -2525,6 +2532,9 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/' f'meta_data', x.meta_data) for x in processes() if x.stype == 'ios' and x.meta_data], + *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/' + f'log', x.log) + for x in processes() if x.stype == 'ios' and x.log], *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/endpoint', str(x.ep)) for x in processes()], diff --git a/utils/update-consul-conf b/utils/update-consul-conf index aee43e4a5..190be29bb 100755 --- a/utils/update-consul-conf +++ b/utils/update-consul-conf @@ -136,6 +136,16 @@ get_ios_meta_data_from_kv_file() { eval $cmd || true } +get_ios_log_from_kv_file() { + local process_fidk=$1 + local key="m0conf/nodes/$(get_node_name)/processes/$process_fidk/log" + + local cmd="jq -r '.[] | select(.key==\"$key\") | + .value' $kv_file | jq -r '.[]'" + + eval $cmd || true +} + get_service_ids_from_kv_file() { local filter=$1 local key="m0conf/nodes/$(get_node_name)/processes/*" @@ -400,6 +410,8 @@ append_ios_svc() { local port=$(get_service_port $ep) local host=$(get_service_host $addr) local meta_data=$(get_ios_meta_data_from_kv_file $id) + local log=$(get_ios_log_from_kv_file $id) + SVCS_CONF+="${SVCS_CONF:+,}{ \"id\": \"$id\", \"name\": \"ios\", From 4361ce77de52ec8ef8dc400efb02c26b619fce99 Mon Sep 17 00:00:00 2001 From: Deepak Nayak Date: Thu, 28 Jul 2022 18:20:57 +0530 Subject: [PATCH 3/5] CORTX-32348: Changes to handle both log device path and size strings Signed-off-by: Deepak Nayak --- cfgen/cfgen | 19 +++++++++++++++---- utils/update-consul-conf | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/cfgen/cfgen b/cfgen/cfgen index 2c1abe9be..ed69fc7cc 100755 --- a/cfgen/cfgen +++ b/cfgen/cfgen @@ -448,6 +448,13 @@ def enrich_cluster_desc(desc: Dict[str, Any], mock_p: bool) -> None: if not m0d['_io_disks']: m0d['_io_disks'] = get_disks_ssh(node['hostname'], mock_p, m0d['io_disks']['data']) + + m0d['_log_disks'] = get_disks_from_cdf(node['hostname'], mock_p, + m0d['io_disks']['log']) + + if not m0d['_log_disks']: + m0d['_log_disks'] = get_disks_ssh(node['hostname'], mock_p, + m0d['io_disks']['log']) if 'profiles' not in desc: sns_pools = [pool['name'] for pool in desc['pools'] @@ -1217,7 +1224,7 @@ class ConfProcess(ToDhall): def __init__(self, nr_cpu: int, memsize_MB: int, endpoint, services: List[Oid], meta_data: str = None, - log: List[str] = [], proc_name: str = None): + log: List[Disk] = [], proc_name: str = None): _assert(nr_cpu > 0, 'nr_cpu must be positive') self.nr_cpu = nr_cpu _assert(memsize_MB > 0, 'memsize_MB must be positive') @@ -1298,7 +1305,7 @@ class ConfProcess(ToDhall): log = [] if proc_desc is not None and proc_t is ProcT.m0_server: meta_data = proc_desc['io_disks'].get('meta_data') - log = proc_desc['io_disks'].get('log') + log = proc_desc['_log_disks'] m0conf[proc_id] = cls(nr_cpu=facts['processorcount'], memsize_MB=facts['_memsize_MB'], @@ -2458,7 +2465,7 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: ConsulService = NamedTuple('ConsulService', [ ('node_name', str), ('proc_id', Oid), ('ep', Any), ('svc_id', Oid), ('stype', str), ('meta_data', Optional[str]), - ('log', Optional[List[str]])]) + ('log', Optional[List[Disk]])]) def processes() -> Iterator[ConsulService]: for node_id, node in m0conf.items(): @@ -2533,7 +2540,11 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: f'meta_data', x.meta_data) for x in processes() if x.stype == 'ios' and x.meta_data], *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/' - f'log', x.log) + f'log', + json.dumps({ + 'path': x.log[0].path, + 'size': x.log[0].size + })) for x in processes() if x.stype == 'ios' and x.log], *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/endpoint', str(x.ep)) diff --git a/utils/update-consul-conf b/utils/update-consul-conf index 190be29bb..1798f9153 100755 --- a/utils/update-consul-conf +++ b/utils/update-consul-conf @@ -141,7 +141,7 @@ get_ios_log_from_kv_file() { local key="m0conf/nodes/$(get_node_name)/processes/$process_fidk/log" local cmd="jq -r '.[] | select(.key==\"$key\") | - .value' $kv_file | jq -r '.[]'" + .value' $kv_file" eval $cmd || true } @@ -410,7 +410,8 @@ append_ios_svc() { local port=$(get_service_port $ep) local host=$(get_service_host $addr) local meta_data=$(get_ios_meta_data_from_kv_file $id) - local log=$(get_ios_log_from_kv_file $id) + local log_path=$(get_ios_log_from_kv_file $id | jq -r .path) + local log_size=$(get_ios_log_from_kv_file $id | jq -r .size) SVCS_CONF+="${SVCS_CONF:+,}{ \"id\": \"$id\", @@ -446,6 +447,16 @@ EOF cat < /dev/null MOTR_NODE_UUID='$UUID' EOF + if [[ $log_path ]]; then + cat < /dev/null +MOTR_LOG_PATH='$log_path' +EOF + fi + if [[ $log_size ]]; then + cat < /dev/null +MOTR_LOG_SIZE='$log_size' +EOF + fi install_motr_conf /tmp/m0d-$fid } From 425f2e1f1f5429bbcf4d7932967e8731c5fa1a35 Mon Sep 17 00:00:00 2001 From: Deepak Nayak Date: Thu, 28 Jul 2022 19:31:11 +0530 Subject: [PATCH 4/5] CORTX-32348: Addressing pylinting typos in cfgen Signed-off-by: Deepak Nayak --- cfgen/cfgen | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cfgen/cfgen b/cfgen/cfgen index ed69fc7cc..817636397 100755 --- a/cfgen/cfgen +++ b/cfgen/cfgen @@ -448,13 +448,13 @@ def enrich_cluster_desc(desc: Dict[str, Any], mock_p: bool) -> None: if not m0d['_io_disks']: m0d['_io_disks'] = get_disks_ssh(node['hostname'], mock_p, m0d['io_disks']['data']) - + m0d['_log_disks'] = get_disks_from_cdf(node['hostname'], mock_p, - m0d['io_disks']['log']) + m0d['io_disks']['log']) if not m0d['_log_disks']: m0d['_log_disks'] = get_disks_ssh(node['hostname'], mock_p, - m0d['io_disks']['log']) + m0d['io_disks']['log']) if 'profiles' not in desc: sns_pools = [pool['name'] for pool in desc['pools'] @@ -2540,10 +2540,10 @@ def generate_consul_kv(cluster: Cluster, dhall_dir: str) -> str: f'meta_data', x.meta_data) for x in processes() if x.stype == 'ios' and x.meta_data], *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/' - f'log', + f'log', json.dumps({ 'path': x.log[0].path, - 'size': x.log[0].size + 'size': x.log[0].size })) for x in processes() if x.stype == 'ios' and x.log], *[(f'm0conf/nodes/{x.node_name}/processes/{x.proc_id.fidk}/endpoint', From e883feaf53242a4eed118ca97fa8e420df2a01a3 Mon Sep 17 00:00:00 2001 From: Deepak Nayak Date: Fri, 29 Jul 2022 21:54:53 +0530 Subject: [PATCH 5/5] CORTX-32348: Validation checks for disks between log/meta_data/data Signed-off-by: Deepak Nayak --- cfgen/cfgen | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cfgen/cfgen b/cfgen/cfgen index 817636397..2bd5c0eb2 100755 --- a/cfgen/cfgen +++ b/cfgen/cfgen @@ -516,6 +516,7 @@ Make sure the value of data_iface in the CDF is correct.""") f'Node {name}: Too many confd services: {nr_confds} >= 2') total_nr_confds += nr_confds + disks_list = [] for m0d in node['m0_servers']: d = m0d['io_disks'] _assert(m0d['runs_confd'] or d['data'], @@ -523,18 +524,20 @@ Make sure the value of data_iface in the CDF is correct.""") "'io_disks.data' must be set") _assert('' not in d['data'], f"Node {name}: Empty strings " "in 'io_disks.data' are not allowed") + for x in d['data']: + disks_list.append(x['path']) if 'meta_data' in d: _assert(d['meta_data'], f"Node {name}: 'io_disks.meta_data'" 'must not be an empty string') - _assert(d['meta_data'] not in d['data'] or d['log'], - f"Node {name}: Meta-data disk must not belong " - 'in io_disks.data or io_disks.log') + disks_list.append(d['meta_data']) if 'log' in d: _assert('' not in d['log'], f"Node {name}: Empty strings " "in 'io_disks.log' are not allowed") - _assert(d['log'] not in d['data'] or d['meta_data'], - f"Node {name}: Log disk must not belong " - 'in io_disks.data or io_disks.meta_data') + for x in d['log']: + disks_list.append(x['path']) + + _assert(all_unique(disks_list), + f'Node {name}: Disk repeated in log / meta_data / data!') disks: List[Disk] = [] for m0d in node['m0_servers']: